Merge pull request #40342 from BerriAI/litellm_prompt_cache_key_session_id

fix(anthropic): key the /v1/messages prompt cache on Claude Code's session_id only

litellm/llms/anthropic/experimental_pass_through/utils.py

@@ -3,6 +3,8 @@ from collections.abc import Mapping
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Final
 
+from pydantic import BaseModel, ConfigDict, ValidationError
+
 import litellm
 from litellm.types.utils import ModelInfo
 
@@ -21,10 +23,27 @@ _EFFORT_DEGRADATION_CHAIN: Final[Mapping[str, tuple[str, ...]]] = MappingProxyTy
 _THINKING_OFF: Final = "none"
 
 
+class _ClaudeCodeUserId(BaseModel):
+    """The JSON Claude Code packs into ``metadata.user_id``; only ``session_id`` is per conversation."""
+
+    model_config = ConfigDict(frozen=True)
+
+    session_id: str
+
+
 def prompt_cache_key_from_user_id(user_id: object) -> str | None:
-    if user_id is None:
+    """The per-session key Claude Code carries inside ``metadata.user_id``, or nothing.
+
+    Anthropic defines ``user_id`` as an opaque end-user id, so a plain string names a person, not
+    a conversation. Keying the provider cache on it pins every parallel session and subagent of that
+    person to one slot, which caches worse than the provider's own prompt-prefix hashing does.
+    """
+    if not isinstance(user_id, str):
+        return None
+    try:
+        return _ClaudeCodeUserId.model_validate_json(user_id).session_id[:OPENAI_MAX_PROMPT_CACHE_KEY_LENGTH] or None
+    except ValidationError:
         return None
-    return str(user_id)[:OPENAI_MAX_PROMPT_CACHE_KEY_LENGTH] or None
 
 
 def litellm_logging_obj_from_kwargs(kwargs: Mapping[str, object]) -> "LiteLLMLoggingObject | None":

tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py

@@ -1,4 +1,5 @@
 import base64
+import json
 from typing import Any, Final, cast
 
 import pytest
@@ -724,9 +725,14 @@ def test_translate_anthropic_to_openai_orders_top_level_and_midturn_system():
     ]
 
 
-def _translate_with_metadata(
-    model: str, metadata: dict[str, str], custom_llm_provider: str | None
-) -> dict[str, Any]:
+def _claude_code_user_id(session_id: str) -> str:
+    return json.dumps({"device_id": "d" * 64, "account_uuid": "", "session_id": session_id})
+
+
+CLAUDE_CODE_USER_ID: Final = _claude_code_user_id("session-abc")
+
+
+def _translate_with_metadata(model: str, metadata: dict[str, str], custom_llm_provider: str | None) -> dict[str, Any]:
     openai_request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
         anthropic_message_request={
             "model": model,
@@ -739,23 +745,51 @@ def _translate_with_metadata(
     return cast(dict[str, Any], openai_request)
 
 
-def test_translate_anthropic_to_openai_maps_user_id_to_prompt_cache_key_for_openai():
-    openai_request = _translate_with_metadata("openai/gpt-5.6-luna", {"user_id": "session-abc"}, "openai")
-    assert openai_request["user"] == "session-abc"
+def test_translate_anthropic_to_openai_maps_claude_code_session_id_to_prompt_cache_key_for_openai():
+    openai_request = _translate_with_metadata("openai/gpt-5.6-luna", {"user_id": CLAUDE_CODE_USER_ID}, "openai")
+    assert openai_request["user"] == CLAUDE_CODE_USER_ID
     assert openai_request["prompt_cache_key"] == "session-abc"
 
 
-def test_translate_anthropic_to_openai_truncates_prompt_cache_key_but_keeps_full_user():
-    long_id = "".join(str(i % 10) for i in range(100))
-    openai_request = _translate_with_metadata("openai/gpt-5.6-luna", {"user_id": long_id}, "openai")
-    assert openai_request["user"] == long_id
-    assert openai_request["prompt_cache_key"] == long_id[:64]
-    assert len(openai_request["prompt_cache_key"]) == 64
+def test_translate_anthropic_to_openai_gives_each_claude_code_session_its_own_prompt_cache_key():
+    """BerriAI/litellm#39145: the first 64 chars of Claude Code's user_id are the per-install device_id."""
+    keys = tuple(
+        _translate_with_metadata("openai/gpt-5.6-luna", {"user_id": _claude_code_user_id(session_id)}, "openai")[
+            "prompt_cache_key"
+        ]
+        for session_id in ("11111111-1111-1111-1111-111111111111", "22222222-2222-2222-2222-222222222222")
+    )
+    assert keys == ("11111111-1111-1111-1111-111111111111", "22222222-2222-2222-2222-222222222222")
+
+
+def test_translate_anthropic_to_openai_truncates_long_session_id_to_openai_limit():
+    long_session_id = "".join(str(i % 10) for i in range(100))
+    openai_request = _translate_with_metadata(
+        "openai/gpt-5.6-luna", {"user_id": _claude_code_user_id(long_session_id)}, "openai"
+    )
+    assert openai_request["prompt_cache_key"] == long_session_id[:64]
+
+
+@pytest.mark.parametrize(
+    "user_id",
+    [
+        "alice",
+        "".join(str(i % 10) for i in range(100)),
+        json.dumps({"device_id": "d" * 64, "account_uuid": ""}),
+        json.dumps({"session_id": ""}),
+        json.dumps({"session_id": 123}),
+        "{not json",
+    ],
+)
+def test_translate_anthropic_to_openai_keeps_plain_user_id_off_prompt_cache_key(user_id: str):
+    openai_request = _translate_with_metadata("openai/gpt-5.6-luna", {"user_id": user_id}, "openai")
+    assert openai_request["user"] == user_id
+    assert "prompt_cache_key" not in openai_request
 
 
 @pytest.mark.parametrize("model", ["azure/my-gpt-5-deployment", "my-gpt-5-deployment"])
 def test_translate_anthropic_to_openai_sets_prompt_cache_key_for_azure(model: str):
-    openai_request = _translate_with_metadata(model, {"user_id": "session-abc"}, "azure")
+    openai_request = _translate_with_metadata(model, {"user_id": CLAUDE_CODE_USER_ID}, "azure")
     assert openai_request["prompt_cache_key"] == "session-abc"
 
 
@@ -772,8 +806,8 @@ def test_translate_anthropic_to_openai_sets_prompt_cache_key_for_azure(model: st
 def test_translate_anthropic_to_openai_skips_prompt_cache_key_when_provider_lacks_it(
     model: str, custom_llm_provider: str
 ):
-    openai_request = _translate_with_metadata(model, {"user_id": "session-abc"}, custom_llm_provider)
-    assert openai_request["user"] == "session-abc"
+    openai_request = _translate_with_metadata(model, {"user_id": CLAUDE_CODE_USER_ID}, custom_llm_provider)
+    assert openai_request["user"] == CLAUDE_CODE_USER_ID
     assert "prompt_cache_key" not in openai_request
 
 
@@ -781,14 +815,14 @@ def test_translate_anthropic_to_openai_skips_prompt_cache_key_for_chained_litell
     assert "prompt_cache_key" in litellm.get_supported_openai_params(
         model="xai", custom_llm_provider="litellm_proxy"
     )
-    openai_request = _translate_with_metadata("litellm_proxy/xai", {"user_id": "session-abc"}, "litellm_proxy")
-    assert openai_request["user"] == "session-abc"
+    openai_request = _translate_with_metadata("litellm_proxy/xai", {"user_id": CLAUDE_CODE_USER_ID}, "litellm_proxy")
+    assert openai_request["user"] == CLAUDE_CODE_USER_ID
     assert "prompt_cache_key" not in openai_request
 
 
 def test_translate_anthropic_to_openai_skips_prompt_cache_key_without_provider():
-    openai_request = _translate_with_metadata("openai/gpt-5.6-luna", {"user_id": "session-abc"}, None)
-    assert openai_request["user"] == "session-abc"
+    openai_request = _translate_with_metadata("openai/gpt-5.6-luna", {"user_id": CLAUDE_CODE_USER_ID}, None)
+    assert openai_request["user"] == CLAUDE_CODE_USER_ID
     assert "prompt_cache_key" not in openai_request
 
 

tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_prompt_cache_key.py

@@ -1,3 +1,4 @@
+import json
 import os
 import sys
 
@@ -10,6 +11,7 @@ from litellm.llms.anthropic.experimental_pass_through.adapters.handler import (
 )
 
 MESSAGES = [{"role": "user", "content": "hello"}]
+CLAUDE_CODE_USER_ID = json.dumps({"device_id": "d" * 64, "account_uuid": "", "session_id": "session-abc"})
 
 
 def _prepare(model: str, extra_kwargs: dict[str, object], thinking: dict[str, object] | None = None):
@@ -17,7 +19,7 @@ def _prepare(model: str, extra_kwargs: dict[str, object], thinking: dict[str, ob
         max_tokens=1024,
         messages=MESSAGES,
         model=model,
-        metadata={"user_id": "session-abc"},
+        metadata={"user_id": CLAUDE_CODE_USER_ID},
         thinking=thinking,
         extra_kwargs=extra_kwargs,
     )
@@ -26,7 +28,7 @@ def _prepare(model: str, extra_kwargs: dict[str, object], thinking: dict[str, ob
 
 def test_prepare_completion_kwargs_derives_prompt_cache_key_for_openai_provider():
     completion_kwargs = _prepare("openai/gpt-5.6-luna", {"custom_llm_provider": "openai"})
-    assert completion_kwargs["user"] == "session-abc"
+    assert completion_kwargs["user"] == CLAUDE_CODE_USER_ID
     assert completion_kwargs["prompt_cache_key"] == "session-abc"
 
 
@@ -35,7 +37,7 @@ def test_prepare_completion_kwargs_prefers_explicit_prompt_cache_key_over_derive
         "openai/gpt-5.6-luna",
         {"custom_llm_provider": "openai", "prompt_cache_key": "explicit-key"},
     )
-    assert completion_kwargs["user"] == "session-abc"
+    assert completion_kwargs["user"] == CLAUDE_CODE_USER_ID
     assert completion_kwargs["prompt_cache_key"] == "explicit-key"
 
 
@@ -50,13 +52,13 @@ def test_prepare_completion_kwargs_skips_prompt_cache_key_without_provider_suppo
     model: str, extra_kwargs: dict[str, object]
 ):
     completion_kwargs = _prepare(model, extra_kwargs)
-    assert completion_kwargs["user"] == "session-abc"
+    assert completion_kwargs["user"] == CLAUDE_CODE_USER_ID
     assert "prompt_cache_key" not in completion_kwargs
 
 
 def test_prepare_completion_kwargs_skips_prompt_cache_key_for_chained_litellm_proxy():
     completion_kwargs = _prepare("litellm_proxy/xai", {"custom_llm_provider": "litellm_proxy"})
-    assert completion_kwargs["user"] == "session-abc"
+    assert completion_kwargs["user"] == CLAUDE_CODE_USER_ID
     assert "prompt_cache_key" not in completion_kwargs
 
 

tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py

@@ -16,6 +16,7 @@ from litellm.llms.anthropic.experimental_pass_through.responses_adapters.handler
 )
 
 MESSAGES = [{"role": "user", "content": "hello"}]
+CLAUDE_CODE_USER_ID = json.dumps({"device_id": "d" * 64, "account_uuid": "", "session_id": "session-abc"})
 
 RESPONSES_SSE_BODY = (
     b"event: response.created\n"
@@ -30,7 +31,19 @@ RESPONSES_SSE_BODY = (
 )
 
 
-def test_build_responses_kwargs_derives_prompt_cache_key_from_user_id():
+def test_build_responses_kwargs_derives_prompt_cache_key_from_claude_code_session_id():
+    responses_kwargs = _build_responses_kwargs(
+        max_tokens=1024,
+        messages=MESSAGES,
+        model="openai/gpt-5.6-luna",
+        metadata={"user_id": CLAUDE_CODE_USER_ID},
+        extra_kwargs={"custom_llm_provider": "openai"},
+    )
+    assert responses_kwargs["user"] == CLAUDE_CODE_USER_ID[:64]
+    assert responses_kwargs["prompt_cache_key"] == "session-abc"
+
+
+def test_build_responses_kwargs_sets_no_prompt_cache_key_for_plain_user_id():
     responses_kwargs = _build_responses_kwargs(
         max_tokens=1024,
         messages=MESSAGES,
@@ -39,7 +52,7 @@ def test_build_responses_kwargs_derives_prompt_cache_key_from_user_id():
         extra_kwargs={"custom_llm_provider": "openai"},
     )
     assert responses_kwargs["user"] == "session-abc"
-    assert responses_kwargs["prompt_cache_key"] == "session-abc"
+    assert "prompt_cache_key" not in responses_kwargs
 
 
 def test_build_responses_kwargs_prefers_explicit_prompt_cache_key_over_derived():
@@ -47,10 +60,10 @@ def test_build_responses_kwargs_prefers_explicit_prompt_cache_key_over_derived()
         max_tokens=1024,
         messages=MESSAGES,
         model="openai/gpt-5.6-luna",
-        metadata={"user_id": "session-abc"},
+        metadata={"user_id": CLAUDE_CODE_USER_ID},
         extra_kwargs={"custom_llm_provider": "openai", "prompt_cache_key": "explicit-key"},
     )
-    assert responses_kwargs["user"] == "session-abc"
+    assert responses_kwargs["user"] == CLAUDE_CODE_USER_ID[:64]
     assert responses_kwargs["prompt_cache_key"] == "explicit-key"
 
 

tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py

@@ -1113,17 +1113,34 @@ class TestTranslateRequestBroaderCoverage:
         kwargs = _ADAPTER.translate_request(req)
         assert len(kwargs["user"]) == 64
 
-    def test_metadata_user_id_mapped_to_prompt_cache_key(self):
-        req = _make_request(metadata={"user_id": "user-42"})
+    def test_metadata_claude_code_session_id_mapped_to_prompt_cache_key(self):
+        user_id = json.dumps({"device_id": "d" * 64, "account_uuid": "", "session_id": "session-42"})
+        req = _make_request(metadata={"user_id": user_id})
         kwargs = _ADAPTER.translate_request(req)
-        assert kwargs["prompt_cache_key"] == "user-42"
+        assert kwargs["user"] == user_id[:64]
+        assert kwargs["prompt_cache_key"] == "session-42"
 
-    def test_metadata_user_id_prompt_cache_key_truncated_to_first_64_chars(self):
-        long_id = "".join(str(i % 10) for i in range(100))
-        req = _make_request(metadata={"user_id": long_id})
+    def test_metadata_claude_code_sessions_get_distinct_prompt_cache_keys(self):
+        """BerriAI/litellm#39145: the first 64 chars of Claude Code's user_id are the per-install device_id."""
+        keys = tuple(
+            _ADAPTER.translate_request(
+                _make_request(
+                    metadata={"user_id": json.dumps({"device_id": "d" * 64, "account_uuid": "", "session_id": sid})}
+                )
+            )["prompt_cache_key"]
+            for sid in ("11111111-1111-1111-1111-111111111111", "22222222-2222-2222-2222-222222222222")
+        )
+        assert keys == ("11111111-1111-1111-1111-111111111111", "22222222-2222-2222-2222-222222222222")
+
+    @pytest.mark.parametrize(
+        "user_id",
+        ["user-42", "".join(str(i % 10) for i in range(100)), json.dumps({"device_id": "d" * 64}), "{not json"],
+    )
+    def test_metadata_plain_user_id_sets_no_prompt_cache_key(self, user_id: str):
+        req = _make_request(metadata={"user_id": user_id})
         kwargs = _ADAPTER.translate_request(req)
-        assert kwargs["prompt_cache_key"] == long_id[:64]
-        assert len(kwargs["prompt_cache_key"]) == 64
+        assert kwargs["user"] == user_id[:64]
+        assert "prompt_cache_key" not in kwargs
 
     def test_metadata_empty_user_id_sets_no_prompt_cache_key(self):
         req = _make_request(metadata={"user_id": ""})