From 634852a18309a97f8cb24f18fabd48bb0b80c6a1 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Tue, 8 Sep 2026 18:00:01 -0700 Subject: [PATCH] fix(anthropic): key the /v1/messages prompt cache on Claude Code's session_id only The bridges derived prompt_cache_key as the first 64 chars of metadata.user_id. Claude Code packs a JSON object into that field whose prefix is the per-install device_id, so every session and subagent on one machine shared a single key, and a plain end-user id pinned all of that user's conversations to one slot. Parse the JSON and use session_id; send no key otherwise so the provider falls back to its own prompt-prefix hashing. An explicit prompt_cache_key still wins. Fixes #39145 --- .../experimental_pass_through/utils.py | 23 +++++- ...al_pass_through_adapters_transformation.py | 72 ++++++++++++++----- .../adapters/test_handler_prompt_cache_key.py | 12 ++-- .../test_responses_adapters_handler.py | 21 ++++-- .../test_responses_adapters_transformation.py | 33 ++++++--- 5 files changed, 123 insertions(+), 38 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/utils.py b/litellm/llms/anthropic/experimental_pass_through/utils.py index 55fe9c47faf..335a0e5641d 100644 --- a/litellm/llms/anthropic/experimental_pass_through/utils.py +++ b/litellm/llms/anthropic/experimental_pass_through/utils.py @@ -3,6 +3,8 @@ from collections.abc import Mapping from types import MappingProxyType from typing import TYPE_CHECKING, Final +from pydantic import BaseModel, ConfigDict, ValidationError + import litellm from litellm.types.utils import ModelInfo @@ -21,10 +23,27 @@ _EFFORT_DEGRADATION_CHAIN: Final[Mapping[str, tuple[str, ...]]] = MappingProxyTy _THINKING_OFF: Final = "none" +class _ClaudeCodeUserId(BaseModel): + """The JSON Claude Code packs into ``metadata.user_id``; only ``session_id`` is per conversation.""" + + model_config = ConfigDict(frozen=True) + + session_id: str + + def prompt_cache_key_from_user_id(user_id: object) -> str | None: - if user_id is None: + """The per-session key Claude Code carries inside ``metadata.user_id``, or nothing. + + Anthropic defines ``user_id`` as an opaque end-user id, so a plain string names a person, not + a conversation. Keying the provider cache on it pins every parallel session and subagent of that + person to one slot, which caches worse than the provider's own prompt-prefix hashing does. + """ + if not isinstance(user_id, str): + return None + try: + return _ClaudeCodeUserId.model_validate_json(user_id).session_id[:OPENAI_MAX_PROMPT_CACHE_KEY_LENGTH] or None + except ValidationError: return None - return str(user_id)[:OPENAI_MAX_PROMPT_CACHE_KEY_LENGTH] or None def litellm_logging_obj_from_kwargs(kwargs: Mapping[str, object]) -> "LiteLLMLoggingObject | None": diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 30465ca25ba..c59ec70b015 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -1,4 +1,5 @@ import base64 +import json from typing import Any, Final, cast import pytest @@ -724,9 +725,14 @@ def test_translate_anthropic_to_openai_orders_top_level_and_midturn_system(): ] -def _translate_with_metadata( - model: str, metadata: dict[str, str], custom_llm_provider: str | None -) -> dict[str, Any]: +def _claude_code_user_id(session_id: str) -> str: + return json.dumps({"device_id": "d" * 64, "account_uuid": "", "session_id": session_id}) + + +CLAUDE_CODE_USER_ID: Final = _claude_code_user_id("session-abc") + + +def _translate_with_metadata(model: str, metadata: dict[str, str], custom_llm_provider: str | None) -> dict[str, Any]: openai_request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( anthropic_message_request={ "model": model, @@ -739,23 +745,51 @@ def _translate_with_metadata( return cast(dict[str, Any], openai_request) -def test_translate_anthropic_to_openai_maps_user_id_to_prompt_cache_key_for_openai(): - openai_request = _translate_with_metadata("openai/gpt-5.6-luna", {"user_id": "session-abc"}, "openai") - assert openai_request["user"] == "session-abc" +def test_translate_anthropic_to_openai_maps_claude_code_session_id_to_prompt_cache_key_for_openai(): + openai_request = _translate_with_metadata("openai/gpt-5.6-luna", {"user_id": CLAUDE_CODE_USER_ID}, "openai") + assert openai_request["user"] == CLAUDE_CODE_USER_ID assert openai_request["prompt_cache_key"] == "session-abc" -def test_translate_anthropic_to_openai_truncates_prompt_cache_key_but_keeps_full_user(): - long_id = "".join(str(i % 10) for i in range(100)) - openai_request = _translate_with_metadata("openai/gpt-5.6-luna", {"user_id": long_id}, "openai") - assert openai_request["user"] == long_id - assert openai_request["prompt_cache_key"] == long_id[:64] - assert len(openai_request["prompt_cache_key"]) == 64 +def test_translate_anthropic_to_openai_gives_each_claude_code_session_its_own_prompt_cache_key(): + """BerriAI/litellm#39145: the first 64 chars of Claude Code's user_id are the per-install device_id.""" + keys = tuple( + _translate_with_metadata("openai/gpt-5.6-luna", {"user_id": _claude_code_user_id(session_id)}, "openai")[ + "prompt_cache_key" + ] + for session_id in ("11111111-1111-1111-1111-111111111111", "22222222-2222-2222-2222-222222222222") + ) + assert keys == ("11111111-1111-1111-1111-111111111111", "22222222-2222-2222-2222-222222222222") + + +def test_translate_anthropic_to_openai_truncates_long_session_id_to_openai_limit(): + long_session_id = "".join(str(i % 10) for i in range(100)) + openai_request = _translate_with_metadata( + "openai/gpt-5.6-luna", {"user_id": _claude_code_user_id(long_session_id)}, "openai" + ) + assert openai_request["prompt_cache_key"] == long_session_id[:64] + + +@pytest.mark.parametrize( + "user_id", + [ + "alice", + "".join(str(i % 10) for i in range(100)), + json.dumps({"device_id": "d" * 64, "account_uuid": ""}), + json.dumps({"session_id": ""}), + json.dumps({"session_id": 123}), + "{not json", + ], +) +def test_translate_anthropic_to_openai_keeps_plain_user_id_off_prompt_cache_key(user_id: str): + openai_request = _translate_with_metadata("openai/gpt-5.6-luna", {"user_id": user_id}, "openai") + assert openai_request["user"] == user_id + assert "prompt_cache_key" not in openai_request @pytest.mark.parametrize("model", ["azure/my-gpt-5-deployment", "my-gpt-5-deployment"]) def test_translate_anthropic_to_openai_sets_prompt_cache_key_for_azure(model: str): - openai_request = _translate_with_metadata(model, {"user_id": "session-abc"}, "azure") + openai_request = _translate_with_metadata(model, {"user_id": CLAUDE_CODE_USER_ID}, "azure") assert openai_request["prompt_cache_key"] == "session-abc" @@ -772,8 +806,8 @@ def test_translate_anthropic_to_openai_sets_prompt_cache_key_for_azure(model: st def test_translate_anthropic_to_openai_skips_prompt_cache_key_when_provider_lacks_it( model: str, custom_llm_provider: str ): - openai_request = _translate_with_metadata(model, {"user_id": "session-abc"}, custom_llm_provider) - assert openai_request["user"] == "session-abc" + openai_request = _translate_with_metadata(model, {"user_id": CLAUDE_CODE_USER_ID}, custom_llm_provider) + assert openai_request["user"] == CLAUDE_CODE_USER_ID assert "prompt_cache_key" not in openai_request @@ -781,14 +815,14 @@ def test_translate_anthropic_to_openai_skips_prompt_cache_key_for_chained_litell assert "prompt_cache_key" in litellm.get_supported_openai_params( model="xai", custom_llm_provider="litellm_proxy" ) - openai_request = _translate_with_metadata("litellm_proxy/xai", {"user_id": "session-abc"}, "litellm_proxy") - assert openai_request["user"] == "session-abc" + openai_request = _translate_with_metadata("litellm_proxy/xai", {"user_id": CLAUDE_CODE_USER_ID}, "litellm_proxy") + assert openai_request["user"] == CLAUDE_CODE_USER_ID assert "prompt_cache_key" not in openai_request def test_translate_anthropic_to_openai_skips_prompt_cache_key_without_provider(): - openai_request = _translate_with_metadata("openai/gpt-5.6-luna", {"user_id": "session-abc"}, None) - assert openai_request["user"] == "session-abc" + openai_request = _translate_with_metadata("openai/gpt-5.6-luna", {"user_id": CLAUDE_CODE_USER_ID}, None) + assert openai_request["user"] == CLAUDE_CODE_USER_ID assert "prompt_cache_key" not in openai_request diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_prompt_cache_key.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_prompt_cache_key.py index f48d51dbe1e..7dc7507120f 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_prompt_cache_key.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_prompt_cache_key.py @@ -1,3 +1,4 @@ +import json import os import sys @@ -10,6 +11,7 @@ from litellm.llms.anthropic.experimental_pass_through.adapters.handler import ( ) MESSAGES = [{"role": "user", "content": "hello"}] +CLAUDE_CODE_USER_ID = json.dumps({"device_id": "d" * 64, "account_uuid": "", "session_id": "session-abc"}) def _prepare(model: str, extra_kwargs: dict[str, object], thinking: dict[str, object] | None = None): @@ -17,7 +19,7 @@ def _prepare(model: str, extra_kwargs: dict[str, object], thinking: dict[str, ob max_tokens=1024, messages=MESSAGES, model=model, - metadata={"user_id": "session-abc"}, + metadata={"user_id": CLAUDE_CODE_USER_ID}, thinking=thinking, extra_kwargs=extra_kwargs, ) @@ -26,7 +28,7 @@ def _prepare(model: str, extra_kwargs: dict[str, object], thinking: dict[str, ob def test_prepare_completion_kwargs_derives_prompt_cache_key_for_openai_provider(): completion_kwargs = _prepare("openai/gpt-5.6-luna", {"custom_llm_provider": "openai"}) - assert completion_kwargs["user"] == "session-abc" + assert completion_kwargs["user"] == CLAUDE_CODE_USER_ID assert completion_kwargs["prompt_cache_key"] == "session-abc" @@ -35,7 +37,7 @@ def test_prepare_completion_kwargs_prefers_explicit_prompt_cache_key_over_derive "openai/gpt-5.6-luna", {"custom_llm_provider": "openai", "prompt_cache_key": "explicit-key"}, ) - assert completion_kwargs["user"] == "session-abc" + assert completion_kwargs["user"] == CLAUDE_CODE_USER_ID assert completion_kwargs["prompt_cache_key"] == "explicit-key" @@ -50,13 +52,13 @@ def test_prepare_completion_kwargs_skips_prompt_cache_key_without_provider_suppo model: str, extra_kwargs: dict[str, object] ): completion_kwargs = _prepare(model, extra_kwargs) - assert completion_kwargs["user"] == "session-abc" + assert completion_kwargs["user"] == CLAUDE_CODE_USER_ID assert "prompt_cache_key" not in completion_kwargs def test_prepare_completion_kwargs_skips_prompt_cache_key_for_chained_litellm_proxy(): completion_kwargs = _prepare("litellm_proxy/xai", {"custom_llm_provider": "litellm_proxy"}) - assert completion_kwargs["user"] == "session-abc" + assert completion_kwargs["user"] == CLAUDE_CODE_USER_ID assert "prompt_cache_key" not in completion_kwargs diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py index 3383813245a..16e8cf0e90e 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py @@ -16,6 +16,7 @@ from litellm.llms.anthropic.experimental_pass_through.responses_adapters.handler ) MESSAGES = [{"role": "user", "content": "hello"}] +CLAUDE_CODE_USER_ID = json.dumps({"device_id": "d" * 64, "account_uuid": "", "session_id": "session-abc"}) RESPONSES_SSE_BODY = ( b"event: response.created\n" @@ -30,7 +31,19 @@ RESPONSES_SSE_BODY = ( ) -def test_build_responses_kwargs_derives_prompt_cache_key_from_user_id(): +def test_build_responses_kwargs_derives_prompt_cache_key_from_claude_code_session_id(): + responses_kwargs = _build_responses_kwargs( + max_tokens=1024, + messages=MESSAGES, + model="openai/gpt-5.6-luna", + metadata={"user_id": CLAUDE_CODE_USER_ID}, + extra_kwargs={"custom_llm_provider": "openai"}, + ) + assert responses_kwargs["user"] == CLAUDE_CODE_USER_ID[:64] + assert responses_kwargs["prompt_cache_key"] == "session-abc" + + +def test_build_responses_kwargs_sets_no_prompt_cache_key_for_plain_user_id(): responses_kwargs = _build_responses_kwargs( max_tokens=1024, messages=MESSAGES, @@ -39,7 +52,7 @@ def test_build_responses_kwargs_derives_prompt_cache_key_from_user_id(): extra_kwargs={"custom_llm_provider": "openai"}, ) assert responses_kwargs["user"] == "session-abc" - assert responses_kwargs["prompt_cache_key"] == "session-abc" + assert "prompt_cache_key" not in responses_kwargs def test_build_responses_kwargs_prefers_explicit_prompt_cache_key_over_derived(): @@ -47,10 +60,10 @@ def test_build_responses_kwargs_prefers_explicit_prompt_cache_key_over_derived() max_tokens=1024, messages=MESSAGES, model="openai/gpt-5.6-luna", - metadata={"user_id": "session-abc"}, + metadata={"user_id": CLAUDE_CODE_USER_ID}, extra_kwargs={"custom_llm_provider": "openai", "prompt_cache_key": "explicit-key"}, ) - assert responses_kwargs["user"] == "session-abc" + assert responses_kwargs["user"] == CLAUDE_CODE_USER_ID[:64] assert responses_kwargs["prompt_cache_key"] == "explicit-key" diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py index 9f8414afa38..edcc7adddb7 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -1113,17 +1113,34 @@ class TestTranslateRequestBroaderCoverage: kwargs = _ADAPTER.translate_request(req) assert len(kwargs["user"]) == 64 - def test_metadata_user_id_mapped_to_prompt_cache_key(self): - req = _make_request(metadata={"user_id": "user-42"}) + def test_metadata_claude_code_session_id_mapped_to_prompt_cache_key(self): + user_id = json.dumps({"device_id": "d" * 64, "account_uuid": "", "session_id": "session-42"}) + req = _make_request(metadata={"user_id": user_id}) kwargs = _ADAPTER.translate_request(req) - assert kwargs["prompt_cache_key"] == "user-42" + assert kwargs["user"] == user_id[:64] + assert kwargs["prompt_cache_key"] == "session-42" - def test_metadata_user_id_prompt_cache_key_truncated_to_first_64_chars(self): - long_id = "".join(str(i % 10) for i in range(100)) - req = _make_request(metadata={"user_id": long_id}) + def test_metadata_claude_code_sessions_get_distinct_prompt_cache_keys(self): + """BerriAI/litellm#39145: the first 64 chars of Claude Code's user_id are the per-install device_id.""" + keys = tuple( + _ADAPTER.translate_request( + _make_request( + metadata={"user_id": json.dumps({"device_id": "d" * 64, "account_uuid": "", "session_id": sid})} + ) + )["prompt_cache_key"] + for sid in ("11111111-1111-1111-1111-111111111111", "22222222-2222-2222-2222-222222222222") + ) + assert keys == ("11111111-1111-1111-1111-111111111111", "22222222-2222-2222-2222-222222222222") + + @pytest.mark.parametrize( + "user_id", + ["user-42", "".join(str(i % 10) for i in range(100)), json.dumps({"device_id": "d" * 64}), "{not json"], + ) + def test_metadata_plain_user_id_sets_no_prompt_cache_key(self, user_id: str): + req = _make_request(metadata={"user_id": user_id}) kwargs = _ADAPTER.translate_request(req) - assert kwargs["prompt_cache_key"] == long_id[:64] - assert len(kwargs["prompt_cache_key"]) == 64 + assert kwargs["user"] == user_id[:64] + assert "prompt_cache_key" not in kwargs def test_metadata_empty_user_id_sets_no_prompt_cache_key(self): req = _make_request(metadata={"user_id": ""})