From a1a9b2e2e0043af3ce08e9c5de855658e3afdf78 Mon Sep 17 00:00:00 2001 From: pylrn Date: Wed, 2 Sep 2026 00:49:51 +0530 Subject: [PATCH] fix(llm_translation): extract session_id from JSON-encoded user_id metadata for prompt_cache_key --- .../experimental_pass_through/utils.py | 18 ++++++++++ .../test_responses_adapters_handler.py | 34 +++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/litellm/llms/anthropic/experimental_pass_through/utils.py b/litellm/llms/anthropic/experimental_pass_through/utils.py index 716a4f54778..2dba3417253 100644 --- a/litellm/llms/anthropic/experimental_pass_through/utils.py +++ b/litellm/llms/anthropic/experimental_pass_through/utils.py @@ -1,3 +1,4 @@ +import json import os from collections.abc import Mapping from types import MappingProxyType @@ -21,6 +22,23 @@ _THINKING_OFF: Final = "none" def prompt_cache_key_from_user_id(user_id: object) -> str | None: if user_id is None: return None + if isinstance(user_id, dict): + if session_id := user_id.get("session_id"): + return str(session_id)[:OPENAI_MAX_PROMPT_CACHE_KEY_LENGTH] or None + if user_key := user_id.get("user_id"): + return str(user_key)[:OPENAI_MAX_PROMPT_CACHE_KEY_LENGTH] or None + elif isinstance(user_id, str): + trimmed = user_id.strip() + if trimmed.startswith("{") and trimmed.endswith("}"): + try: + parsed = json.loads(trimmed) + if isinstance(parsed, dict): + if session_id := parsed.get("session_id"): + return str(session_id)[:OPENAI_MAX_PROMPT_CACHE_KEY_LENGTH] or None + if user_key := parsed.get("user_id"): + return str(user_key)[:OPENAI_MAX_PROMPT_CACHE_KEY_LENGTH] or None + except Exception: + pass return str(user_id)[:OPENAI_MAX_PROMPT_CACHE_KEY_LENGTH] or None diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py index 589dc64f9b9..f2fb17d16c6 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py @@ -28,6 +28,40 @@ def test_build_responses_kwargs_derives_prompt_cache_key_from_user_id(): assert responses_kwargs["prompt_cache_key"] == "session-abc" +def test_build_responses_kwargs_derives_prompt_cache_key_from_json_user_id_metadata(): + json_user_id = json.dumps( + { + "device_id": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "account_uuid": "", + "session_id": "12345678-abcd-ef01-2345-6789abcdef01", + } + ) + responses_kwargs = _build_responses_kwargs( + max_tokens=1024, + messages=MESSAGES, + model="openai/gpt-5.6-luna", + metadata={"user_id": json_user_id}, + extra_kwargs={"custom_llm_provider": "openai"}, + ) + assert responses_kwargs["user"] == json_user_id[:64] + assert responses_kwargs["prompt_cache_key"] == "12345678-abcd-ef01-2345-6789abcdef01" + + +def test_build_responses_kwargs_derives_prompt_cache_key_from_dict_user_id(): + dict_user_id = { + "device_id": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "session_id": "sess-guid-999", + } + responses_kwargs = _build_responses_kwargs( + max_tokens=1024, + messages=MESSAGES, + model="openai/gpt-5.6-luna", + metadata={"user_id": dict_user_id}, + extra_kwargs={"custom_llm_provider": "openai"}, + ) + assert responses_kwargs["prompt_cache_key"] == "sess-guid-999" + + def test_build_responses_kwargs_prefers_explicit_prompt_cache_key_over_derived(): responses_kwargs = _build_responses_kwargs( max_tokens=1024,