feat(chatgpt): opt-in stable session id derived from the conversation prefix

When chatgpt_derive_session_id is set on the deployment and the request
carries no explicit session identifier, derive the session header from a
hash of the caller's api key hash, the instructions, and the first input
item instead of a fresh uuid4 per request. The same conversation then
lands on the same upstream prompt cache shard on every turn, so the
backend serves the resent history from its prompt cache. Explicit
session ids (litellm_session_id, session_id, metadata.session_id) still
win; proxy-generated litellm_trace_id and litellm_call_id fallbacks do
not block derivation, since the proxy stamps a fresh one on every
request and they would make the flag a no-op. Namespacing the anchor by
the caller's key keeps one tenant of a shared proxy from deterministically
confirming another tenant's prompts through cache-hit timing. The flag
parses quoted true/false strictly so a quoted "false" stays off, and
without the flag nothing changes.

Ref #37280

litellm/llms/chatgpt/common_utils.py

@@ -9,13 +9,14 @@ import os
 import platform
 from collections.abc import Mapping, Sequence
 from dataclasses import dataclass
+from hashlib import sha256
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Final, TypeAlias
 from urllib.parse import urlsplit
 from uuid import uuid4
 
 import httpx
-from pydantic import JsonValue, TypeAdapter
+from pydantic import JsonValue, TypeAdapter, ValidationError
 
 from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils
 from litellm.litellm_core_utils.provider_affinity import get_caller_litellm_session_id, get_requested_session_id
@@ -407,22 +408,58 @@ def _prompt_cache_session_id(prompt_cache_key: object) -> str | None:
     )
 
 
-def get_chatgpt_session_id(litellm_params: object) -> str | None:
+def get_explicit_chatgpt_session_id(litellm_params: object) -> str | None:
     params: Final = _normalize_litellm_params(litellm_params)
     return (
         get_caller_litellm_session_id(params)
         or get_requested_session_id(params)
         or _prompt_cache_session_id(params.get("prompt_cache_key"))
-        or next(
-            (
-                str(value)
-                for key in ("litellm_session_id", "litellm_trace_id", "litellm_call_id")
-                if (value := params.get(key))
-            ),
-            None,
-        )
+        or (str(params["litellm_session_id"]) if params.get("litellm_session_id") else None)
+    )
+
+
+def get_chatgpt_session_id(litellm_params: object) -> str | None:
+    explicit: Final = get_explicit_chatgpt_session_id(litellm_params)
+    if explicit:
+        return explicit
+    params: Final = _normalize_litellm_params(litellm_params)
+    return next(
+        (str(value) for key in ("litellm_trace_id", "litellm_call_id") if (value := params.get(key))),
+        None,
     )
 
 
 def ensure_chatgpt_session_id(litellm_params: object) -> str:
     return get_chatgpt_session_id(litellm_params) or str(uuid4())
+
+
+def should_derive_chatgpt_session_id(litellm_params: object) -> bool:
+    value: Final = _normalize_litellm_params(litellm_params).get("chatgpt_derive_session_id")
+    if isinstance(value, str):
+        return value.strip().lower() == "true"
+    return value is True
+
+
+_INPUT_ITEMS: Final = TypeAdapter(tuple[object, ...])
+
+
+def _first_input_item(value: object) -> object:
+    """The first element of a list-shaped ``input``; any other shape hashes whole."""
+    try:
+        items: Final = _INPUT_ITEMS.validate_python(value)
+    except ValidationError:
+        return value
+    return items[0] if items else value
+
+
+def derive_chatgpt_session_id(litellm_params: object, instructions: str | None, input: object) -> str:
+    params: Final = _normalize_litellm_params(litellm_params)
+    metadata: Final = _normalize_litellm_params(params.get("litellm_metadata") or params.get("metadata"))
+    tenant: Final = str(metadata.get("user_api_key_hash") or "")
+    first_item: Final = _first_input_item(input)
+    anchor: Final = json.dumps(
+        (tenant, instructions or "", first_item),
+        sort_keys=True,
+        default=str,
+    )
+    return f"litellm-derived-{sha256(anchor.encode('utf-8')).hexdigest()[:32]}"

litellm/llms/chatgpt/responses/transformation.py

@@ -31,10 +31,13 @@ from ..authenticator import Authenticator, get_cached_authenticator, get_chatgpt
 from ..common_utils import (
     CHATGPT_API_BASE,
     GetAccessTokenError,
+    derive_chatgpt_session_id,
     ensure_chatgpt_session_id,
     get_chatgpt_client_credential,
     get_chatgpt_default_headers,
     get_chatgpt_default_instructions,
+    get_explicit_chatgpt_session_id,
+    should_derive_chatgpt_session_id,
     strip_chatgpt_client_credential_headers,
 )
 
@@ -122,13 +125,20 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig):
         )
         base_instructions: Final = get_chatgpt_default_instructions()
         existing_instructions: Final = request.get("instructions")
-        if existing_instructions:
-            if base_instructions not in existing_instructions:
-                request["instructions"] = f"{base_instructions}\n\n{existing_instructions}"
-        else:
-            request["instructions"] = base_instructions
+        instructions: Final = (
+            f"{base_instructions}\n\n{existing_instructions}"
+            if existing_instructions and base_instructions not in existing_instructions
+            else f"{existing_instructions}"
+            if existing_instructions
+            else base_instructions
+        )
+        request["instructions"] = instructions
         request["store"] = False
         request["stream"] = True
+        should_derive: Final = should_derive_chatgpt_session_id(litellm_params)
+        if should_derive and get_explicit_chatgpt_session_id(litellm_params) is None:
+            derived: Final = derive_chatgpt_session_id(litellm_params, instructions, input)
+            headers["session_id"] = derived  # rebind-ok: must land in the dict validate_environment built
         include: Final = list(request.get("include") or [])
         if "reasoning.encrypted_content" not in include:
             include.append("reasoning.encrypted_content")

tests/unit/llms/chatgpt/responses/test_chatgpt_responses_transformation.py

@@ -266,6 +266,109 @@ class TestChatGPTResponsesAPITransformation:
             "function": {"name": "hello"},
         }
 
+    def _transform_headers(
+        self, input: list[dict[str, str]], litellm_params: GenericLiteLLMParams
+    ) -> dict[str, str]:
+        headers: dict[str, str] = {}
+        ChatGPTResponsesAPIConfig().transform_responses_api_request(
+            model="chatgpt/gpt-5.4",
+            input=input,
+            response_api_optional_request_params={},
+            litellm_params=litellm_params,
+            headers=headers,
+        )
+        return headers
+
+    def test_derived_session_id_is_stable_as_conversation_grows(self):
+        params = GenericLiteLLMParams(chatgpt_derive_session_id=True)
+        first_turn = [{"role": "user", "content": "start of conversation"}]
+        later_turn = first_turn + [
+            {"role": "assistant", "content": "reply"},
+            {"role": "user", "content": "follow-up"},
+        ]
+
+        headers_one = self._transform_headers(first_turn, params)
+        headers_two = self._transform_headers(later_turn, params)
+
+        assert headers_one["session_id"].startswith("litellm-derived-")
+        assert headers_one["session_id"] == headers_two["session_id"]
+
+    def test_derived_session_id_differs_across_conversations(self):
+        params = GenericLiteLLMParams(chatgpt_derive_session_id=True)
+
+        headers_one = self._transform_headers(
+            [{"role": "user", "content": "conversation a"}], params
+        )
+        headers_two = self._transform_headers(
+            [{"role": "user", "content": "conversation b"}], params
+        )
+
+        assert headers_one["session_id"] != headers_two["session_id"]
+
+    def test_explicit_session_id_wins_over_derivation(self):
+        params = GenericLiteLLMParams(
+            chatgpt_derive_session_id=True, litellm_session_id="explicit-1"
+        )
+
+        headers = self._transform_headers([{"role": "user", "content": "hi"}], params)
+
+        assert "session_id" not in headers
+
+    def test_proxy_trace_id_does_not_block_derivation(self):
+        params = GenericLiteLLMParams(
+            chatgpt_derive_session_id=True,
+            litellm_trace_id="b6c2977a-0652-43de-a9a6-165822f439f6",
+        )
+        headers = {"session_id": params.litellm_trace_id}
+
+        ChatGPTResponsesAPIConfig().transform_responses_api_request(
+            model="chatgpt/gpt-5.4",
+            input=[{"role": "user", "content": "hi"}],
+            response_api_optional_request_params={},
+            litellm_params=params,
+            headers=headers,
+        )
+
+        assert headers["session_id"].startswith("litellm-derived-")
+
+    def test_different_api_keys_derive_different_session_ids(self):
+        input = [{"role": "user", "content": "shared prefix"}]
+        params_a = GenericLiteLLMParams(
+            chatgpt_derive_session_id=True,
+            litellm_metadata={"user_api_key_hash": "key-a"},
+        )
+        params_b = GenericLiteLLMParams(
+            chatgpt_derive_session_id=True,
+            litellm_metadata={"user_api_key_hash": "key-b"},
+        )
+
+        headers_a = self._transform_headers(input, params_a)
+        headers_b = self._transform_headers(input, params_b)
+
+        assert headers_a["session_id"] != headers_b["session_id"]
+        assert headers_a["session_id"] == self._transform_headers(input, params_a)["session_id"]
+
+    def test_quoted_false_string_does_not_enable_derivation(self):
+        params = GenericLiteLLMParams(chatgpt_derive_session_id="false")
+
+        headers = self._transform_headers([{"role": "user", "content": "hi"}], params)
+
+        assert "session_id" not in headers
+
+    def test_quoted_true_string_enables_derivation(self):
+        params = GenericLiteLLMParams(chatgpt_derive_session_id="true")
+
+        headers = self._transform_headers([{"role": "user", "content": "hi"}], params)
+
+        assert headers["session_id"].startswith("litellm-derived-")
+
+    def test_no_derivation_without_flag(self):
+        headers = self._transform_headers(
+            [{"role": "user", "content": "hi"}], GenericLiteLLMParams()
+        )
+
+        assert "session_id" not in headers
+
     @pytest.mark.parametrize(
         ("model_name", "response_model"),
         [