feat(chatgpt): opt-in stable session id derived from the conversation prefix

When chatgpt_derive_session_id is set on the deployment and the request
carries no explicit session identifier, derive the session header from a
hash of the caller's api key hash, the instructions, and the first input
item instead of a fresh uuid4 per request. The same conversation then
lands on the same upstream prompt cache shard on every turn, so the
backend serves the resent history from its prompt cache. Explicit
session ids (litellm_session_id, session_id, metadata.session_id) still
win; proxy-generated litellm_trace_id and litellm_call_id fallbacks do
not block derivation, since the proxy stamps a fresh one on every
request and they would make the flag a no-op. Namespacing the anchor by
the caller's key keeps one tenant of a shared proxy from deterministically
confirming another tenant's prompts through cache-hit timing. The flag
parses quoted true/false strictly so a quoted "false" stays off, and
without the flag nothing changes.

Ref #37280

litellm/llms/chatgpt/common_utils.py

@@ -2,8 +2,10 @@
 Constants and helpers for ChatGPT subscription OAuth.
 """
 
+import json
 import os
 import platform
+from hashlib import sha256
 from typing import Any, Final
 from uuid import uuid4
 
@@ -268,7 +270,7 @@ def _normalize_litellm_params(litellm_params: Any | None) -> dict:
     return {}
 
 
-def get_chatgpt_session_id(litellm_params: object) -> str | None:
+def get_explicit_chatgpt_session_id(litellm_params: object) -> str | None:
     params: Final = _normalize_litellm_params(litellm_params)
     for key in ("litellm_session_id", "session_id"):
         value = params.get(key)
@@ -279,6 +281,14 @@ def get_chatgpt_session_id(litellm_params: object) -> str | None:
         value = metadata.get("session_id")
         if value:
             return str(value)
+    return None
+
+
+def get_chatgpt_session_id(litellm_params: object) -> str | None:
+    explicit: Final = get_explicit_chatgpt_session_id(litellm_params)
+    if explicit:
+        return explicit
+    params: Final = _normalize_litellm_params(litellm_params)
     for key in ("litellm_trace_id", "litellm_call_id"):
         value = params.get(key)
         if value:
@@ -288,3 +298,23 @@ def get_chatgpt_session_id(litellm_params: object) -> str | None:
 
 def ensure_chatgpt_session_id(litellm_params: object) -> str:
     return get_chatgpt_session_id(litellm_params) or str(uuid4())
+
+
+def should_derive_chatgpt_session_id(litellm_params: object) -> bool:
+    value: Final = _normalize_litellm_params(litellm_params).get("chatgpt_derive_session_id")
+    if isinstance(value, str):
+        return value.strip().lower() == "true"
+    return value is True
+
+
+def derive_chatgpt_session_id(litellm_params: object, instructions: str | None, input: object) -> str:
+    params: Final = _normalize_litellm_params(litellm_params)
+    metadata: Final = params.get("litellm_metadata") or params.get("metadata")
+    tenant: Final = str(metadata.get("user_api_key_hash") or "") if isinstance(metadata, dict) else ""
+    first_item: Final = input[0] if isinstance(input, list) and input else input
+    anchor: Final = json.dumps(
+        (tenant, instructions or "", first_item),
+        sort_keys=True,
+        default=str,
+    )
+    return f"litellm-derived-{sha256(anchor.encode('utf-8')).hexdigest()[:32]}"

litellm/llms/chatgpt/responses/transformation.py

@@ -27,9 +27,12 @@ from ..authenticator import Authenticator
 from ..common_utils import (
     CHATGPT_API_BASE,
     GetAccessTokenError,
+    derive_chatgpt_session_id,
     ensure_chatgpt_session_id,
     get_chatgpt_default_headers,
     get_chatgpt_default_instructions,
+    get_explicit_chatgpt_session_id,
+    should_derive_chatgpt_session_id,
 )
 
 if TYPE_CHECKING:
@@ -89,6 +92,10 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig):
             request["instructions"] = base_instructions
         request["store"] = False
         request["stream"] = True
+        should_derive: Final = should_derive_chatgpt_session_id(litellm_params)
+        if should_derive and get_explicit_chatgpt_session_id(litellm_params) is None:
+            derived: Final = derive_chatgpt_session_id(litellm_params, request.get("instructions"), input)
+            headers["session_id"] = derived  # rebind-ok: must land in the dict validate_environment built
         include: Final = list(request.get("include") or [])
         if "reasoning.encrypted_content" not in include:
             include.append("reasoning.encrypted_content")

tests/unit/llms/chatgpt/responses/test_chatgpt_responses_transformation.py

@@ -196,6 +196,109 @@ class TestChatGPTResponsesAPITransformation:
             "function": {"name": "hello"},
         }
 
+    def _transform_headers(
+        self, input: list[dict[str, str]], litellm_params: GenericLiteLLMParams
+    ) -> dict[str, str]:
+        headers: dict[str, str] = {}
+        ChatGPTResponsesAPIConfig().transform_responses_api_request(
+            model="chatgpt/gpt-5.4",
+            input=input,
+            response_api_optional_request_params={},
+            litellm_params=litellm_params,
+            headers=headers,
+        )
+        return headers
+
+    def test_derived_session_id_is_stable_as_conversation_grows(self):
+        params = GenericLiteLLMParams(chatgpt_derive_session_id=True)
+        first_turn = [{"role": "user", "content": "start of conversation"}]
+        later_turn = first_turn + [
+            {"role": "assistant", "content": "reply"},
+            {"role": "user", "content": "follow-up"},
+        ]
+
+        headers_one = self._transform_headers(first_turn, params)
+        headers_two = self._transform_headers(later_turn, params)
+
+        assert headers_one["session_id"].startswith("litellm-derived-")
+        assert headers_one["session_id"] == headers_two["session_id"]
+
+    def test_derived_session_id_differs_across_conversations(self):
+        params = GenericLiteLLMParams(chatgpt_derive_session_id=True)
+
+        headers_one = self._transform_headers(
+            [{"role": "user", "content": "conversation a"}], params
+        )
+        headers_two = self._transform_headers(
+            [{"role": "user", "content": "conversation b"}], params
+        )
+
+        assert headers_one["session_id"] != headers_two["session_id"]
+
+    def test_explicit_session_id_wins_over_derivation(self):
+        params = GenericLiteLLMParams(
+            chatgpt_derive_session_id=True, litellm_session_id="explicit-1"
+        )
+
+        headers = self._transform_headers([{"role": "user", "content": "hi"}], params)
+
+        assert "session_id" not in headers
+
+    def test_proxy_trace_id_does_not_block_derivation(self):
+        params = GenericLiteLLMParams(
+            chatgpt_derive_session_id=True,
+            litellm_trace_id="b6c2977a-0652-43de-a9a6-165822f439f6",
+        )
+        headers = {"session_id": params.litellm_trace_id}
+
+        ChatGPTResponsesAPIConfig().transform_responses_api_request(
+            model="chatgpt/gpt-5.4",
+            input=[{"role": "user", "content": "hi"}],
+            response_api_optional_request_params={},
+            litellm_params=params,
+            headers=headers,
+        )
+
+        assert headers["session_id"].startswith("litellm-derived-")
+
+    def test_different_api_keys_derive_different_session_ids(self):
+        input = [{"role": "user", "content": "shared prefix"}]
+        params_a = GenericLiteLLMParams(
+            chatgpt_derive_session_id=True,
+            litellm_metadata={"user_api_key_hash": "key-a"},
+        )
+        params_b = GenericLiteLLMParams(
+            chatgpt_derive_session_id=True,
+            litellm_metadata={"user_api_key_hash": "key-b"},
+        )
+
+        headers_a = self._transform_headers(input, params_a)
+        headers_b = self._transform_headers(input, params_b)
+
+        assert headers_a["session_id"] != headers_b["session_id"]
+        assert headers_a["session_id"] == self._transform_headers(input, params_a)["session_id"]
+
+    def test_quoted_false_string_does_not_enable_derivation(self):
+        params = GenericLiteLLMParams(chatgpt_derive_session_id="false")
+
+        headers = self._transform_headers([{"role": "user", "content": "hi"}], params)
+
+        assert "session_id" not in headers
+
+    def test_quoted_true_string_enables_derivation(self):
+        params = GenericLiteLLMParams(chatgpt_derive_session_id="true")
+
+        headers = self._transform_headers([{"role": "user", "content": "hi"}], params)
+
+        assert headers["session_id"].startswith("litellm-derived-")
+
+    def test_no_derivation_without_flag(self):
+        headers = self._transform_headers(
+            [{"role": "user", "content": "hi"}], GenericLiteLLMParams()
+        )
+
+        assert "session_id" not in headers
+
     @pytest.mark.parametrize(
         ("model_name", "response_model"),
         [