From 37691428bf6444fcb376166951f0b11829f058ce Mon Sep 17 00:00:00 2001 From: Devon Krisman Date: Tue, 18 Aug 2026 09:03:21 -0400 Subject: [PATCH] feat(chatgpt): opt-in stable session id derived from the conversation prefix When chatgpt_derive_session_id is set on the deployment and the request carries no explicit session identifier, derive the session header from a hash of the caller's api key hash, the instructions, and the first input item instead of a fresh uuid4 per request. The same conversation then lands on the same upstream prompt cache shard on every turn, so the backend serves the resent history from its prompt cache. Explicit session ids (litellm_session_id, session_id, metadata.session_id) still win; proxy-generated litellm_trace_id and litellm_call_id fallbacks do not block derivation, since the proxy stamps a fresh one on every request and they would make the flag a no-op. Namespacing the anchor by the caller's key keeps one tenant of a shared proxy from deterministically confirming another tenant's prompts through cache-hit timing. The flag parses quoted true/false strictly so a quoted "false" stays off, and without the flag nothing changes. Ref #37280 --- litellm/llms/chatgpt/common_utils.py | 32 +++++- .../llms/chatgpt/responses/transformation.py | 7 ++ .../test_chatgpt_responses_transformation.py | 103 ++++++++++++++++++ 3 files changed, 141 insertions(+), 1 deletion(-) diff --git a/litellm/llms/chatgpt/common_utils.py b/litellm/llms/chatgpt/common_utils.py index fe33219f110..2e0a1d898b8 100644 --- a/litellm/llms/chatgpt/common_utils.py +++ b/litellm/llms/chatgpt/common_utils.py @@ -2,8 +2,10 @@ Constants and helpers for ChatGPT subscription OAuth. """ +import json import os import platform +from hashlib import sha256 from typing import Any, Final from uuid import uuid4 @@ -268,7 +270,7 @@ def _normalize_litellm_params(litellm_params: Any | None) -> dict: return {} -def get_chatgpt_session_id(litellm_params: object) -> str | None: +def get_explicit_chatgpt_session_id(litellm_params: object) -> str | None: params: Final = _normalize_litellm_params(litellm_params) for key in ("litellm_session_id", "session_id"): value = params.get(key) @@ -279,6 +281,14 @@ def get_chatgpt_session_id(litellm_params: object) -> str | None: value = metadata.get("session_id") if value: return str(value) + return None + + +def get_chatgpt_session_id(litellm_params: object) -> str | None: + explicit: Final = get_explicit_chatgpt_session_id(litellm_params) + if explicit: + return explicit + params: Final = _normalize_litellm_params(litellm_params) for key in ("litellm_trace_id", "litellm_call_id"): value = params.get(key) if value: @@ -288,3 +298,23 @@ def get_chatgpt_session_id(litellm_params: object) -> str | None: def ensure_chatgpt_session_id(litellm_params: object) -> str: return get_chatgpt_session_id(litellm_params) or str(uuid4()) + + +def should_derive_chatgpt_session_id(litellm_params: object) -> bool: + value: Final = _normalize_litellm_params(litellm_params).get("chatgpt_derive_session_id") + if isinstance(value, str): + return value.strip().lower() == "true" + return value is True + + +def derive_chatgpt_session_id(litellm_params: object, instructions: str | None, input: object) -> str: + params: Final = _normalize_litellm_params(litellm_params) + metadata: Final = params.get("litellm_metadata") or params.get("metadata") + tenant: Final = str(metadata.get("user_api_key_hash") or "") if isinstance(metadata, dict) else "" + first_item: Final = input[0] if isinstance(input, list) and input else input + anchor: Final = json.dumps( + (tenant, instructions or "", first_item), + sort_keys=True, + default=str, + ) + return f"litellm-derived-{sha256(anchor.encode('utf-8')).hexdigest()[:32]}" diff --git a/litellm/llms/chatgpt/responses/transformation.py b/litellm/llms/chatgpt/responses/transformation.py index 9774b762396..45efdde902a 100644 --- a/litellm/llms/chatgpt/responses/transformation.py +++ b/litellm/llms/chatgpt/responses/transformation.py @@ -27,9 +27,12 @@ from ..authenticator import Authenticator from ..common_utils import ( CHATGPT_API_BASE, GetAccessTokenError, + derive_chatgpt_session_id, ensure_chatgpt_session_id, get_chatgpt_default_headers, get_chatgpt_default_instructions, + get_explicit_chatgpt_session_id, + should_derive_chatgpt_session_id, ) if TYPE_CHECKING: @@ -89,6 +92,10 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): request["instructions"] = base_instructions request["store"] = False request["stream"] = True + should_derive: Final = should_derive_chatgpt_session_id(litellm_params) + if should_derive and get_explicit_chatgpt_session_id(litellm_params) is None: + derived: Final = derive_chatgpt_session_id(litellm_params, request.get("instructions"), input) + headers["session_id"] = derived # rebind-ok: must land in the dict validate_environment built include: Final = list(request.get("include") or []) if "reasoning.encrypted_content" not in include: include.append("reasoning.encrypted_content") diff --git a/tests/unit/llms/chatgpt/responses/test_chatgpt_responses_transformation.py b/tests/unit/llms/chatgpt/responses/test_chatgpt_responses_transformation.py index 0b04dd0ed78..4e2fb36b38c 100644 --- a/tests/unit/llms/chatgpt/responses/test_chatgpt_responses_transformation.py +++ b/tests/unit/llms/chatgpt/responses/test_chatgpt_responses_transformation.py @@ -196,6 +196,109 @@ class TestChatGPTResponsesAPITransformation: "function": {"name": "hello"}, } + def _transform_headers( + self, input: list[dict[str, str]], litellm_params: GenericLiteLLMParams + ) -> dict[str, str]: + headers: dict[str, str] = {} + ChatGPTResponsesAPIConfig().transform_responses_api_request( + model="chatgpt/gpt-5.4", + input=input, + response_api_optional_request_params={}, + litellm_params=litellm_params, + headers=headers, + ) + return headers + + def test_derived_session_id_is_stable_as_conversation_grows(self): + params = GenericLiteLLMParams(chatgpt_derive_session_id=True) + first_turn = [{"role": "user", "content": "start of conversation"}] + later_turn = first_turn + [ + {"role": "assistant", "content": "reply"}, + {"role": "user", "content": "follow-up"}, + ] + + headers_one = self._transform_headers(first_turn, params) + headers_two = self._transform_headers(later_turn, params) + + assert headers_one["session_id"].startswith("litellm-derived-") + assert headers_one["session_id"] == headers_two["session_id"] + + def test_derived_session_id_differs_across_conversations(self): + params = GenericLiteLLMParams(chatgpt_derive_session_id=True) + + headers_one = self._transform_headers( + [{"role": "user", "content": "conversation a"}], params + ) + headers_two = self._transform_headers( + [{"role": "user", "content": "conversation b"}], params + ) + + assert headers_one["session_id"] != headers_two["session_id"] + + def test_explicit_session_id_wins_over_derivation(self): + params = GenericLiteLLMParams( + chatgpt_derive_session_id=True, litellm_session_id="explicit-1" + ) + + headers = self._transform_headers([{"role": "user", "content": "hi"}], params) + + assert "session_id" not in headers + + def test_proxy_trace_id_does_not_block_derivation(self): + params = GenericLiteLLMParams( + chatgpt_derive_session_id=True, + litellm_trace_id="b6c2977a-0652-43de-a9a6-165822f439f6", + ) + headers = {"session_id": params.litellm_trace_id} + + ChatGPTResponsesAPIConfig().transform_responses_api_request( + model="chatgpt/gpt-5.4", + input=[{"role": "user", "content": "hi"}], + response_api_optional_request_params={}, + litellm_params=params, + headers=headers, + ) + + assert headers["session_id"].startswith("litellm-derived-") + + def test_different_api_keys_derive_different_session_ids(self): + input = [{"role": "user", "content": "shared prefix"}] + params_a = GenericLiteLLMParams( + chatgpt_derive_session_id=True, + litellm_metadata={"user_api_key_hash": "key-a"}, + ) + params_b = GenericLiteLLMParams( + chatgpt_derive_session_id=True, + litellm_metadata={"user_api_key_hash": "key-b"}, + ) + + headers_a = self._transform_headers(input, params_a) + headers_b = self._transform_headers(input, params_b) + + assert headers_a["session_id"] != headers_b["session_id"] + assert headers_a["session_id"] == self._transform_headers(input, params_a)["session_id"] + + def test_quoted_false_string_does_not_enable_derivation(self): + params = GenericLiteLLMParams(chatgpt_derive_session_id="false") + + headers = self._transform_headers([{"role": "user", "content": "hi"}], params) + + assert "session_id" not in headers + + def test_quoted_true_string_enables_derivation(self): + params = GenericLiteLLMParams(chatgpt_derive_session_id="true") + + headers = self._transform_headers([{"role": "user", "content": "hi"}], params) + + assert headers["session_id"].startswith("litellm-derived-") + + def test_no_derivation_without_flag(self): + headers = self._transform_headers( + [{"role": "user", "content": "hi"}], GenericLiteLLMParams() + ) + + assert "session_id" not in headers + @pytest.mark.parametrize( ("model_name", "response_model"), [