From 9256f7cb818e7321e5d8413d12ada0ca541fde01 Mon Sep 17 00:00:00 2001 From: Devon Krisman Date: Tue, 18 Aug 2026 09:03:21 -0400 Subject: [PATCH] feat(chatgpt): opt-in stable session id derived from the conversation prefix When chatgpt_derive_session_id is set on the deployment and the request carries no explicit session identifier, derive the session header from a hash of the caller's api key hash, the instructions, and the first input item instead of a fresh uuid4 per request. The same conversation then lands on the same upstream prompt cache shard on every turn, so the backend serves the resent history from its prompt cache. Explicit session ids (litellm_session_id, session_id, metadata.session_id) still win; proxy-generated litellm_trace_id and litellm_call_id fallbacks do not block derivation, since the proxy stamps a fresh one on every request and they would make the flag a no-op. Namespacing the anchor by the caller's key keeps one tenant of a shared proxy from deterministically confirming another tenant's prompts through cache-hit timing. The flag parses quoted true/false strictly so a quoted "false" stays off, and without the flag nothing changes. Ref #37280 --- litellm/llms/chatgpt/common_utils.py | 32 +++++- .../llms/chatgpt/responses/transformation.py | 7 ++ .../test_chatgpt_responses_transformation.py | 103 ++++++++++++++++++ 3 files changed, 141 insertions(+), 1 deletion(-) diff --git a/litellm/llms/chatgpt/common_utils.py b/litellm/llms/chatgpt/common_utils.py index 35e32e4172f..bbeb8ca2e33 100644 --- a/litellm/llms/chatgpt/common_utils.py +++ b/litellm/llms/chatgpt/common_utils.py @@ -2,8 +2,10 @@ Constants and helpers for ChatGPT subscription OAuth. """ +import json import os import platform +from hashlib import sha256 from typing import Any, Final from uuid import uuid4 @@ -268,7 +270,7 @@ def _normalize_litellm_params(litellm_params: Any | None) -> dict: return {} -def get_chatgpt_session_id(litellm_params: Any | None) -> str | None: +def get_explicit_chatgpt_session_id(litellm_params: object) -> str | None: params: Final = _normalize_litellm_params(litellm_params) for key in ("litellm_session_id", "session_id"): value = params.get(key) @@ -279,6 +281,14 @@ def get_chatgpt_session_id(litellm_params: Any | None) -> str | None: value = metadata.get("session_id") if value: return str(value) + return None + + +def get_chatgpt_session_id(litellm_params: object) -> str | None: + explicit: Final = get_explicit_chatgpt_session_id(litellm_params) + if explicit: + return explicit + params: Final = _normalize_litellm_params(litellm_params) for key in ("litellm_trace_id", "litellm_call_id"): value = params.get(key) if value: @@ -288,3 +298,23 @@ def get_chatgpt_session_id(litellm_params: Any | None) -> str | None: def ensure_chatgpt_session_id(litellm_params: Any | None) -> str: return get_chatgpt_session_id(litellm_params) or str(uuid4()) + + +def should_derive_chatgpt_session_id(litellm_params: object) -> bool: + value: Final = _normalize_litellm_params(litellm_params).get("chatgpt_derive_session_id") + if isinstance(value, str): + return value.strip().lower() == "true" + return value is True + + +def derive_chatgpt_session_id(litellm_params: object, instructions: str | None, input: object) -> str: + params: Final = _normalize_litellm_params(litellm_params) + metadata: Final = params.get("litellm_metadata") or params.get("metadata") + tenant: Final = str(metadata.get("user_api_key_hash") or "") if isinstance(metadata, dict) else "" + first_item: Final = input[0] if isinstance(input, list) and input else input + anchor: Final = json.dumps( + (tenant, instructions or "", first_item), + sort_keys=True, + default=str, + ) + return f"litellm-derived-{sha256(anchor.encode('utf-8')).hexdigest()[:32]}" diff --git a/litellm/llms/chatgpt/responses/transformation.py b/litellm/llms/chatgpt/responses/transformation.py index 8e4bbf1d3c9..0485e46caf2 100644 --- a/litellm/llms/chatgpt/responses/transformation.py +++ b/litellm/llms/chatgpt/responses/transformation.py @@ -23,9 +23,12 @@ from ..authenticator import Authenticator from ..common_utils import ( CHATGPT_API_BASE, GetAccessTokenError, + derive_chatgpt_session_id, ensure_chatgpt_session_id, get_chatgpt_default_headers, get_chatgpt_default_instructions, + get_explicit_chatgpt_session_id, + should_derive_chatgpt_session_id, ) @@ -82,6 +85,10 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): request["instructions"] = base_instructions request["store"] = False request["stream"] = True + should_derive: Final = should_derive_chatgpt_session_id(litellm_params) + if should_derive and get_explicit_chatgpt_session_id(litellm_params) is None: + derived: Final = derive_chatgpt_session_id(litellm_params, request.get("instructions"), input) + headers["session_id"] = derived # rebind-ok: must land in the dict validate_environment built include: Final = list(request.get("include") or []) if "reasoning.encrypted_content" not in include: include.append("reasoning.encrypted_content") diff --git a/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py b/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py index 90a1c24bada..5746a60cf96 100644 --- a/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py +++ b/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py @@ -158,6 +158,109 @@ class TestChatGPTResponsesAPITransformation: "function": {"name": "hello"}, } + def _transform_headers( + self, input: list[dict[str, str]], litellm_params: GenericLiteLLMParams + ) -> dict[str, str]: + headers: dict[str, str] = {} + ChatGPTResponsesAPIConfig().transform_responses_api_request( + model="chatgpt/gpt-5.4", + input=input, + response_api_optional_request_params={}, + litellm_params=litellm_params, + headers=headers, + ) + return headers + + def test_derived_session_id_is_stable_as_conversation_grows(self): + params = GenericLiteLLMParams(chatgpt_derive_session_id=True) + first_turn = [{"role": "user", "content": "start of conversation"}] + later_turn = first_turn + [ + {"role": "assistant", "content": "reply"}, + {"role": "user", "content": "follow-up"}, + ] + + headers_one = self._transform_headers(first_turn, params) + headers_two = self._transform_headers(later_turn, params) + + assert headers_one["session_id"].startswith("litellm-derived-") + assert headers_one["session_id"] == headers_two["session_id"] + + def test_derived_session_id_differs_across_conversations(self): + params = GenericLiteLLMParams(chatgpt_derive_session_id=True) + + headers_one = self._transform_headers( + [{"role": "user", "content": "conversation a"}], params + ) + headers_two = self._transform_headers( + [{"role": "user", "content": "conversation b"}], params + ) + + assert headers_one["session_id"] != headers_two["session_id"] + + def test_explicit_session_id_wins_over_derivation(self): + params = GenericLiteLLMParams( + chatgpt_derive_session_id=True, litellm_session_id="explicit-1" + ) + + headers = self._transform_headers([{"role": "user", "content": "hi"}], params) + + assert "session_id" not in headers + + def test_proxy_trace_id_does_not_block_derivation(self): + params = GenericLiteLLMParams( + chatgpt_derive_session_id=True, + litellm_trace_id="b6c2977a-0652-43de-a9a6-165822f439f6", + ) + headers = {"session_id": params.litellm_trace_id} + + ChatGPTResponsesAPIConfig().transform_responses_api_request( + model="chatgpt/gpt-5.4", + input=[{"role": "user", "content": "hi"}], + response_api_optional_request_params={}, + litellm_params=params, + headers=headers, + ) + + assert headers["session_id"].startswith("litellm-derived-") + + def test_different_api_keys_derive_different_session_ids(self): + input = [{"role": "user", "content": "shared prefix"}] + params_a = GenericLiteLLMParams( + chatgpt_derive_session_id=True, + litellm_metadata={"user_api_key_hash": "key-a"}, + ) + params_b = GenericLiteLLMParams( + chatgpt_derive_session_id=True, + litellm_metadata={"user_api_key_hash": "key-b"}, + ) + + headers_a = self._transform_headers(input, params_a) + headers_b = self._transform_headers(input, params_b) + + assert headers_a["session_id"] != headers_b["session_id"] + assert headers_a["session_id"] == self._transform_headers(input, params_a)["session_id"] + + def test_quoted_false_string_does_not_enable_derivation(self): + params = GenericLiteLLMParams(chatgpt_derive_session_id="false") + + headers = self._transform_headers([{"role": "user", "content": "hi"}], params) + + assert "session_id" not in headers + + def test_quoted_true_string_enables_derivation(self): + params = GenericLiteLLMParams(chatgpt_derive_session_id="true") + + headers = self._transform_headers([{"role": "user", "content": "hi"}], params) + + assert headers["session_id"].startswith("litellm-derived-") + + def test_no_derivation_without_flag(self): + headers = self._transform_headers( + [{"role": "user", "content": "hi"}], GenericLiteLLMParams() + ) + + assert "session_id" not in headers + @pytest.mark.parametrize( ("model_name", "response_model"), [