From 2cddcf31b2143fbe196653cc74bf88eda99eec0e Mon Sep 17 00:00:00 2001 From: Devon Krisman Date: Tue, 18 Aug 2026 09:03:21 -0400 Subject: [PATCH] feat(chatgpt): opt-in stable session id derived from the conversation prefix When chatgpt_derive_session_id is set on the deployment and the request carries no explicit session identifier, derive the session header from a hash of the caller's api key hash, the instructions, and the first input item instead of a fresh uuid4 per request. The same conversation then lands on the same upstream prompt cache shard on every turn, so the backend serves the resent history from its prompt cache. Explicit session ids (litellm_session_id, session_id, metadata.session_id) still win; proxy-generated litellm_trace_id and litellm_call_id fallbacks do not block derivation, since the proxy stamps a fresh one on every request and they would make the flag a no-op. Namespacing the anchor by the caller's key keeps one tenant of a shared proxy from deterministically confirming another tenant's prompts through cache-hit timing. The flag parses quoted true/false strictly so a quoted "false" stays off, and without the flag nothing changes. Ref #37280 --- litellm/llms/chatgpt/common_utils.py | 57 ++++++++-- .../llms/chatgpt/responses/transformation.py | 20 +++- .../test_chatgpt_responses_transformation.py | 103 ++++++++++++++++++ 3 files changed, 165 insertions(+), 15 deletions(-) diff --git a/litellm/llms/chatgpt/common_utils.py b/litellm/llms/chatgpt/common_utils.py index cd2ea6b9ccc..abaf6c68717 100644 --- a/litellm/llms/chatgpt/common_utils.py +++ b/litellm/llms/chatgpt/common_utils.py @@ -9,13 +9,14 @@ import os import platform from collections.abc import Mapping, Sequence from dataclasses import dataclass +from hashlib import sha256 from types import MappingProxyType from typing import TYPE_CHECKING, Final, TypeAlias from urllib.parse import urlsplit from uuid import uuid4 import httpx -from pydantic import JsonValue, TypeAdapter +from pydantic import JsonValue, TypeAdapter, ValidationError from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils from litellm.litellm_core_utils.provider_affinity import get_caller_litellm_session_id, get_requested_session_id @@ -407,22 +408,58 @@ def _prompt_cache_session_id(prompt_cache_key: object) -> str | None: ) -def get_chatgpt_session_id(litellm_params: object) -> str | None: +def get_explicit_chatgpt_session_id(litellm_params: object) -> str | None: params: Final = _normalize_litellm_params(litellm_params) return ( get_caller_litellm_session_id(params) or get_requested_session_id(params) or _prompt_cache_session_id(params.get("prompt_cache_key")) - or next( - ( - str(value) - for key in ("litellm_session_id", "litellm_trace_id", "litellm_call_id") - if (value := params.get(key)) - ), - None, - ) + or (str(params["litellm_session_id"]) if params.get("litellm_session_id") else None) + ) + + +def get_chatgpt_session_id(litellm_params: object) -> str | None: + explicit: Final = get_explicit_chatgpt_session_id(litellm_params) + if explicit: + return explicit + params: Final = _normalize_litellm_params(litellm_params) + return next( + (str(value) for key in ("litellm_trace_id", "litellm_call_id") if (value := params.get(key))), + None, ) def ensure_chatgpt_session_id(litellm_params: object) -> str: return get_chatgpt_session_id(litellm_params) or str(uuid4()) + + +def should_derive_chatgpt_session_id(litellm_params: object) -> bool: + value: Final = _normalize_litellm_params(litellm_params).get("chatgpt_derive_session_id") + if isinstance(value, str): + return value.strip().lower() == "true" + return value is True + + +_INPUT_ITEMS: Final = TypeAdapter(tuple[object, ...]) + + +def _first_input_item(value: object) -> object: + """The first element of a list-shaped ``input``; any other shape hashes whole.""" + try: + items: Final = _INPUT_ITEMS.validate_python(value) + except ValidationError: + return value + return items[0] if items else value + + +def derive_chatgpt_session_id(litellm_params: object, instructions: str | None, input: object) -> str: + params: Final = _normalize_litellm_params(litellm_params) + metadata: Final = _normalize_litellm_params(params.get("litellm_metadata") or params.get("metadata")) + tenant: Final = str(metadata.get("user_api_key_hash") or "") + first_item: Final = _first_input_item(input) + anchor: Final = json.dumps( + (tenant, instructions or "", first_item), + sort_keys=True, + default=str, + ) + return f"litellm-derived-{sha256(anchor.encode('utf-8')).hexdigest()[:32]}" diff --git a/litellm/llms/chatgpt/responses/transformation.py b/litellm/llms/chatgpt/responses/transformation.py index 72b5cde21d1..dd827942232 100644 --- a/litellm/llms/chatgpt/responses/transformation.py +++ b/litellm/llms/chatgpt/responses/transformation.py @@ -31,10 +31,13 @@ from ..authenticator import Authenticator, get_cached_authenticator, get_chatgpt from ..common_utils import ( CHATGPT_API_BASE, GetAccessTokenError, + derive_chatgpt_session_id, ensure_chatgpt_session_id, get_chatgpt_client_credential, get_chatgpt_default_headers, get_chatgpt_default_instructions, + get_explicit_chatgpt_session_id, + should_derive_chatgpt_session_id, strip_chatgpt_client_credential_headers, ) @@ -122,13 +125,20 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): ) base_instructions: Final = get_chatgpt_default_instructions() existing_instructions: Final = request.get("instructions") - if existing_instructions: - if base_instructions not in existing_instructions: - request["instructions"] = f"{base_instructions}\n\n{existing_instructions}" - else: - request["instructions"] = base_instructions + instructions: Final = ( + f"{base_instructions}\n\n{existing_instructions}" + if existing_instructions and base_instructions not in existing_instructions + else f"{existing_instructions}" + if existing_instructions + else base_instructions + ) + request["instructions"] = instructions request["store"] = False request["stream"] = True + should_derive: Final = should_derive_chatgpt_session_id(litellm_params) + if should_derive and get_explicit_chatgpt_session_id(litellm_params) is None: + derived: Final = derive_chatgpt_session_id(litellm_params, instructions, input) + headers["session_id"] = derived # rebind-ok: must land in the dict validate_environment built include: Final = list(request.get("include") or []) if "reasoning.encrypted_content" not in include: include.append("reasoning.encrypted_content") diff --git a/tests/unit/llms/chatgpt/responses/test_chatgpt_responses_transformation.py b/tests/unit/llms/chatgpt/responses/test_chatgpt_responses_transformation.py index 635363e77ea..fdfa157d4ca 100644 --- a/tests/unit/llms/chatgpt/responses/test_chatgpt_responses_transformation.py +++ b/tests/unit/llms/chatgpt/responses/test_chatgpt_responses_transformation.py @@ -266,6 +266,109 @@ class TestChatGPTResponsesAPITransformation: "function": {"name": "hello"}, } + def _transform_headers( + self, input: list[dict[str, str]], litellm_params: GenericLiteLLMParams + ) -> dict[str, str]: + headers: dict[str, str] = {} + ChatGPTResponsesAPIConfig().transform_responses_api_request( + model="chatgpt/gpt-5.4", + input=input, + response_api_optional_request_params={}, + litellm_params=litellm_params, + headers=headers, + ) + return headers + + def test_derived_session_id_is_stable_as_conversation_grows(self): + params = GenericLiteLLMParams(chatgpt_derive_session_id=True) + first_turn = [{"role": "user", "content": "start of conversation"}] + later_turn = first_turn + [ + {"role": "assistant", "content": "reply"}, + {"role": "user", "content": "follow-up"}, + ] + + headers_one = self._transform_headers(first_turn, params) + headers_two = self._transform_headers(later_turn, params) + + assert headers_one["session_id"].startswith("litellm-derived-") + assert headers_one["session_id"] == headers_two["session_id"] + + def test_derived_session_id_differs_across_conversations(self): + params = GenericLiteLLMParams(chatgpt_derive_session_id=True) + + headers_one = self._transform_headers( + [{"role": "user", "content": "conversation a"}], params + ) + headers_two = self._transform_headers( + [{"role": "user", "content": "conversation b"}], params + ) + + assert headers_one["session_id"] != headers_two["session_id"] + + def test_explicit_session_id_wins_over_derivation(self): + params = GenericLiteLLMParams( + chatgpt_derive_session_id=True, litellm_session_id="explicit-1" + ) + + headers = self._transform_headers([{"role": "user", "content": "hi"}], params) + + assert "session_id" not in headers + + def test_proxy_trace_id_does_not_block_derivation(self): + params = GenericLiteLLMParams( + chatgpt_derive_session_id=True, + litellm_trace_id="b6c2977a-0652-43de-a9a6-165822f439f6", + ) + headers = {"session_id": params.litellm_trace_id} + + ChatGPTResponsesAPIConfig().transform_responses_api_request( + model="chatgpt/gpt-5.4", + input=[{"role": "user", "content": "hi"}], + response_api_optional_request_params={}, + litellm_params=params, + headers=headers, + ) + + assert headers["session_id"].startswith("litellm-derived-") + + def test_different_api_keys_derive_different_session_ids(self): + input = [{"role": "user", "content": "shared prefix"}] + params_a = GenericLiteLLMParams( + chatgpt_derive_session_id=True, + litellm_metadata={"user_api_key_hash": "key-a"}, + ) + params_b = GenericLiteLLMParams( + chatgpt_derive_session_id=True, + litellm_metadata={"user_api_key_hash": "key-b"}, + ) + + headers_a = self._transform_headers(input, params_a) + headers_b = self._transform_headers(input, params_b) + + assert headers_a["session_id"] != headers_b["session_id"] + assert headers_a["session_id"] == self._transform_headers(input, params_a)["session_id"] + + def test_quoted_false_string_does_not_enable_derivation(self): + params = GenericLiteLLMParams(chatgpt_derive_session_id="false") + + headers = self._transform_headers([{"role": "user", "content": "hi"}], params) + + assert "session_id" not in headers + + def test_quoted_true_string_enables_derivation(self): + params = GenericLiteLLMParams(chatgpt_derive_session_id="true") + + headers = self._transform_headers([{"role": "user", "content": "hi"}], params) + + assert headers["session_id"].startswith("litellm-derived-") + + def test_no_derivation_without_flag(self): + headers = self._transform_headers( + [{"role": "user", "content": "hi"}], GenericLiteLLMParams() + ) + + assert "session_id" not in headers + @pytest.mark.parametrize( ("model_name", "response_model"), [