From 9a0a37fffa1e7fe61e70b0d13738ed1bc2f0212b Mon Sep 17 00:00:00 2001 From: Xianzong Xie Date: Thu, 4 Dec 2025 14:11:13 -0800 Subject: [PATCH] feat: extract all ResponsesAPIResponse fields from response.completed - Add support for all ResponsesAPIResponse fields in update_state - Extract model, instructions, temperature, top_p, max_output_tokens, previous_response_id, text, truncation, parallel_tool_calls, user, store, and incomplete_details from response.completed event - Pass all fields to final update_state call Committed-By-Agent: cursor --- .../response_polling/background_streaming.py | 47 ++++++++++++++++++- .../proxy/response_polling/polling_handler.py | 47 +++++++++++++++++++ 2 files changed, 92 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/response_polling/background_streaming.py b/litellm/proxy/response_polling/background_streaming.py index a0ce4d82214..b0dcb69a82e 100644 --- a/litellm/proxy/response_polling/background_streaming.py +++ b/litellm/proxy/response_polling/background_streaming.py @@ -87,10 +87,25 @@ async def background_streaming_task( # noqa: PLR0915 # https://platform.openai.com/docs/api-reference/responses-streaming output_items = {} # Track output items by ID accumulated_text = {} # Track accumulated text deltas by (item_id, content_index) + + # ResponsesAPIResponse fields to extract from response.completed usage_data = None reasoning_data = None tool_choice_data = None tools_data = None + model_data = None + instructions_data = None + temperature_data = None + top_p_data = None + max_output_tokens_data = None + previous_response_id_data = None + text_data = None + truncation_data = None + parallel_tool_calls_data = None + user_data = None + store_data = None + incomplete_details_data = None + state_dirty = False # Track if state needs to be synced last_update_time = asyncio.get_event_loop().time() UPDATE_INTERVAL = 0.150 # 150ms batching interval @@ -201,14 +216,30 @@ async def background_streaming_task( # noqa: PLR0915 ) elif event_type == "response.completed": - # Response completed - includes usage, reasoning, tools, tool_choice + # Response completed - extract all ResponsesAPIResponse fields # https://platform.openai.com/docs/api-reference/responses-streaming/response-completed response_data = event.get("response", {}) + + # Core response fields usage_data = response_data.get("usage") reasoning_data = response_data.get("reasoning") tool_choice_data = response_data.get("tool_choice") tools_data = response_data.get("tools") + # Additional ResponsesAPIResponse fields + model_data = response_data.get("model") + instructions_data = response_data.get("instructions") + temperature_data = response_data.get("temperature") + top_p_data = response_data.get("top_p") + max_output_tokens_data = response_data.get("max_output_tokens") + previous_response_id_data = response_data.get("previous_response_id") + text_data = response_data.get("text") + truncation_data = response_data.get("truncation") + parallel_tool_calls_data = response_data.get("parallel_tool_calls") + user_data = response_data.get("user") + store_data = response_data.get("store") + incomplete_details_data = response_data.get("incomplete_details") + # Also update output from final response if available if "output" in response_data: final_output = response_data.get("output", []) @@ -230,7 +261,7 @@ async def background_streaming_task( # noqa: PLR0915 # Final flush to ensure all accumulated state is saved await flush_state_if_needed(force=True) - # Mark as completed with all response data + # Mark as completed with all ResponsesAPIResponse fields await polling_handler.update_state( polling_id=polling_id, status="completed", @@ -238,6 +269,18 @@ async def background_streaming_task( # noqa: PLR0915 reasoning=reasoning_data, tool_choice=tool_choice_data, tools=tools_data, + model=model_data, + instructions=instructions_data, + temperature=temperature_data, + top_p=top_p_data, + max_output_tokens=max_output_tokens_data, + previous_response_id=previous_response_id_data, + text=text_data, + truncation=truncation_data, + parallel_tool_calls=parallel_tool_calls_data, + user=user_data, + store=store_data, + incomplete_details=incomplete_details_data, ) verbose_proxy_logger.info( diff --git a/litellm/proxy/response_polling/polling_handler.py b/litellm/proxy/response_polling/polling_handler.py index 44ba835726e..650846663e7 100644 --- a/litellm/proxy/response_polling/polling_handler.py +++ b/litellm/proxy/response_polling/polling_handler.py @@ -93,6 +93,18 @@ class ResponsePollingHandler: tool_choice: Optional[Any] = None, tools: Optional[list] = None, output: Optional[list] = None, + # Additional ResponsesAPIResponse fields + model: Optional[str] = None, + instructions: Optional[str] = None, + temperature: Optional[float] = None, + top_p: Optional[float] = None, + max_output_tokens: Optional[int] = None, + previous_response_id: Optional[str] = None, + text: Optional[Dict] = None, + truncation: Optional[str] = None, + parallel_tool_calls: Optional[bool] = None, + user: Optional[str] = None, + store: Optional[bool] = None, ) -> None: """ Update the polling state in Redis @@ -110,6 +122,17 @@ class ResponsePollingHandler: tool_choice: Tool choice configuration from response.completed tools: Tools list from response.completed output: Full output list to replace current output + model: Model identifier + instructions: System instructions + temperature: Sampling temperature + top_p: Nucleus sampling parameter + max_output_tokens: Maximum output tokens + previous_response_id: ID of previous response in conversation + text: Text configuration + truncation: Truncation setting + parallel_tool_calls: Whether parallel tool calls are enabled + user: User identifier + store: Whether to store the response """ if not self.redis_cache: return @@ -156,6 +179,30 @@ class ResponsePollingHandler: if tools is not None: state["tools"] = tools + # Update additional ResponsesAPIResponse fields + if model is not None: + state["model"] = model + if instructions is not None: + state["instructions"] = instructions + if temperature is not None: + state["temperature"] = temperature + if top_p is not None: + state["top_p"] = top_p + if max_output_tokens is not None: + state["max_output_tokens"] = max_output_tokens + if previous_response_id is not None: + state["previous_response_id"] = previous_response_id + if text is not None: + state["text"] = text + if truncation is not None: + state["truncation"] = truncation + if parallel_tool_calls is not None: + state["parallel_tool_calls"] = parallel_tool_calls + if user is not None: + state["user"] = user + if store is not None: + state["store"] = store + # Update cache with configured TTL await self.redis_cache.async_set_cache( key=cache_key,