diff --git a/docs/my-website/docs/providers/openai/responses_api.md b/docs/my-website/docs/providers/openai/responses_api.md index e88512ecfd4..3dcf3096159 100644 --- a/docs/my-website/docs/providers/openai/responses_api.md +++ b/docs/my-website/docs/providers/openai/responses_api.md @@ -207,6 +207,50 @@ print(delete_response) |----------|---------------------| | `openai` | [All Responses API parameters are supported](https://github.com/BerriAI/litellm/blob/7c3df984da8e4dff9201e4c5353fdc7a2b441831/litellm/llms/openai/responses/transformation.py#L23) | +### Reusable Prompts + +Use the `prompt` parameter to reference a stored prompt template and optionally supply variables. + +```python showLineNumbers title="Stored Prompt" +import litellm + +response = litellm.responses( + model="openai/o1-pro", + prompt={ + "id": "pmpt_abc123", + "version": "2", + "variables": { + "customer_name": "Jane Doe", + "product": "40oz juice box", + }, + }, +) + +print(response) +``` + +The same parameter is supported when calling the LiteLLM proxy with the OpenAI SDK: + +```python showLineNumbers title="Stored Prompt via Proxy" +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:4000", api_key="your-api-key") + +response = client.responses.create( + model="openai/o1-pro", + prompt={ + "id": "pmpt_abc123", + "version": "2", + "variables": { + "customer_name": "Jane Doe", + "product": "40oz juice box", + }, + }, +) + +print(response) +``` + ## Computer Use diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index f6065198c39..871662eccaf 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -38,6 +38,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): "store", "background", "stream", + "prompt", "temperature", "text", "tool_choice", diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 61d160c7a78..888bf496d9a 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -15,6 +15,7 @@ from litellm.responses.litellm_completion_transformation.handler import ( ) from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ( + PromptObject, Reasoning, ResponseIncludable, ResponseInputParam, @@ -96,6 +97,7 @@ async def aresponses( include: Optional[List[ResponseIncludable]] = None, instructions: Optional[str] = None, max_output_tokens: Optional[int] = None, + prompt: Optional[PromptObject] = None, metadata: Optional[Dict[str, Any]] = None, parallel_tool_calls: Optional[bool] = None, previous_response_id: Optional[str] = None, @@ -141,6 +143,7 @@ async def aresponses( include=include, instructions=instructions, max_output_tokens=max_output_tokens, + prompt=prompt, metadata=metadata, parallel_tool_calls=parallel_tool_calls, previous_response_id=previous_response_id, @@ -197,6 +200,7 @@ def responses( include: Optional[List[ResponseIncludable]] = None, instructions: Optional[str] = None, max_output_tokens: Optional[int] = None, + prompt: Optional[PromptObject] = None, metadata: Optional[Dict[str, Any]] = None, parallel_tool_calls: Optional[bool] = None, previous_response_id: Optional[str] = None, @@ -255,11 +259,11 @@ def responses( ) # get provider config - responses_api_provider_config: Optional[ - BaseResponsesAPIConfig - ] = ProviderConfigManager.get_provider_responses_api_config( - model=model, - provider=litellm.LlmProviders(custom_llm_provider), + responses_api_provider_config: Optional[BaseResponsesAPIConfig] = ( + ProviderConfigManager.get_provider_responses_api_config( + model=model, + provider=litellm.LlmProviders(custom_llm_provider), + ) ) local_vars.update(kwargs) @@ -449,11 +453,11 @@ def delete_responses( raise ValueError("custom_llm_provider is required but passed as None") # get provider config - responses_api_provider_config: Optional[ - BaseResponsesAPIConfig - ] = ProviderConfigManager.get_provider_responses_api_config( - model=None, - provider=litellm.LlmProviders(custom_llm_provider), + responses_api_provider_config: Optional[BaseResponsesAPIConfig] = ( + ProviderConfigManager.get_provider_responses_api_config( + model=None, + provider=litellm.LlmProviders(custom_llm_provider), + ) ) if responses_api_provider_config is None: @@ -628,11 +632,11 @@ def get_responses( raise ValueError("custom_llm_provider is required but passed as None") # get provider config - responses_api_provider_config: Optional[ - BaseResponsesAPIConfig - ] = ProviderConfigManager.get_provider_responses_api_config( - model=None, - provider=litellm.LlmProviders(custom_llm_provider), + responses_api_provider_config: Optional[BaseResponsesAPIConfig] = ( + ProviderConfigManager.get_provider_responses_api_config( + model=None, + provider=litellm.LlmProviders(custom_llm_provider), + ) ) if responses_api_provider_config is None: @@ -784,11 +788,11 @@ def list_input_items( if custom_llm_provider is None: raise ValueError("custom_llm_provider is required but passed as None") - responses_api_provider_config: Optional[ - BaseResponsesAPIConfig - ] = ProviderConfigManager.get_provider_responses_api_config( - model=None, - provider=litellm.LlmProviders(custom_llm_provider), + responses_api_provider_config: Optional[BaseResponsesAPIConfig] = ( + ProviderConfigManager.get_provider_responses_api_config( + model=None, + provider=litellm.LlmProviders(custom_llm_provider), + ) ) if responses_api_provider_config is None: diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 4d95c0f837b..65dacd199bb 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -930,6 +930,19 @@ class ComputerToolParam(TypedDict, total=False): ALL_RESPONSES_API_TOOL_PARAMS = Union[ToolParam, ComputerToolParam] +class PromptObject(TypedDict, total=False): + """Reference to a stored prompt template.""" + + id: Required[str] + """The unique identifier of the prompt template to use.""" + + variables: Optional[Dict] + """Variables to substitute into the prompt template.""" + + version: Optional[str] + """Optional version of the prompt template.""" + + class ResponsesAPIOptionalRequestParams(TypedDict, total=False): """TypedDict for Optional parameters supported by the responses API.""" @@ -950,6 +963,7 @@ class ResponsesAPIOptionalRequestParams(TypedDict, total=False): top_p: Optional[float] truncation: Optional[Literal["auto", "disabled"]] user: Optional[str] + prompt: Optional[PromptObject] class ResponsesAPIRequestParams(ResponsesAPIOptionalRequestParams, total=False): diff --git a/tests/llm_responses_api_testing/test_openai_responses_api.py b/tests/llm_responses_api_testing/test_openai_responses_api.py index f1c2fc515c3..981b7efc77b 100644 --- a/tests/llm_responses_api_testing/test_openai_responses_api.py +++ b/tests/llm_responses_api_testing/test_openai_responses_api.py @@ -689,6 +689,80 @@ async def test_openai_responses_litellm_router_with_metadata(): mock_post.assert_called_once() +@pytest.mark.asyncio +async def test_openai_responses_litellm_router_with_prompt(): + """Test that prompt object is passed through the Router for responses API""" + + prompt_obj = { + "id": "pmpt_abc123", + "version": "2", + "variables": {"random_variable": "ishaan_from_litellm"}, + } + + mock_response = { + "id": "resp_123", + "object": "response", + "created_at": 1741476542, + "status": "completed", + "model": "gpt-4o", + "output": [], + "parallel_tool_calls": True, + "usage": {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}, + "text": {"format": {"type": "text"}}, + "error": None, + "incomplete_details": None, + "instructions": None, + "metadata": {}, + "temperature": 1.0, + "tool_choice": "auto", + "tools": [], + "top_p": 1.0, + "max_output_tokens": None, + "previous_response_id": None, + "reasoning": {"effort": None, "summary": None}, + "truncation": "disabled", + "user": None, + } + + class MockResponse: + def __init__(self, json_data, status_code): + self._json_data = json_data + self.status_code = status_code + self.text = str(json_data) + + def json(self): + return self._json_data + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new_callable=AsyncMock, + ) as mock_post: + mock_post.return_value = MockResponse(mock_response, 200) + + litellm._turn_on_debug() + router = litellm.Router( + model_list=[ + { + "model_name": "gpt4o-special-alias", + "litellm_params": { + "model": "gpt-4o", + "api_key": "fake-key", + }, + } + ] + ) + + await router.aresponses( + model="gpt4o-special-alias", + input="Hello", + prompt=prompt_obj, + ) + + request_body = mock_post.call_args.kwargs["json"] + assert request_body["prompt"] == prompt_obj + mock_post.assert_called_once() + + def test_bad_request_bad_param_error(): """Raise a BadRequestError when an invalid parameter value is provided""" try: diff --git a/tests/test_litellm/responses/test_responses_utils.py b/tests/test_litellm/responses/test_responses_utils.py index af01ca56ddb..950b342b347 100644 --- a/tests/test_litellm/responses/test_responses_utils.py +++ b/tests/test_litellm/responses/test_responses_utils.py @@ -25,7 +25,11 @@ class TestResponsesAPIRequestUtils: model = "gpt-4o" config = OpenAIResponsesAPIConfig() optional_params = ResponsesAPIOptionalRequestParams( - {"temperature": 0.7, "max_output_tokens": 100} + { + "temperature": 0.7, + "max_output_tokens": 100, + "prompt": {"id": "pmpt_123"}, + } ) # Execute @@ -41,6 +45,8 @@ class TestResponsesAPIRequestUtils: assert result["temperature"] == 0.7 assert "max_output_tokens" in result assert result["max_output_tokens"] == 100 + assert "prompt" in result + assert result["prompt"] == {"id": "pmpt_123"} def test_get_optional_params_responses_api_unsupported_param(self): """Test that unsupported parameters raise an error""" @@ -68,6 +74,7 @@ class TestResponsesAPIRequestUtils: params = { "temperature": 0.7, "max_output_tokens": 100, + "prompt": {"id": "pmpt_456"}, "invalid_param": "value", "model": "gpt-4o", # This is not in ResponsesAPIOptionalRequestParams } @@ -84,6 +91,7 @@ class TestResponsesAPIRequestUtils: assert "model" not in result assert result["temperature"] == 0.7 assert result["max_output_tokens"] == 100 + assert result["prompt"] == {"id": "pmpt_456"} def test_decode_previous_response_id_to_original_previous_response_id(self): """Test decoding a LiteLLM encoded previous_response_id to the original previous_response_id"""