scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-08 03:08:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 115

Add support for prompt management for responses

Browse source
This commit is contained in:
Sameer Kankute 2026-03-18 15:48:31 +05:30
parent cec3e9e7d4
commit 0d70864d09
5 changed files with 314 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

48
docs/my-website/docs/prompt_management.md Normal file
View file

@ -0,0 +1,48 @@
---
title: Prompt Management with Responses API
---
# Prompt Management with Responses API
Use LiteLLM Prompt Management with `/v1/responses` by passing `prompt_id` and optional `prompt_variables`.
## Basic Usage
```bash
curl -X POST "http://localhost:4000/v1/responses" \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "gpt-4o",
"prompt_id": "my-responses-prompt",
"prompt_variables": {"topic": "large language models"},
"input": []
}'
```
## Multi-turn Follow-up in `input`
To send follow-up turns in one request, pass message history in `input`.
```bash
curl -X POST "http://localhost:4000/v1/responses" \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "gpt-4o",
"prompt_id": "my-responses-prompt",
"prompt_variables": {"topic": "large language models"},
"input": [
{"role": "user", "content": "Topic is LLMs. Start short."},
{"role": "assistant", "content": "Sure, go ahead."},
{"role": "user", "content": "Now give me 3 bullets and include pricing caveat."}
]
}'
```
## Notes
- Prompt template messages are merged with your `input` messages.
- Prompt variable substitution applies to prompt message content.
- Tool call payload fields are not substituted by prompt variables.
- For follow-ups with `previous_response_id`, include `prompt_id` again if you want prompt management applied on that turn.

19
docs/my-website/docs/proxy/prompt_management.md
View file

@ -311,7 +311,7 @@ litellm_settings:
1. **At Startup**: When the proxy starts, it reads the `prompts` field from `config.yaml`
2. **Initialization**: Each prompt is initialized based on its `prompt_integration` type
3. **In-Memory Storage**: Prompts are stored in the `IN_MEMORY_PROMPT_REGISTRY`
4. **Access**: Use these prompts via the `/v1/chat/completions` endpoint with `prompt_id` in the request
4. **Access**: Use these prompts via `/v1/chat/completions` or `/v1/responses` with `prompt_id` in the request
### Using Config-Loaded Prompts
@ -331,6 +331,23 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
}'
```
You can also use the same `prompt_id` with the Responses API:
```bash
curl -L -X POST 'http://0.0.0.0:4000/v1/responses' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"model": "gpt-4o",
"prompt_id": "coding_assistant",
"prompt_variables": {
"language": "python",
"task": "create a web scraper"
},
"input": []
}'
```
### Prompt Schema Reference
Each prompt in the `prompts` list requires:

1
docs/my-website/sidebars.js
View file

@ -687,6 +687,7 @@ const sidebars = {
"proxy/realtime_webrtc",
"rerank",
"response_api",
"prompt_management",
"response_api_compact",
{
type: "category",

36
litellm/responses/main.py
View file

@ -37,6 +37,7 @@ from litellm.responses.litellm_completion_transformation.handler import (
)
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.llms.openai import (
AllMessageValues,
PromptObject,
Reasoning,
ResponseIncludable,
@ -623,6 +624,41 @@ def responses(
if dynamic_api_base is not None:
litellm_params.api_base = dynamic_api_base
#########################################################
# PROMPT MANAGEMENT
#########################################################
prompt_id = cast(Optional[str], kwargs.get("prompt_id", None))
prompt_variables = cast(Optional[dict], kwargs.get("prompt_variables", None))
if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and (
litellm_logging_obj.should_run_prompt_management_hooks(
prompt_id=prompt_id, non_default_params=kwargs
)
):
client_input: List[AllMessageValues] = (
[{"role": "user", "content": input}]
if isinstance(input, str)
else cast(List[AllMessageValues], list(input))
)
(
model,
merged_input,
merged_optional_params,
) = litellm_logging_obj.get_chat_completion_prompt(
model=model,
messages=client_input,
non_default_params=kwargs,
prompt_id=prompt_id,
prompt_variables=prompt_variables,
prompt_label=kwargs.get("prompt_label", None),
prompt_version=kwargs.get("prompt_version", None),
)
input = cast(Union[str, ResponseInputParam], merged_input)
local_vars["input"] = input
# Apply prompt_template_optional_params (e.g. temperature, instructions)
# by updating kwargs so they flow into local_vars → response_api_optional_params
kwargs.update(merged_optional_params)
#########################################################
# Update input and tools with provider-specific file IDs if managed files are used
#########################################################

211
tests/test_litellm/responses/test_responses_prompt_management.py Normal file
View file

@ -0,0 +1,211 @@
"""
Unit tests for prompt management support in the Responses API.
Covers:
A) str input is coerced to a message list before merging with the template
B) list input is merged with the template
C) no prompt_id → hook is skipped, input is unchanged
D) model override from the prompt template is applied
"""
from typing import List
from unittest.mock import MagicMock, patch
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.types.llms.openai import AllMessageValues
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _make_logging_obj(
merged_model: str,
merged_messages: List[AllMessageValues],
should_run: bool = True,
) -> MagicMock:
"""Return a mock LiteLLMLoggingObj pre-configured for prompt management."""
logging_obj = MagicMock()
# Make isinstance(logging_obj, LiteLLMLoggingObj) return True
logging_obj.__class__ = LiteLLMLoggingObj
logging_obj.should_run_prompt_management_hooks.return_value = should_run
logging_obj.get_chat_completion_prompt.return_value = (
merged_model,
merged_messages,
{},
)
# Instance attribute accessed by post-call metadata utilities
logging_obj.model_call_details = {}
return logging_obj
def _patch_responses_dispatch():
"""Patch everything after the prompt management block so tests stay unit-level."""
return [
patch(
"litellm.responses.main.litellm.get_llm_provider",
return_value=("gpt-4o", "openai", None, None),
),
patch(
"litellm.responses.mcp.litellm_proxy_mcp_handler."
"LiteLLM_Proxy_MCP_Handler._should_use_litellm_mcp_gateway",
return_value=False,
),
patch(
"litellm.responses.main.ProviderConfigManager"
".get_provider_responses_api_config",
return_value=None,
),
patch(
"litellm.responses.main.litellm_completion_transformation_handler"
".response_api_handler",
return_value=MagicMock(),
),
]
# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------
class TestResponsesAPIPromptManagement:
def test_str_input_coerced_and_merged(self):
"""[A] str input is wrapped into a message list before being passed to the hook."""
template_messages: List[AllMessageValues] = [
{"role": "system", "content": "You are a summariser."}, # type: ignore[list-item]
]
client_message: List[AllMessageValues] = [
{"role": "user", "content": "Tell me about AI."}, # type: ignore[list-item]
]
expected_merged = template_messages + client_message
logging_obj = _make_logging_obj(
merged_model="openai/gpt-4o",
merged_messages=expected_merged,
)
patches = _patch_responses_dispatch()
with patches[0], patches[1], patches[2], patches[3]:
import litellm
litellm.responses(
input="Tell me about AI.",
model="gpt-4o",
prompt_id="summariser-prompt",
prompt_variables={},
litellm_logging_obj=logging_obj,
)
logging_obj.get_chat_completion_prompt.assert_called_once()
call_kwargs = logging_obj.get_chat_completion_prompt.call_args.kwargs
# str was coerced to a single user message before being passed to the hook
assert call_kwargs["messages"] == [
{"role": "user", "content": "Tell me about AI."}
]
assert call_kwargs["prompt_id"] == "summariser-prompt"
def test_list_input_merged_with_template(self):
"""[B] list input is passed directly to the hook and merged with the template."""
template_messages: List[AllMessageValues] = [
{"role": "system", "content": "You are helpful."}, # type: ignore[list-item]
]
client_messages = [
{"role": "user", "content": [{"type": "input_text", "text": "Hello"}]},
]
expected_merged = template_messages + client_messages # type: ignore[operator]
logging_obj = _make_logging_obj(
merged_model="openai/gpt-4o",
merged_messages=expected_merged, # type: ignore[arg-type]
)
patches = _patch_responses_dispatch()
with patches[0], patches[1], patches[2], patches[3]:
import litellm
litellm.responses(
input=client_messages, # type: ignore[arg-type]
model="gpt-4o",
prompt_id="helper-prompt",
litellm_logging_obj=logging_obj,
)
logging_obj.get_chat_completion_prompt.assert_called_once()
call_kwargs = logging_obj.get_chat_completion_prompt.call_args.kwargs
assert call_kwargs["messages"] == client_messages
def test_no_prompt_id_skips_hook(self):
"""[C] When prompt_id is absent, prompt management hooks are not called."""
logging_obj = _make_logging_obj(
merged_model="openai/gpt-4o",
merged_messages=[],
should_run=False,
)
patches = _patch_responses_dispatch()
with patches[0], patches[1], patches[2], patches[3]:
import litellm
litellm.responses(
input="Hello",
model="gpt-4o",
litellm_logging_obj=logging_obj,
)
logging_obj.get_chat_completion_prompt.assert_not_called()
def test_optional_params_from_template_applied(self):
"""[E] prompt_template_optional_params (e.g. temperature) flow into the request."""
template_messages: List[AllMessageValues] = [
{"role": "user", "content": "Hello"}, # type: ignore[list-item]
]
# Simulate get_chat_completion_prompt returning merged optional params
# that include a template-defined temperature
merged_kwargs = {"temperature": 0.2, "prompt_id": "t", "litellm_logging_obj": None}
logging_obj = MagicMock()
logging_obj.__class__ = LiteLLMLoggingObj
logging_obj.should_run_prompt_management_hooks.return_value = True
logging_obj.get_chat_completion_prompt.return_value = (
"openai/gpt-4o",
template_messages,
merged_kwargs,
)
logging_obj.model_call_details = {}
patches = _patch_responses_dispatch()
with patches[0], patches[1], patches[2], patches[3] as mock_handler:
import litellm
litellm.responses(
input="Hello",
model="gpt-4o",
prompt_id="t",
litellm_logging_obj=logging_obj,
)
# temperature from the template should reach the downstream handler via local_vars
handler_call_kwargs = mock_handler.call_args.kwargs
request_params = handler_call_kwargs.get("responses_api_request", {})
assert request_params.get("temperature") == 0.2
def test_model_override_from_template(self):
"""[D] Model returned by the prompt hook overrides the original request model."""
template_messages: List[AllMessageValues] = [
{"role": "user", "content": "{{query}}"}, # type: ignore[list-item]
]
logging_obj = _make_logging_obj(
merged_model="openai/gpt-4o-mini", # overridden model from template
merged_messages=template_messages,
)
patches = _patch_responses_dispatch()
with patches[0], patches[1], patches[2], patches[3] as mock_handler:
import litellm
litellm.responses(
input="What is AI?",
model="gpt-4o",
prompt_id="query-prompt",
prompt_variables={"query": "What is AI?"},
litellm_logging_obj=logging_obj,
)
# The model passed to the downstream handler should be the overridden one
handler_call_kwargs = mock_handler.call_args.kwargs
assert handler_call_kwargs.get("model") == "openai/gpt-4o-mini"