diff --git a/litellm/__init__.py b/litellm/__init__.py index ccfbf80369f..3668e6efb0c 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -343,6 +343,7 @@ _anthropic_prompt_caching_ttl_env: Optional[str] = os.getenv("LITELLM_ANTHROPIC_ anthropic_prompt_caching_ttl: Optional[Literal["5m", "1h"]] = ( "1h" if _anthropic_prompt_caching_ttl_env == "1h" else "5m" if _anthropic_prompt_caching_ttl_env == "5m" else None ) +openai_system_messages_first: bool = False disable_vertex_batch_output_transformation: bool = False extra_spend_tag_headers: Optional[List[str]] = None in_memory_llm_clients_cache: "LLMClientCache" diff --git a/litellm/constants.py b/litellm/constants.py index ba5ec73d435..1dbb8a842fb 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1776,6 +1776,7 @@ DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD = float(os.getenv("DEFAULT_PROMPT_ LENGTH_OF_LITELLM_GENERATED_KEY: Final = int(os.getenv("LENGTH_OF_LITELLM_GENERATED_KEY", 16)) MINIMUM_CUSTOM_KEY_LENGTH: Final = int(os.getenv("MINIMUM_CUSTOM_KEY_LENGTH", 16)) SECRET_MANAGER_REFRESH_INTERVAL: Final = int(os.getenv("SECRET_MANAGER_REFRESH_INTERVAL", 86400)) +OPENAI_SYSTEM_MESSAGES_FIRST_PROVIDERS: Final = frozenset({"openai", "azure"}) LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [ "default_internal_user_params", "default_team_params", @@ -1793,6 +1794,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [ # test_general_settings_ui_fields_are_db_overridable enforces that pairing. "enable_anthropic_prompt_caching", "anthropic_prompt_caching_ttl", + "openai_system_messages_first", "max_ui_session_budget", "budget_rollover", "mcp_tool_search", diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 2485896184e..7fedefa4025 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -2256,6 +2256,22 @@ def drop_tool_reference_parts_from_tool_messages( return [_drop_tool_reference_parts(message) for message in messages] # mutable-ok: pipelines mutate message lists +INSTRUCTION_MESSAGE_ROLES: Final = frozenset({"system", "developer"}) + + +def _is_instruction_message(message: AllMessageValues) -> bool: + return message.get("role") in INSTRUCTION_MESSAGE_ROLES + + +def system_messages_first( + messages: list[AllMessageValues], # mutable-ok: message pipelines type messages as mutable lists +) -> list[AllMessageValues]: # mutable-ok: message pipelines type messages as mutable lists + return [ # mutable-ok: pipelines mutate message lists + *(message for message in messages if _is_instruction_message(message)), + *(message for message in messages if not _is_instruction_message(message)), + ] + + def _attempt_json_repair(s: str) -> object | None: """ Attempt to repair truncated JSON produced by LLM tool calls. diff --git a/litellm/llms/azure/chat/gpt_transformation.py b/litellm/llms/azure/chat/gpt_transformation.py index ed16d7f3de0..6d17a1359bc 100644 --- a/litellm/llms/azure/chat/gpt_transformation.py +++ b/litellm/llms/azure/chat/gpt_transformation.py @@ -9,6 +9,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( drop_tool_reference_parts_from_tool_messages, flatten_combinators_and_drop_non_python_regex_patterns, hoist_images_from_tool_messages, + system_messages_first, tool_with_sanitized_parameters, ) from litellm.litellm_core_utils.prompt_templates.factory import ( @@ -276,7 +277,8 @@ class AzureOpenAIConfig(BaseConfig): litellm_params: dict, headers: dict, ) -> dict: - stripped_messages: Final = drop_tool_reference_parts_from_tool_messages(messages) + ordered_messages: Final = system_messages_first(messages) if litellm.openai_system_messages_first else messages + stripped_messages: Final = drop_tool_reference_parts_from_tool_messages(ordered_messages) azure_messages: Final = convert_to_azure_openai_messages(hoist_images_from_tool_messages(stripped_messages)) return { "model": model, diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 9b410cf073e..9dbcf0cc089 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -12,6 +12,7 @@ from urllib.parse import urlparse import httpx import litellm +from litellm.constants import OPENAI_SYSTEM_MESSAGES_FIRST_PROVIDERS from litellm.litellm_core_utils.core_helpers import map_finish_reason from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( _extract_reasoning_content, @@ -24,6 +25,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( flatten_combinators_and_drop_non_python_regex_patterns, get_tool_call_names, hoist_images_from_tool_messages, + system_messages_first, tool_with_sanitized_parameters, ) from litellm.litellm_core_utils.prompt_templates.image_handling import ( @@ -463,6 +465,15 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): ] return MappingProxyType({"tools": sanitized}) + def _prompt_cache_ordered_messages( + self, messages: list[AllMessageValues], litellm_params: Mapping[str, object] + ) -> list[AllMessageValues]: + if not litellm.openai_system_messages_first: + return messages + if litellm_params.get("custom_llm_provider") not in OPENAI_SYSTEM_MESSAGES_FIRST_PROVIDERS: + return messages + return system_messages_first(messages) + def transform_request( self, model: str, @@ -477,7 +488,9 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): Returns: dict: The transformed request. Sent as the body of the API call. """ - messages = self._transform_messages(messages=messages, model=model) + messages = self._transform_messages( + messages=self._prompt_cache_ordered_messages(messages, litellm_params), model=model + ) if not self._should_preserve_cache_control_for_endpoint( litellm_params.get("custom_llm_provider"), litellm_params.get("api_base") ): @@ -506,7 +519,9 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): litellm_params: dict, headers: dict, ) -> dict: - transformed_messages = await self._transform_messages(messages=messages, model=model, is_async=True) + transformed_messages = await self._transform_messages( + messages=self._prompt_cache_ordered_messages(messages, litellm_params), model=model, is_async=True + ) if not self._should_preserve_cache_control_for_endpoint( litellm_params.get("custom_llm_provider"), litellm_params.get("api_base") ): diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f63e088ebf7..a375f76dbdb 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -17465,6 +17465,16 @@ _GENERAL_SETTINGS_UI_LITELLM_FIELDS: Final[dict[str, GeneralSettingsUILiteLLMFie "tab": "prompt_caching", "description": "Empty uses Anthropic's 5m default. 1h suits long sessions but doubles the cache write cost.", }, + "openai_system_messages_first": { + "type": "Boolean", + "tab": "prompt_caching", + "description": ( + "Moves system and developer messages to the front of the messages array on OpenAI and " + "Azure OpenAI chat completions requests, keeping their relative order. OpenAI's prompt cache " + "matches on the exact prefix, so a system message that arrives mid-conversation otherwise " + "breaks the cached prefix on every turn." + ), + }, "budget_rollover": { # mutable-ok: registry literal, frozen with its siblings below "type": "Boolean", "description": ( diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index b5890d1a5b0..c67f72680a8 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -23,6 +23,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( responses_reasoning_items_from_thinking_blocks, split_concatenated_json_objects, strip_encrypted_reasoning_from_messages, + system_messages_first, update_messages_with_model_file_ids, ) @@ -1107,6 +1108,38 @@ def test_drop_tool_reference_parts_leaves_non_tool_messages_alone(): assert result[2]["content"] == "" +class TestSystemMessagesFirst: + def test_stable_partition_keeps_order_within_each_group(self): + messages = [ + {"role": "user", "content": "u1"}, + {"role": "system", "content": "s1"}, + {"role": "assistant", "content": "a1"}, + {"role": "developer", "content": "d1"}, + {"role": "tool", "tool_call_id": "c1", "content": "t1"}, + {"role": "system", "content": "s2"}, + ] + + result = system_messages_first(messages) + + assert [m["content"] for m in result] == ["s1", "d1", "s2", "u1", "a1", "t1"] + assert [m["content"] for m in messages] == ["u1", "s1", "a1", "d1", "t1", "s2"] + assert all( + result_message is original for result_message, original in zip(result[3:], messages[::2], strict=True) + ) + + @pytest.mark.parametrize( + "messages", + [ + [], + [{"role": "user", "content": "u1"}, {"role": "assistant", "content": "a1"}], + [{"role": "system", "content": "s1"}, {"role": "user", "content": "u1"}], + [{"role": "system", "content": "s1"}, {"role": "system", "content": "s2"}], + ], + ) + def test_already_ordered_messages_come_back_unchanged(self, messages): + assert system_messages_first(messages) == messages + + class TestFlattenTopLevelSchemaCombinators: def _customer_anyof_schema(self): return { diff --git a/tests/test_litellm/llms/azure/chat/test_azure_chat_gpt_transformation.py b/tests/test_litellm/llms/azure/chat/test_azure_chat_gpt_transformation.py index bc6cb0c0fed..e8b98c696e1 100644 --- a/tests/test_litellm/llms/azure/chat/test_azure_chat_gpt_transformation.py +++ b/tests/test_litellm/llms/azure/chat/test_azure_chat_gpt_transformation.py @@ -132,6 +132,32 @@ def test_transform_request_drops_tool_reference_parts(): assert request["messages"][2]["content"] == "" +@pytest.mark.parametrize( + "enabled, expected", [(False, ("hi", "sys", "reply", "more")), (True, ("sys", "hi", "reply", "more"))] +) +def test_transform_request_system_messages_first_follows_global_flag(monkeypatch, enabled, expected): + """Azure OpenAI shares OpenAI's prefix-matched prompt cache, so the same flag moves + system messages ahead of the conversation on the Azure request body.""" + monkeypatch.setattr(litellm, "openai_system_messages_first", enabled) + messages = [ + {"role": "user", "content": "hi"}, + {"role": "system", "content": "sys"}, + {"role": "assistant", "content": "reply"}, + {"role": "user", "content": "more"}, + ] + + request = AzureOpenAIConfig().transform_request( + model="gpt-4o", + messages=messages, + optional_params={}, + litellm_params={"custom_llm_provider": "azure"}, + headers={}, + ) + + assert tuple(m["content"] for m in request["messages"]) == expected + assert [m["content"] for m in messages] == ["hi", "sys", "reply", "more"] + + @pytest.mark.parametrize( "model, emitted_key, absent_key", [ diff --git a/tests/test_litellm/llms/azure/chat/test_azure_chat_o_series_transformation.py b/tests/test_litellm/llms/azure/chat/test_azure_chat_o_series_transformation.py index 202f81f1252..9db9ab971a0 100644 --- a/tests/test_litellm/llms/azure/chat/test_azure_chat_o_series_transformation.py +++ b/tests/test_litellm/llms/azure/chat/test_azure_chat_o_series_transformation.py @@ -68,3 +68,24 @@ def test_azure_o_series_transform_request_flattens_top_level_anyof(): assert parameters["required"] == ["id"] assert "anyOf" in tool["function"]["parameters"] assert optional_params["tools"][0] is tool + + +def test_azure_o_series_transform_request_moves_system_messages_first(monkeypatch): + monkeypatch.setattr(litellm, "openai_system_messages_first", True) + messages = [ + {"role": "user", "content": "hi"}, + {"role": "developer", "content": "dev"}, + {"role": "assistant", "content": "reply"}, + {"role": "user", "content": "more"}, + ] + + request = AzureOpenAIO1Config().transform_request( + model="o3-mini", + messages=messages, + optional_params={}, + litellm_params={"custom_llm_provider": "azure"}, + headers={}, + ) + + assert [m["content"] for m in request["messages"]] == ["dev", "hi", "reply", "more"] + assert [m["content"] for m in messages] == ["hi", "dev", "reply", "more"] diff --git a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py index b110586ae5b..53c5b9d7cbc 100644 --- a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py @@ -1124,6 +1124,80 @@ class TestToolReferenceStripping: assert request["messages"][2]["content"] == "" +class TestSystemMessagesFirst: + """With litellm.openai_system_messages_first on, requests bound for OpenAI put system and + developer messages ahead of the conversation, keeping each group's order, so the instruction + prefix stays byte-stable for OpenAI's prefix-matched prompt cache.""" + + MESSAGES: Final = ( + {"role": "user", "content": "first turn"}, + {"role": "system", "content": "sys 1"}, + {"role": "assistant", "content": "reply"}, + {"role": "developer", "content": "dev"}, + {"role": "user", "content": "second turn"}, + {"role": "system", "content": "sys 2"}, + ) + ORIGINAL_ORDER: Final = ("first turn", "sys 1", "reply", "dev", "second turn", "sys 2") + ORDERED: Final = ("sys 1", "dev", "sys 2", "first turn", "reply", "second turn") + + def setup_method(self): + self.config = OpenAIGPTConfig() + + def _messages(self): + return [dict(m) for m in self.MESSAGES] + + def _transform(self, provider): + return self.config.transform_request( + model="gpt-4.1", + messages=self._messages(), + optional_params={}, + litellm_params={"custom_llm_provider": provider}, + headers={}, + ) + + def test_default_off_keeps_caller_order(self, monkeypatch): + monkeypatch.setattr(litellm, "openai_system_messages_first", False) + assert tuple(m["content"] for m in self._transform("openai")["messages"]) == self.ORIGINAL_ORDER + + def test_moves_system_and_developer_messages_first_for_openai(self, monkeypatch): + monkeypatch.setattr(litellm, "openai_system_messages_first", True) + assert tuple(m["content"] for m in self._transform("openai")["messages"]) == self.ORDERED + + def test_leaves_openai_compatible_providers_alone(self, monkeypatch): + monkeypatch.setattr(litellm, "openai_system_messages_first", True) + assert tuple(m["content"] for m in self._transform("deepseek")["messages"]) == self.ORIGINAL_ORDER + + def test_does_not_mutate_caller_messages(self, monkeypatch): + monkeypatch.setattr(litellm, "openai_system_messages_first", True) + messages = self._messages() + self.config.transform_request( + model="gpt-4.1", + messages=messages, + optional_params={}, + litellm_params={"custom_llm_provider": "openai"}, + headers={}, + ) + assert tuple(m["content"] for m in messages) == self.ORIGINAL_ORDER + + @pytest.mark.asyncio + async def test_async_transform_request_moves_system_messages_first(self, monkeypatch): + class UninstantiatedOpenAIGPTConfig(OpenAIGPTConfig): + _is_base_class = True + + def __init__(self) -> None: + pass + + monkeypatch.setattr(litellm, "openai_system_messages_first", True) + request = await UninstantiatedOpenAIGPTConfig().async_transform_request( + model="gpt-4.1", + messages=self._messages(), + optional_params={}, + litellm_params={"custom_llm_provider": "openai"}, + headers={}, + ) + assert tuple(m["content"] for m in request["messages"]) == self.ORDERED + + class TestOpenAIPromptCacheBreakpointChatPath: """Chat-path shape for OpenAI explicit prompt caching (#37509).""" diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 42af8e0af21..c5f08632c43 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -10750,6 +10750,7 @@ def test_get_config_list_includes_anthropic_prompt_caching_fields(monkeypatch): monkeypatch.setattr(ps, "prisma_client", mock_prisma) monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) monkeypatch.setattr(litellm, "anthropic_prompt_caching_ttl", "1h") + monkeypatch.setattr(litellm, "openai_system_messages_first", False) app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN ) @@ -10771,6 +10772,10 @@ def test_get_config_list_includes_anthropic_prompt_caching_fields(monkeypatch): assert fields["enable_anthropic_prompt_caching"]["field_tab"] == "prompt_caching" assert fields["anthropic_prompt_caching_ttl"]["field_tab"] == "prompt_caching" assert fields["budget_exceeded_throttle_percentage"]["field_tab"] is None + + assert fields["openai_system_messages_first"]["field_type"] == "Boolean" + assert fields["openai_system_messages_first"]["field_value"] is False + assert fields["openai_system_messages_first"]["field_tab"] == "prompt_caching" finally: app.dependency_overrides.clear() @@ -10887,6 +10892,7 @@ def test_general_settings_ui_defaults_unchanged_for_existing_fields(): [ ("enable_anthropic_prompt_caching", True), ("anthropic_prompt_caching_ttl", "1h"), + ("openai_system_messages_first", True), ], ) def test_prompt_caching_settings_propagate_on_config_reload(monkeypatch, field_name, db_value): @@ -10945,6 +10951,8 @@ def test_get_config_list_marks_untouched_prompt_caching_flag_as_not_set(monkeypa ("enable_anthropic_prompt_caching", False), ("anthropic_prompt_caching_ttl", "5m"), ("anthropic_prompt_caching_ttl", "1h"), + ("openai_system_messages_first", True), + ("openai_system_messages_first", False), ], ) @pytest.mark.asyncio @@ -10993,6 +11001,8 @@ async def test_update_config_field_prompt_caching_persists_to_litellm_settings(m ("anthropic_prompt_caching_ttl", "10m"), ("anthropic_prompt_caching_ttl", "1H"), ("anthropic_prompt_caching_ttl", 3600), + ("openai_system_messages_first", "yes"), + ("openai_system_messages_first", 1), ], ) @pytest.mark.asyncio @@ -11032,6 +11042,7 @@ async def test_update_config_field_prompt_caching_rejects_invalid(monkeypatch, f [ ("enable_anthropic_prompt_caching", False), ("anthropic_prompt_caching_ttl", None), + ("openai_system_messages_first", False), ("budget_exceeded_throttle_percentage", None), ], ) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.integration.test.tsx index 984996351df..e79d382ae39 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.integration.test.tsx @@ -35,10 +35,12 @@ beforeEach(() => { Element.prototype.scrollIntoView = () => {}; }); -const CHAT_REQUEST_ARG_COUNT = 26; +const CHAT_REQUEST_ARG_COUNT = 27; const STREAMING_ENABLED_ARG_INDEX = 25; -const MESSAGES_REQUEST_ARG_COUNT = 19; +const CHAT_CUSTOM_HEADERS_ARG_INDEX = 26; +const MESSAGES_REQUEST_ARG_COUNT = 20; const MESSAGES_STREAMING_ENABLED_ARG_INDEX = 18; +const MESSAGES_CUSTOM_HEADERS_ARG_INDEX = 19; async function openComboboxByPlaceholder(placeholder: string) { const user = userEvent.setup(); @@ -447,6 +449,63 @@ describe("ChatUI", () => { expect(requestArgs[MESSAGES_STREAMING_ENABLED_ARG_INDEX]).toBe(false); }); + it("should send custom headers entered in the sidebar with /v1/chat/completions and /v1/messages requests", async () => { + const user = userEvent.setup(); + render( + , + ); + + await waitFor(() => { + expect(screen.getByText("Test Key")).toBeInTheDocument(); + }); + + await selectComboboxOption("Select a Model", "Model 1"); + await user.click(screen.getByRole("button", { name: "Add Header" })); + await user.click(screen.getByRole("button", { name: "Add Header" })); + const [firstName] = screen.getAllByPlaceholderText("Header Name"); + const [firstValue, secondValue] = screen.getAllByPlaceholderText("Header Value"); + fireEvent.change(firstName, { target: { value: "anthropic-beta" } }); + fireEvent.change(firstValue, { target: { value: "context-1m-2025-08-07" } }); + fireEvent.change(secondValue, { target: { value: "ignored because the name is blank" } }); + + const messageInput = screen.getByPlaceholderText("Type your message... (Shift+Enter for new line)"); + await act(async () => { + fireEvent.change(messageInput, { target: { value: "hello" } }); + }); + await act(async () => { + fireEvent.keyDown(messageInput, { key: "Enter", code: "Enter" }); + }); + + await waitFor(() => { + expect(makeOpenAIChatCompletionRequest).toHaveBeenCalledTimes(1); + }); + const chatArgs = vi.mocked(makeOpenAIChatCompletionRequest).mock.calls[0]; + expect(chatArgs).toHaveLength(CHAT_REQUEST_ARG_COUNT); + expect(chatArgs[CHAT_CUSTOM_HEADERS_ARG_INDEX]).toEqual({ "anthropic-beta": "context-1m-2025-08-07" }); + + await selectComboboxOption("Select an endpoint", "/v1/messages"); + await selectComboboxOption("Select a Model", "Model 1"); + await act(async () => { + fireEvent.change(messageInput, { target: { value: "hello again" } }); + }); + await act(async () => { + fireEvent.keyDown(messageInput, { key: "Enter", code: "Enter" }); + }); + + await waitFor(() => { + expect(makeAnthropicMessagesRequest).toHaveBeenCalledTimes(1); + }); + const messagesArgs = vi.mocked(makeAnthropicMessagesRequest).mock.calls[0]; + expect(messagesArgs).toHaveLength(MESSAGES_REQUEST_ARG_COUNT); + expect(messagesArgs[MESSAGES_CUSTOM_HEADERS_ARG_INDEX]).toEqual({ "anthropic-beta": "context-1m-2025-08-07" }); + }); + it("should force streaming in simplified mode even when the playground setting is off", async () => { sessionStorage.setItem("streamingEnabled", "false"); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx index ed8679cfdc1..ae0fabe5ef2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx @@ -9,6 +9,7 @@ import { Info, Key, Link2, + ListPlus, Loader2, Settings, Shield, @@ -40,6 +41,8 @@ import { makeAnthropicMessagesRequest } from "../../llm_calls/anthropic_messages import { makeOpenAIAudioSpeechRequest } from "../../llm_calls/audio_speech"; import { makeOpenAIAudioTranscriptionRequest } from "../../llm_calls/audio_transcriptions"; import { makeOpenAIChatCompletionRequest } from "@/components/llm_calls/chat_completion"; +import { customHeadersFromPairs, parseStoredHeaderPairs } from "@/components/llm_calls/request_headers"; +import KeyValueInput, { type KeyValuePair } from "@/components/key_value_input"; import { makeOpenAIEmbeddingsRequest } from "../../llm_calls/embeddings_api"; import { Agent, fetchAvailableAgents } from "../../llm_calls/fetch_agents"; import { fetchAvailableModels, ModelGroup } from "@/components/llm_calls/fetch_models"; @@ -220,6 +223,10 @@ const ChatUI: React.FC = ({ return []; } }); + const [customHeaderPairs, setCustomHeaderPairs] = useState(() => + parseStoredHeaderPairs(getSecureItem("customHeaders")), + ); + const customHeaders = useMemo(() => customHeadersFromPairs(customHeaderPairs), [customHeaderPairs]); const [selectedVoice, setSelectedVoice] = useState(() => { const saved = sessionStorage.getItem("selectedVoice"); if (!saved) return "alloy"; @@ -346,6 +353,7 @@ const ChatUI: React.FC = ({ selectedSdk, selectedVoice, proxySettings, + customHeaders, }); setGeneratedCode(code); } @@ -367,12 +375,14 @@ const ChatUI: React.FC = ({ endpointType, selectedModel, proxySettings, + customHeaders, ]); useEffect(() => { try { setSecureItem("apiKeySource", JSON.stringify(apiKeySource)); setSecureItem("apiKey", apiKey); + setSecureItem("customHeaders", JSON.stringify(customHeaderPairs)); } catch { // Storage full or unavailable — non-critical, skip persisting. } @@ -410,6 +420,7 @@ const ChatUI: React.FC = ({ mcpServerToolRestrictions, selectedVoice, streamingEnabled, + customHeaderPairs, ]); useEffect(() => { @@ -921,6 +932,7 @@ const ChatUI: React.FC = ({ mockTestFallbacks, mcpToolsets, streamingEnabled, + customHeaders, ); } else if (endpointType === EndpointType.IMAGE) { // For image generation @@ -932,6 +944,7 @@ const ChatUI: React.FC = ({ selectedTags, signal, customProxyBaseUrl || undefined, + customHeaders, ); } else if (endpointType === EndpointType.SPEECH) { // For audio speech @@ -946,6 +959,7 @@ const ChatUI: React.FC = ({ undefined, // responseFormat undefined, // speed customProxyBaseUrl || undefined, + customHeaders, ); } else if (endpointType === EndpointType.IMAGE_EDITS) { // For image edits @@ -959,6 +973,7 @@ const ChatUI: React.FC = ({ selectedTags, signal, customProxyBaseUrl || undefined, + customHeaders, ); } } else if (endpointType === EndpointType.RESPONSES) { @@ -1004,6 +1019,7 @@ const ChatUI: React.FC = ({ mcpToolsets, streamingEnabled, updateTotalLatency, + customHeaders, ); } else if (endpointType === EndpointType.ANTHROPIC_MESSAGES) { const apiChatHistory = [ @@ -1033,6 +1049,7 @@ const ChatUI: React.FC = ({ mcpServerToolRestrictions, mcpToolsets, streamingEnabled, + customHeaders, ); } else if (endpointType === EndpointType.EMBEDDINGS) { await makeOpenAIEmbeddingsRequest( @@ -1042,6 +1059,7 @@ const ChatUI: React.FC = ({ effectiveApiKey, selectedTags, customProxyBaseUrl || undefined, + customHeaders, ); } else if (endpointType === EndpointType.TRANSCRIPTION) { // For audio transcriptions @@ -1058,6 +1076,7 @@ const ChatUI: React.FC = ({ undefined, // responseFormat undefined, // temperature customProxyBaseUrl || undefined, + customHeaders, ); } } else if (endpointType === EndpointType.INTERACTIONS) { @@ -1069,6 +1088,8 @@ const ChatUI: React.FC = ({ selectedTags, signal, customProxyBaseUrl || undefined, + undefined, + customHeaders, ); } } @@ -1086,13 +1107,10 @@ const ChatUI: React.FC = ({ resolvedServerId = toolEntry?.server_id ?? rawSelected; } if (resolvedServerId && !resolvedServerId.startsWith("toolset:") && selectedMCPDirectTool) { - const result = await callMCPTool( - effectiveApiKey, - resolvedServerId, - selectedMCPDirectTool, - mcpToolArguments, - selectedGuardrails.length > 0 ? { guardrails: selectedGuardrails } : undefined, - ); + const result = await callMCPTool(effectiveApiKey, resolvedServerId, selectedMCPDirectTool, mcpToolArguments, { + ...(selectedGuardrails.length > 0 ? { guardrails: selectedGuardrails } : {}), + customHeaders, + }); const resultText = result?.content?.length > 0 ? JSON.stringify( @@ -1118,6 +1136,7 @@ const ChatUI: React.FC = ({ updateA2AMetadata, customProxyBaseUrl || undefined, selectedGuardrails.length > 0 ? selectedGuardrails : undefined, + customHeaders, ); } } catch (error) { @@ -1485,6 +1504,18 @@ const ChatUI: React.FC = ({ /> + {endpointType !== EndpointType.REALTIME && ( +
+ + +

+ Sent with every playground request, e.g. provider-specific headers like anthropic-beta. +

+
+ )} +
)} + + {systemFirstSetting && ( +
+
+

System messages first for OpenAI

+

{systemFirstSetting.field_description}

+
+ persist(OPENAI_SYSTEM_MESSAGES_FIRST, checked)} + /> +
+ )} ); diff --git a/ui/litellm-dashboard/src/components/chat_ui/CodeSnippets.test.tsx b/ui/litellm-dashboard/src/components/chat_ui/CodeSnippets.test.tsx index 0113f7b2832..65f5ade7a5c 100644 --- a/ui/litellm-dashboard/src/components/chat_ui/CodeSnippets.test.tsx +++ b/ui/litellm-dashboard/src/components/chat_ui/CodeSnippets.test.tsx @@ -48,6 +48,28 @@ describe("CodeSnippets", () => { expect(code).toContain("print(response.data[0].embedding)"); }); + describe("custom headers", () => { + const customHeaders = { "anthropic-beta": "context-1m-2025-08-07", "x-request-source": "playground" }; + + it("passes configured headers as default_headers on the OpenAI client", () => { + const code = generateCodeSnippet({ ...baseParams, endpointType: EndpointType.CHAT, customHeaders }); + expect(code).toContain('base_url="http://localhost:4000",\n\tdefault_headers={'); + expect(code).toContain('"anthropic-beta": "context-1m-2025-08-07"'); + expect(code).toContain('"x-request-source": "playground"'); + }); + + it("passes configured headers as default_headers on the Azure client", () => { + const code = generateCodeSnippet({ ...baseParams, selectedSdk: "azure", customHeaders }); + expect(code).toContain('api_version="2024-02-01",\n\tdefault_headers={'); + expect(code).toContain('"anthropic-beta": "context-1m-2025-08-07"'); + }); + + it("omits default_headers when no custom headers are configured", () => { + expect(generateCodeSnippet(baseParams)).not.toContain("default_headers"); + expect(generateCodeSnippet({ ...baseParams, customHeaders: {} })).not.toContain("default_headers"); + }); + }); + describe("base URL selection", () => { it("should use LITELLM_UI_API_DOC_BASE_URL when provided", () => { const customBaseUrl = "https://custom-doc.example.com"; diff --git a/ui/litellm-dashboard/src/components/chat_ui/CodeSnippets.tsx b/ui/litellm-dashboard/src/components/chat_ui/CodeSnippets.tsx index f458e563b4d..9150ba0e632 100644 --- a/ui/litellm-dashboard/src/components/chat_ui/CodeSnippets.tsx +++ b/ui/litellm-dashboard/src/components/chat_ui/CodeSnippets.tsx @@ -1,6 +1,7 @@ import { MessageType } from "./types"; import { EndpointType } from "./mode_endpoint_mapping"; import { MCPServer } from "@/components/mcp_tools/types"; +import type { CustomHeaders } from "@/components/llm_calls/request_headers"; interface CodeGenMetadata { tags?: string[]; @@ -30,6 +31,7 @@ interface GenerateCodeParams { PROXY_BASE_URL?: string; LITELLM_UI_API_DOC_BASE_URL?: string | null; }; + customHeaders?: CustomHeaders; } export const generateCodeSnippet = (params: GenerateCodeParams): string => { @@ -48,6 +50,7 @@ export const generateCodeSnippet = (params: GenerateCodeParams): string => { selectedModel, selectedSdk, proxySettings, + customHeaders, } = params; const effectiveApiKey = apiKeySource === "session" ? accessToken : apiKey; @@ -76,6 +79,11 @@ export const generateCodeSnippet = (params: GenerateCodeParams): string => { const modelNameForCode = selectedModel || "your-model-name"; + const defaultHeadersCode = + customHeaders && Object.keys(customHeaders).length > 0 + ? `,\n\tdefault_headers=${JSON.stringify(customHeaders, null, 2).replace(/\n/g, "\n\t")}` + : ""; + const clientInitialization = selectedSdk === "azure" ? `import openai @@ -83,13 +91,13 @@ export const generateCodeSnippet = (params: GenerateCodeParams): string => { client = openai.AzureOpenAI( api_key="${effectiveApiKey || "YOUR_LITELLM_API_KEY"}", azure_endpoint="${apiBase}", - api_version="2024-02-01" + api_version="2024-02-01"${defaultHeadersCode} )` : `import openai client = openai.OpenAI( api_key="${effectiveApiKey || "YOUR_LITELLM_API_KEY"}", - base_url="${apiBase}" + base_url="${apiBase}"${defaultHeadersCode} )`; let endpointSpecificCode; diff --git a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.test.tsx b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.test.tsx index ec477441586..f10502df77b 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.test.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.test.tsx @@ -1,4 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import openai from "openai"; import { makeOpenAIChatCompletionRequest } from "./chat_completion"; import type { TokenUsage } from "../chat_ui/ResponseMetrics"; @@ -615,3 +616,47 @@ describe("chat_completion response cache", () => { expect(onUsageData).toHaveBeenCalledWith(expect.not.objectContaining({ servedFromResponseCache: true })); }); }); + +describe("chat_completion custom headers", () => { + afterEach(() => { + vi.clearAllMocks(); + }); + + it("sends custom headers alongside the tags header on the OpenAI client", async () => { + mockCreate.mockReturnValueOnce(nonStreamingResponse({ choices: [{ message: { content: "Hi" } }] })); + + await makeOpenAIChatCompletionRequest( + [{ role: "user", content: "Hello" }], + vi.fn(), + "gpt-4", + "test-token", + ["team-a"], + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + false, + { "anthropic-beta": "context-1m-2025-08-07", "x-litellm-tags": "overridden" }, + ); + + expect(vi.mocked(openai.OpenAI).mock.calls[0][0]).toMatchObject({ + defaultHeaders: { "anthropic-beta": "context-1m-2025-08-07", "x-litellm-tags": "overridden" }, + }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx index ffa2877fbd9..fe5b6fb6e39 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx @@ -6,6 +6,7 @@ import { getProxyBaseUrl } from "@/components/networking"; import { MCPServer, MCPToolset, type MCPEvent } from "@/components/mcp_tools/types"; import { extractPromptCacheTokens } from "@/utils/promptCacheUsage"; import { parseUsageCost } from "./usage_cost"; +import { buildPlaygroundHeaders, type CustomHeaders } from "./request_headers"; const completionAsSingleChunk = (completion: ChatCompletion): ChatCompletionChunk => ({ @@ -50,6 +51,7 @@ export async function makeOpenAIChatCompletionRequest( mockTestFallbacks?: boolean, mcpToolsets?: MCPToolset[], streamingEnabled: boolean = true, + customHeaders?: CustomHeaders, ) { // base url should be the current base_url const isLocal = process.env.NODE_ENV === "development"; @@ -57,11 +59,7 @@ export async function makeOpenAIChatCompletionRequest( console.log = function () {}; } const proxyBaseUrl = customBaseUrl || getProxyBaseUrl(); - // Prepare headers with tags and trace ID - const headers: Record = {}; - if (tags && tags.length > 0) { - headers["x-litellm-tags"] = tags.join(","); - } + const headers = buildPlaygroundHeaders(tags, customHeaders); const client = new openai.OpenAI({ apiKey: accessToken, diff --git a/ui/litellm-dashboard/src/components/llm_calls/request_headers.test.ts b/ui/litellm-dashboard/src/components/llm_calls/request_headers.test.ts new file mode 100644 index 00000000000..cacedea3c87 --- /dev/null +++ b/ui/litellm-dashboard/src/components/llm_calls/request_headers.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from "vitest"; +import { + buildPlaygroundHeaders, + customHeadersFromPairs, + parseStoredHeaderPairs, + withRequiredHeaders, +} from "./request_headers"; + +describe("customHeadersFromPairs", () => { + it("trims header names and drops rows without a name", () => { + expect( + customHeadersFromPairs([ + [" anthropic-beta ", "context-1m-2025-08-07"], + ["", "orphan value"], + [" ", "whitespace name"], + ["x-empty", ""], + ]), + ).toEqual({ "anthropic-beta": "context-1m-2025-08-07", "x-empty": "" }); + }); +}); + +describe("parseStoredHeaderPairs", () => { + it("round-trips pairs persisted as JSON", () => { + const pairs = [["anthropic-beta", "context-1m-2025-08-07"]] as const; + expect(parseStoredHeaderPairs(JSON.stringify(pairs))).toEqual(pairs); + }); + + it("returns no pairs for missing, malformed, or wrongly shaped storage", () => { + expect(parseStoredHeaderPairs(null)).toEqual([]); + expect(parseStoredHeaderPairs("not json")).toEqual([]); + expect(parseStoredHeaderPairs(JSON.stringify({ "anthropic-beta": "x" }))).toEqual([]); + expect(parseStoredHeaderPairs(JSON.stringify([["ok", "pair"], ["one"], [1, 2], "str"]))).toEqual([["ok", "pair"]]); + }); +}); + +describe("buildPlaygroundHeaders", () => { + it("joins tags into x-litellm-tags and lets custom headers override it", () => { + expect(buildPlaygroundHeaders(["a", "b"], { "x-custom": "1" })).toEqual({ + "x-litellm-tags": "a,b", + "x-custom": "1", + }); + expect(buildPlaygroundHeaders(["a"], { "x-litellm-tags": "b" })).toEqual({ "x-litellm-tags": "b" }); + }); + + it("omits x-litellm-tags when there are no tags", () => { + expect(buildPlaygroundHeaders([], { "x-custom": "1" })).toEqual({ "x-custom": "1" }); + expect(buildPlaygroundHeaders(undefined, undefined)).toEqual({}); + }); +}); + +describe("withRequiredHeaders", () => { + it("keeps required headers regardless of custom header name casing", () => { + expect( + withRequiredHeaders( + { authorization: "Bearer stolen", "content-type": "text/plain", "x-custom": "1" }, + { Authorization: "Bearer real", "Content-Type": "application/json" }, + ), + ).toEqual({ Authorization: "Bearer real", "Content-Type": "application/json", "x-custom": "1" }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/llm_calls/request_headers.ts b/ui/litellm-dashboard/src/components/llm_calls/request_headers.ts new file mode 100644 index 00000000000..8c0c0056fca --- /dev/null +++ b/ui/litellm-dashboard/src/components/llm_calls/request_headers.ts @@ -0,0 +1,38 @@ +import type { KeyValuePair } from "@/components/key_value_input"; + +export type CustomHeaders = Readonly>; + +export const customHeadersFromPairs = (pairs: readonly KeyValuePair[]): CustomHeaders => + Object.fromEntries(pairs.map(([name, value]) => [name.trim(), value]).filter(([name]) => name !== "")); + +const isHeaderPair = (entry: unknown): entry is KeyValuePair => + Array.isArray(entry) && entry.length === 2 && entry.every((part) => typeof part === "string"); + +export const parseStoredHeaderPairs = (raw: string | null): readonly KeyValuePair[] => { + if (!raw) return []; + try { + const parsed: unknown = JSON.parse(raw); + return Array.isArray(parsed) ? parsed.filter(isHeaderPair) : []; + } catch { + return []; + } +}; + +export const buildPlaygroundHeaders = ( + tags?: readonly string[], + customHeaders?: CustomHeaders, +): Record => ({ + ...(tags && tags.length > 0 ? { "x-litellm-tags": tags.join(",") } : {}), + ...customHeaders, +}); + +export const withRequiredHeaders = ( + headers: Readonly>, + required: Readonly>, +): Record => { + const reserved = new Set(Object.keys(required).map((name) => name.toLowerCase())); + return { + ...Object.fromEntries(Object.entries(headers).filter(([name]) => !reserved.has(name.toLowerCase()))), + ...required, + }; +}; diff --git a/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx b/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx index 290c8b0b619..a2260d25ca6 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx @@ -1,4 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import openai from "openai"; import { makeOpenAIResponsesRequest } from "./responses_api"; import { MessageType } from "../chat_ui/types"; import type { TokenUsage } from "../chat_ui/ResponseMetrics"; @@ -611,3 +612,46 @@ describe("responses_api response cache", () => { expect(onUsageData).toHaveBeenCalledWith(expect.not.objectContaining({ servedFromResponseCache: true }), ""); }); }); + +describe("responses_api custom headers", () => { + afterEach(() => { + vi.clearAllMocks(); + }); + + it("sends custom headers alongside the tags header on the OpenAI client", async () => { + mockResponsesCreate.mockReturnValueOnce(nonStreamingResponse({ id: "resp_1", output: [] })); + + await makeOpenAIResponsesRequest( + [{ role: "user", content: "Hello" }], + vi.fn(), + "gpt-4", + "test-token", + ["team-a"], + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + false, + undefined, + { "anthropic-beta": "context-1m-2025-08-07" }, + ); + + expect(vi.mocked(openai.OpenAI).mock.calls[0][0]).toMatchObject({ + defaultHeaders: { "x-litellm-tags": "team-a", "anthropic-beta": "context-1m-2025-08-07" }, + }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx index ce54c7c6b40..ec196a8d9a2 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx @@ -5,6 +5,7 @@ import { getProxyBaseUrl } from "@/components/networking"; import { toast } from "@/lib/toast"; import { extractPromptCacheTokens } from "@/utils/promptCacheUsage"; import { parseUsageCost } from "./usage_cost"; +import { buildPlaygroundHeaders, type CustomHeaders } from "./request_headers"; import type { MCPEvent } from "@/components/mcp_tools/types"; import { MCPServer, MCPToolset } from "@/components/mcp_tools/types"; import { @@ -85,6 +86,7 @@ export async function makeOpenAIResponsesRequest( mcpToolsets?: MCPToolset[], streamingEnabled: boolean = true, onTotalLatency?: (latency: number) => void, + customHeaders?: CustomHeaders, ) { if (!accessToken) { throw new Error("Virtual Key is required"); @@ -101,11 +103,7 @@ export async function makeOpenAIResponsesRequest( } const proxyBaseUrl = customBaseUrl || getProxyBaseUrl(); - // Prepare headers with tags and trace ID - const headers: Record = {}; - if (tags && tags.length > 0) { - headers["x-litellm-tags"] = tags.join(","); - } + const headers = buildPlaygroundHeaders(tags, customHeaders); const client = new openai.OpenAI({ apiKey: accessToken,