feat(openai): add openai_system_messages_first to put system messages first for prompt caching

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

litellm/__init__.py

@@ -343,6 +343,7 @@ _anthropic_prompt_caching_ttl_env: Optional[str] = os.getenv("LITELLM_ANTHROPIC_
 anthropic_prompt_caching_ttl: Optional[Literal["5m", "1h"]] = (
     "1h" if _anthropic_prompt_caching_ttl_env == "1h" else "5m" if _anthropic_prompt_caching_ttl_env == "5m" else None
 )
+openai_system_messages_first: bool = os.getenv("LITELLM_OPENAI_SYSTEM_MESSAGES_FIRST", "false").lower() == "true"
 disable_vertex_batch_output_transformation: bool = False
 extra_spend_tag_headers: Optional[List[str]] = None
 in_memory_llm_clients_cache: "LLMClientCache"

litellm/constants.py

@@ -1776,6 +1776,7 @@ DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD = float(os.getenv("DEFAULT_PROMPT_
 LENGTH_OF_LITELLM_GENERATED_KEY: Final = int(os.getenv("LENGTH_OF_LITELLM_GENERATED_KEY", 16))
 MINIMUM_CUSTOM_KEY_LENGTH: Final = int(os.getenv("MINIMUM_CUSTOM_KEY_LENGTH", 16))
 SECRET_MANAGER_REFRESH_INTERVAL: Final = int(os.getenv("SECRET_MANAGER_REFRESH_INTERVAL", 86400))
+OPENAI_SYSTEM_MESSAGES_FIRST_PROVIDERS: Final = frozenset({"openai", "azure"})
 LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [
     "default_internal_user_params",
     "default_team_params",
@@ -1793,6 +1794,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [
     # test_general_settings_ui_fields_are_db_overridable enforces that pairing.
     "enable_anthropic_prompt_caching",
     "anthropic_prompt_caching_ttl",
+    "openai_system_messages_first",
     "max_ui_session_budget",
     "budget_rollover",
     "mcp_tool_search",

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -2256,6 +2256,22 @@ def drop_tool_reference_parts_from_tool_messages(
     return [_drop_tool_reference_parts(message) for message in messages]  # mutable-ok: pipelines mutate message lists
 
 
+INSTRUCTION_MESSAGE_ROLES: Final = frozenset({"system", "developer"})
+
+
+def _is_instruction_message(message: AllMessageValues) -> bool:
+    return message.get("role") in INSTRUCTION_MESSAGE_ROLES
+
+
+def system_messages_first(
+    messages: list[AllMessageValues],  # mutable-ok: message pipelines type messages as mutable lists
+) -> list[AllMessageValues]:  # mutable-ok: message pipelines type messages as mutable lists
+    return [  # mutable-ok: pipelines mutate message lists
+        *(message for message in messages if _is_instruction_message(message)),
+        *(message for message in messages if not _is_instruction_message(message)),
+    ]
+
+
 def _attempt_json_repair(s: str) -> object | None:
     """
     Attempt to repair truncated JSON produced by LLM tool calls.

litellm/llms/azure/chat/gpt_transformation.py

@@ -9,6 +9,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
     drop_tool_reference_parts_from_tool_messages,
     flatten_combinators_and_drop_non_python_regex_patterns,
     hoist_images_from_tool_messages,
+    system_messages_first,
     tool_with_sanitized_parameters,
 )
 from litellm.litellm_core_utils.prompt_templates.factory import (
@@ -276,7 +277,8 @@ class AzureOpenAIConfig(BaseConfig):
         litellm_params: dict,
         headers: dict,
     ) -> dict:
-        stripped_messages: Final = drop_tool_reference_parts_from_tool_messages(messages)
+        ordered_messages: Final = system_messages_first(messages) if litellm.openai_system_messages_first else messages
+        stripped_messages: Final = drop_tool_reference_parts_from_tool_messages(ordered_messages)
         azure_messages: Final = convert_to_azure_openai_messages(hoist_images_from_tool_messages(stripped_messages))
         return {
             "model": model,

litellm/llms/openai/chat/gpt_transformation.py

@@ -12,6 +12,7 @@ from urllib.parse import urlparse
 import httpx
 
 import litellm
+from litellm.constants import OPENAI_SYSTEM_MESSAGES_FIRST_PROVIDERS
 from litellm.litellm_core_utils.core_helpers import map_finish_reason
 from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
     _extract_reasoning_content,
@@ -24,6 +25,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
     flatten_combinators_and_drop_non_python_regex_patterns,
     get_tool_call_names,
     hoist_images_from_tool_messages,
+    system_messages_first,
     tool_with_sanitized_parameters,
 )
 from litellm.litellm_core_utils.prompt_templates.image_handling import (
@@ -463,6 +465,15 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
         ]
         return MappingProxyType({"tools": sanitized})
 
+    def _prompt_cache_ordered_messages(
+        self, messages: list[AllMessageValues], litellm_params: Mapping[str, object]
+    ) -> list[AllMessageValues]:
+        if not litellm.openai_system_messages_first:
+            return messages
+        if litellm_params.get("custom_llm_provider") not in OPENAI_SYSTEM_MESSAGES_FIRST_PROVIDERS:
+            return messages
+        return system_messages_first(messages)
+
     def transform_request(
         self,
         model: str,
@@ -477,7 +488,9 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
         Returns:
             dict: The transformed request. Sent as the body of the API call.
         """
-        messages = self._transform_messages(messages=messages, model=model)
+        messages = self._transform_messages(
+            messages=self._prompt_cache_ordered_messages(messages, litellm_params), model=model
+        )
         if not self._should_preserve_cache_control_for_endpoint(
             litellm_params.get("custom_llm_provider"), litellm_params.get("api_base")
         ):
@@ -506,7 +519,9 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
         litellm_params: dict,
         headers: dict,
     ) -> dict:
-        transformed_messages = await self._transform_messages(messages=messages, model=model, is_async=True)
+        transformed_messages = await self._transform_messages(
+            messages=self._prompt_cache_ordered_messages(messages, litellm_params), model=model, is_async=True
+        )
         if not self._should_preserve_cache_control_for_endpoint(
             litellm_params.get("custom_llm_provider"), litellm_params.get("api_base")
         ):

litellm/proxy/proxy_server.py

@@ -17465,6 +17465,16 @@ _GENERAL_SETTINGS_UI_LITELLM_FIELDS: Final[dict[str, GeneralSettingsUILiteLLMFie
         "tab": "prompt_caching",
         "description": "Empty uses Anthropic's 5m default. 1h suits long sessions but doubles the cache write cost.",
     },
+    "openai_system_messages_first": {
+        "type": "Boolean",
+        "tab": "prompt_caching",
+        "description": (
+            "Moves system and developer messages to the front of the messages array on OpenAI and "
+            "Azure OpenAI chat completions requests, keeping their relative order. OpenAI's prompt cache "
+            "matches on the exact prefix, so a system message that arrives mid-conversation otherwise "
+            "breaks the cached prefix on every turn."
+        ),
+    },
     "budget_rollover": {  # mutable-ok: registry literal, frozen with its siblings below
         "type": "Boolean",
         "description": (

tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py

@@ -23,6 +23,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
     responses_reasoning_items_from_thinking_blocks,
     split_concatenated_json_objects,
     strip_encrypted_reasoning_from_messages,
+    system_messages_first,
     update_messages_with_model_file_ids,
 )
 
@@ -1107,6 +1108,38 @@ def test_drop_tool_reference_parts_leaves_non_tool_messages_alone():
     assert result[2]["content"] == ""
 
 
+class TestSystemMessagesFirst:
+    def test_stable_partition_keeps_order_within_each_group(self):
+        messages = [
+            {"role": "user", "content": "u1"},
+            {"role": "system", "content": "s1"},
+            {"role": "assistant", "content": "a1"},
+            {"role": "developer", "content": "d1"},
+            {"role": "tool", "tool_call_id": "c1", "content": "t1"},
+            {"role": "system", "content": "s2"},
+        ]
+
+        result = system_messages_first(messages)
+
+        assert [m["content"] for m in result] == ["s1", "d1", "s2", "u1", "a1", "t1"]
+        assert [m["content"] for m in messages] == ["u1", "s1", "a1", "d1", "t1", "s2"]
+        assert all(
+            result_message is original for result_message, original in zip(result[3:], messages[::2], strict=True)
+        )
+
+    @pytest.mark.parametrize(
+        "messages",
+        [
+            [],
+            [{"role": "user", "content": "u1"}, {"role": "assistant", "content": "a1"}],
+            [{"role": "system", "content": "s1"}, {"role": "user", "content": "u1"}],
+            [{"role": "system", "content": "s1"}, {"role": "system", "content": "s2"}],
+        ],
+    )
+    def test_already_ordered_messages_come_back_unchanged(self, messages):
+        assert system_messages_first(messages) == messages
+
+
 class TestFlattenTopLevelSchemaCombinators:
     def _customer_anyof_schema(self):
         return {

tests/test_litellm/llms/azure/chat/test_azure_chat_gpt_transformation.py

@@ -132,6 +132,32 @@ def test_transform_request_drops_tool_reference_parts():
     assert request["messages"][2]["content"] == ""
 
 
+@pytest.mark.parametrize(
+    "enabled, expected", [(False, ("hi", "sys", "reply", "more")), (True, ("sys", "hi", "reply", "more"))]
+)
+def test_transform_request_system_messages_first_follows_global_flag(monkeypatch, enabled, expected):
+    """Azure OpenAI shares OpenAI's prefix-matched prompt cache, so the same flag moves
+    system messages ahead of the conversation on the Azure request body."""
+    monkeypatch.setattr(litellm, "openai_system_messages_first", enabled)
+    messages = [
+        {"role": "user", "content": "hi"},
+        {"role": "system", "content": "sys"},
+        {"role": "assistant", "content": "reply"},
+        {"role": "user", "content": "more"},
+    ]
+
+    request = AzureOpenAIConfig().transform_request(
+        model="gpt-4o",
+        messages=messages,
+        optional_params={},
+        litellm_params={"custom_llm_provider": "azure"},
+        headers={},
+    )
+
+    assert tuple(m["content"] for m in request["messages"]) == expected
+    assert [m["content"] for m in messages] == ["hi", "sys", "reply", "more"]
+
+
 @pytest.mark.parametrize(
     "model, emitted_key, absent_key",
     [

tests/test_litellm/llms/azure/chat/test_azure_chat_o_series_transformation.py

@@ -68,3 +68,24 @@ def test_azure_o_series_transform_request_flattens_top_level_anyof():
     assert parameters["required"] == ["id"]
     assert "anyOf" in tool["function"]["parameters"]
     assert optional_params["tools"][0] is tool
+
+
+def test_azure_o_series_transform_request_moves_system_messages_first(monkeypatch):
+    monkeypatch.setattr(litellm, "openai_system_messages_first", True)
+    messages = [
+        {"role": "user", "content": "hi"},
+        {"role": "developer", "content": "dev"},
+        {"role": "assistant", "content": "reply"},
+        {"role": "user", "content": "more"},
+    ]
+
+    request = AzureOpenAIO1Config().transform_request(
+        model="o3-mini",
+        messages=messages,
+        optional_params={},
+        litellm_params={"custom_llm_provider": "azure"},
+        headers={},
+    )
+
+    assert [m["content"] for m in request["messages"]] == ["dev", "hi", "reply", "more"]
+    assert [m["content"] for m in messages] == ["hi", "dev", "reply", "more"]

tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py

@@ -1124,6 +1124,80 @@ class TestToolReferenceStripping:
         assert request["messages"][2]["content"] == ""
 
 
+class TestSystemMessagesFirst:
+    """With litellm.openai_system_messages_first on, requests bound for OpenAI put system and
+    developer messages ahead of the conversation, keeping each group's order, so the instruction
+    prefix stays byte-stable for OpenAI's prefix-matched prompt cache."""
+
+    MESSAGES: Final = (
+        {"role": "user", "content": "first turn"},
+        {"role": "system", "content": "sys 1"},
+        {"role": "assistant", "content": "reply"},
+        {"role": "developer", "content": "dev"},
+        {"role": "user", "content": "second turn"},
+        {"role": "system", "content": "sys 2"},
+    )
+    ORIGINAL_ORDER: Final = ("first turn", "sys 1", "reply", "dev", "second turn", "sys 2")
+    ORDERED: Final = ("sys 1", "dev", "sys 2", "first turn", "reply", "second turn")
+
+    def setup_method(self):
+        self.config = OpenAIGPTConfig()
+
+    def _messages(self):
+        return [dict(m) for m in self.MESSAGES]
+
+    def _transform(self, provider):
+        return self.config.transform_request(
+            model="gpt-4.1",
+            messages=self._messages(),
+            optional_params={},
+            litellm_params={"custom_llm_provider": provider},
+            headers={},
+        )
+
+    def test_default_off_keeps_caller_order(self, monkeypatch):
+        monkeypatch.setattr(litellm, "openai_system_messages_first", False)
+        assert tuple(m["content"] for m in self._transform("openai")["messages"]) == self.ORIGINAL_ORDER
+
+    def test_moves_system_and_developer_messages_first_for_openai(self, monkeypatch):
+        monkeypatch.setattr(litellm, "openai_system_messages_first", True)
+        assert tuple(m["content"] for m in self._transform("openai")["messages"]) == self.ORDERED
+
+    def test_leaves_openai_compatible_providers_alone(self, monkeypatch):
+        monkeypatch.setattr(litellm, "openai_system_messages_first", True)
+        assert tuple(m["content"] for m in self._transform("deepseek")["messages"]) == self.ORIGINAL_ORDER
+
+    def test_does_not_mutate_caller_messages(self, monkeypatch):
+        monkeypatch.setattr(litellm, "openai_system_messages_first", True)
+        messages = self._messages()
+        self.config.transform_request(
+            model="gpt-4.1",
+            messages=messages,
+            optional_params={},
+            litellm_params={"custom_llm_provider": "openai"},
+            headers={},
+        )
+        assert tuple(m["content"] for m in messages) == self.ORIGINAL_ORDER
+
+    @pytest.mark.asyncio
+    async def test_async_transform_request_moves_system_messages_first(self, monkeypatch):
+        class UninstantiatedOpenAIGPTConfig(OpenAIGPTConfig):
+            _is_base_class = True
+
+            def __init__(self) -> None:
+                pass
+
+        monkeypatch.setattr(litellm, "openai_system_messages_first", True)
+        request = await UninstantiatedOpenAIGPTConfig().async_transform_request(
+            model="gpt-4.1",
+            messages=self._messages(),
+            optional_params={},
+            litellm_params={"custom_llm_provider": "openai"},
+            headers={},
+        )
+        assert tuple(m["content"] for m in request["messages"]) == self.ORDERED
+
+
 class TestOpenAIPromptCacheBreakpointChatPath:
     """Chat-path shape for OpenAI explicit prompt caching (#37509)."""
 

tests/test_litellm/proxy/test_proxy_server.py

@@ -10750,6 +10750,7 @@ def test_get_config_list_includes_anthropic_prompt_caching_fields(monkeypatch):
     monkeypatch.setattr(ps, "prisma_client", mock_prisma)
     monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True)
     monkeypatch.setattr(litellm, "anthropic_prompt_caching_ttl", "1h")
+    monkeypatch.setattr(litellm, "openai_system_messages_first", False)
     app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth(
         user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN
     )
@@ -10771,6 +10772,10 @@ def test_get_config_list_includes_anthropic_prompt_caching_fields(monkeypatch):
         assert fields["enable_anthropic_prompt_caching"]["field_tab"] == "prompt_caching"
         assert fields["anthropic_prompt_caching_ttl"]["field_tab"] == "prompt_caching"
         assert fields["budget_exceeded_throttle_percentage"]["field_tab"] is None
+
+        assert fields["openai_system_messages_first"]["field_type"] == "Boolean"
+        assert fields["openai_system_messages_first"]["field_value"] is False
+        assert fields["openai_system_messages_first"]["field_tab"] == "prompt_caching"
     finally:
         app.dependency_overrides.clear()
 
@@ -10887,6 +10892,7 @@ def test_general_settings_ui_defaults_unchanged_for_existing_fields():
     [
         ("enable_anthropic_prompt_caching", True),
         ("anthropic_prompt_caching_ttl", "1h"),
+        ("openai_system_messages_first", True),
     ],
 )
 def test_prompt_caching_settings_propagate_on_config_reload(monkeypatch, field_name, db_value):
@@ -10945,6 +10951,8 @@ def test_get_config_list_marks_untouched_prompt_caching_flag_as_not_set(monkeypa
         ("enable_anthropic_prompt_caching", False),
         ("anthropic_prompt_caching_ttl", "5m"),
         ("anthropic_prompt_caching_ttl", "1h"),
+        ("openai_system_messages_first", True),
+        ("openai_system_messages_first", False),
     ],
 )
 @pytest.mark.asyncio
@@ -10993,6 +11001,8 @@ async def test_update_config_field_prompt_caching_persists_to_litellm_settings(m
         ("anthropic_prompt_caching_ttl", "10m"),
         ("anthropic_prompt_caching_ttl", "1H"),
         ("anthropic_prompt_caching_ttl", 3600),
+        ("openai_system_messages_first", "yes"),
+        ("openai_system_messages_first", 1),
     ],
 )
 @pytest.mark.asyncio
@@ -11032,6 +11042,7 @@ async def test_update_config_field_prompt_caching_rejects_invalid(monkeypatch, f
     [
         ("enable_anthropic_prompt_caching", False),
         ("anthropic_prompt_caching_ttl", None),
+        ("openai_system_messages_first", False),
         ("budget_exceeded_throttle_percentage", None),
     ],
 )

ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.integration.test.tsx

@@ -45,6 +45,15 @@ const SETTINGS_FIXTURE = [
     field_tab: "prompt_caching",
     field_default_value: null,
   },
+  {
+    field_name: "openai_system_messages_first",
+    field_type: "Boolean",
+    field_value: false,
+    field_description: "openai system first toggle",
+    stored_in_db: null,
+    field_tab: "prompt_caching",
+    field_default_value: false,
+  },
   {
     field_name: "max_ui_session_budget",
     field_type: "Dollar",
@@ -101,6 +110,39 @@ describe("GeneralSettings General tab", () => {
   });
 });
 
+describe("GeneralSettings Prompt Caching tab", () => {
+  beforeEach(() => {
+    vi.mocked(getGeneralSettingsCall).mockResolvedValue([...SETTINGS_FIXTURE.map((s) => ({ ...s }))]);
+    vi.mocked(updateConfigFieldSetting).mockClear();
+    vi.mocked(deleteConfigFieldSetting).mockClear();
+  });
+
+  it("persists openai_system_messages_first when its switch is turned on", async () => {
+    const user = userEvent.setup();
+    renderWithProviders(<GeneralSettings accessToken="token" userRole="Admin" userID="user" />);
+
+    await user.click(await screen.findByRole("tab", { name: "Prompt Caching" }));
+    const toggle = await screen.findByRole("switch", { name: "System messages first for OpenAI" });
+    expect(toggle).not.toBeChecked();
+
+    await user.click(toggle);
+
+    expect(toggle).toBeChecked();
+    expect(updateConfigFieldSetting).toHaveBeenCalledWith("token", "openai_system_messages_first", true);
+    expect(deleteConfigFieldSetting).not.toHaveBeenCalled();
+  });
+
+  it("keeps the prompt caching rows off the General tab table", async () => {
+    const user = userEvent.setup();
+    renderWithProviders(<GeneralSettings accessToken="token" userRole="Admin" userID="user" />);
+
+    await user.click(screen.getByText("General"));
+    await settingsRow("max_ui_session_budget");
+
+    expect(screen.queryByText("openai_system_messages_first")).not.toBeInTheDocument();
+  });
+});
+
 // The five tabs here are proxy-wide settings. Auto-routers moved to Models + Endpoints.
 describe("GeneralSettings tabs", () => {
   beforeEach(() => {

ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx

@@ -18,6 +18,9 @@ import RoutingGroups from "@/components/routing_groups";
 const PROMPT_CACHING_TAB = "prompt_caching";
 const ENABLE_ANTHROPIC_PROMPT_CACHING = "enable_anthropic_prompt_caching";
 const ANTHROPIC_PROMPT_CACHING_TTL = "anthropic_prompt_caching_ttl";
+const OPENAI_SYSTEM_MESSAGES_FIRST = "openai_system_messages_first";
+
+const isOn = (value: unknown) => value === true || value === "true";
 
 interface GeneralSettingsPageProps {
   accessToken: string | null;
@@ -117,14 +120,15 @@ export const PromptCachingPanel: React.FC<{
 }> = ({ accessToken, settings, onChange }) => {
   const enableSetting = settings.find((s) => s.field_name === ENABLE_ANTHROPIC_PROMPT_CACHING);
   const ttlSetting = settings.find((s) => s.field_name === ANTHROPIC_PROMPT_CACHING_TTL);
+  const systemFirstSetting = settings.find((s) => s.field_name === OPENAI_SYSTEM_MESSAGES_FIRST);
 
-  // The two rows come from the same registry the General tab reads; if they
+  // The rows come from the same registry the General tab reads; if they
   // are not loaded yet there is nothing to render.
   if (!enableSetting) {
     return null;
   }
 
-  const enabled = enableSetting.field_value === true || enableSetting.field_value === "true";
+  const enabled = isOn(enableSetting.field_value);
 
   // Apply immediately: a toggle and a dropdown are direct controls, so there is
   // no separate Update button. Clearing the ttl resets it to the provider default.
@@ -175,6 +179,20 @@ export const PromptCachingPanel: React.FC<{
             </Select>
           </div>
         )}
+
+        {systemFirstSetting && (
+          <div className="mt-6 flex items-start justify-between gap-8">
+            <div className="min-w-0 max-w-2xl">
+              <p className="font-medium">System messages first for OpenAI</p>
+              <p className="mt-1 break-words text-xs text-muted-foreground">{systemFirstSetting.field_description}</p>
+            </div>
+            <Switch
+              aria-label="System messages first for OpenAI"
+              checked={isOn(systemFirstSetting.field_value)}
+              onCheckedChange={(checked) => persist(OPENAI_SYSTEM_MESSAGES_FIRST, checked)}
+            />
+          </div>
+        )}
       </CardContent>
     </Card>
   );