fix(responses): keep over-cap system prompts out of instructions

OpenAI rejects Responses instructions longer than 1,048,576 characters.
When the chat to Responses bridge would put a joined system prompt over
that cap into instructions, send it as a system input item instead, matching
the existing system-only and json_object routing from #49957.

Fixes #50243

Signed-off-by: KVR Vishnu <46535652+VISHNU0906@users.noreply.github.com>

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -102,6 +102,7 @@ _CHAT_COMPLETION_FIELDS: Final = frozenset((*ModelResponse.model_fields, "usage"
 _RESPONSES_API_ONLY_FIELDS: Final = frozenset((*Response.model_fields, *ResponsesAPIResponse.model_fields)) - frozenset(
     ChatCompletion.model_fields
 )
+_RESPONSES_INSTRUCTIONS_MAX_LENGTH: Final = 1_048_576
 
 
 def _bridged_prompt_cache_breakpoint(marker: object, drop_params: bool, target_trims_breakpoints: bool) -> object:
@@ -1181,10 +1182,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             replay_openai_history=provider == "openai",
         )
         # Keep the system text in input instead of instructions when the Responses API needs it there.
-        # It rejects an empty input, and json_object requires the word "json" in input, ignoring instructions.
+        # It rejects an empty input, caps instructions at 1_048_576 chars, and json_object looks for "json" in input only.
         typed_optional_params: Final = cast("Mapping[str, object]", optional_params)  # cast-ok: untyped dict
         system_goes_in_input: Final = converted_instructions is not None and (
-            not converted_input_items or _payload_type(typed_optional_params.get("response_format")) == "json_object"
+            not converted_input_items
+            or _payload_type(typed_optional_params.get("response_format")) == "json_object"
+            or len(converted_instructions) > _RESPONSES_INSTRUCTIONS_MAX_LENGTH
         )
         target_input_items: Final = (
             _without_audio_input_parts(converted_input_items)

tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py

@@ -1901,6 +1901,86 @@ def test_transform_request_json_object_keeps_system_prompt_where_the_json_word_c
     assert (result["input"], result.get("instructions")) == (expected_input, expected_instructions), result
 
 
+@pytest.mark.parametrize(
+    ("system_len", "expect_system_in_input"),
+    [
+        # OpenAI Responses instructions max length (error text; issue #50243, 2026-11-05)
+        (1_048_576, False),
+        (1_048_576 + 1, True),
+    ],
+    ids=["at_cap_keeps_instructions", "over_cap_moves_system_into_input"],
+)
+def test_transform_request_routes_oversized_system_prompt_out_of_instructions(
+    system_len: int, expect_system_in_input: bool
+) -> None:
+    """OpenAI caps Responses instructions at 1_048_576 chars; bridge must not send over-cap text there.
+
+    Regression for #50243. Same routing as system-only / json_object: carry the joined
+    system text as the first system input item when it would exceed the cap.
+    """
+    from litellm.completion_extras.litellm_responses_transformation.transformation import (
+        _RESPONSES_INSTRUCTIONS_MAX_LENGTH,
+    )
+
+    assert _RESPONSES_INSTRUCTIONS_MAX_LENGTH == 1_048_576
+    system_text = "a" * system_len
+    user_text = "Reply with only the word OK."
+    result = LiteLLMResponsesTransformationHandler().transform_request(
+        model="gpt-5.4-mini",
+        messages=[
+            {"role": "system", "content": system_text},
+            {"role": "user", "content": user_text},
+        ],
+        optional_params={},
+        litellm_params={},
+        headers={},
+        litellm_logging_obj=Mock(),
+    )
+
+    user_item = {
+        "type": "message",
+        "role": "user",
+        "content": [{"type": "input_text", "text": user_text}],
+    }
+    system_item = {
+        "type": "message",
+        "role": "system",
+        "content": [{"type": "input_text", "text": system_text}],
+    }
+    if expect_system_in_input:
+        assert result["input"] == [system_item, user_item], result
+        assert not result.get("instructions")
+    else:
+        assert result["input"] == [user_item], result
+        assert result.get("instructions") == system_text
+
+
+def test_transform_request_short_system_prompt_still_uses_instructions() -> None:
+    """Under-cap string system prompts keep the existing instructions path."""
+    system_text = "You are a helpful assistant."
+    user_text = "Hi"
+    result = LiteLLMResponsesTransformationHandler().transform_request(
+        model="gpt-5.4-mini",
+        messages=[
+            {"role": "system", "content": system_text},
+            {"role": "user", "content": user_text},
+        ],
+        optional_params={},
+        litellm_params={},
+        headers={},
+        litellm_logging_obj=Mock(),
+    )
+
+    assert result["input"] == [
+        {
+            "type": "message",
+            "role": "user",
+            "content": [{"type": "input_text", "text": user_text}],
+        }
+    ]
+    assert result.get("instructions") == system_text
+
+
 def test_transform_request_single_char_keys_not_matched():
     """Test that single-character keys are not incorrectly matched to 'metadata' or 'previous_response_id'