fix: preserve routed prompt generation settings (#51702)

* fix: preserve async prompt generation settings

* fix: honor ignored settings on routed prompts

* fix: type prompt option handoff boundaries

* fix: bound prompt hook types without duplicate inference

* fix: preserve legacy prompt initializer parameters

---------

Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com>

litellm/integrations/dotprompt/__init__.py

@@ -77,6 +77,11 @@ def prompt_initializer(litellm_params: "PromptLiteLLMParams", prompt_spec: "Prom
             prompt_data=prompt_data,
             prompt_file=prompt_file,
             prompt_id=registration_prompt_id,
+            ignore_prompt_manager_optional_params=(
+                litellm_params.ignore_prompt_manager_optional_params
+                if hasattr(litellm_params, "ignore_prompt_manager_optional_params")
+                else False
+            ),
         )
 
         return dot_prompt_manager

litellm/integrations/dotprompt/dotprompt_manager.py

@@ -54,9 +54,11 @@ class DotpromptManager(CustomPromptManagement):
         prompt_file: str | None = None,
         prompt_data: dict | str | None = None,
         prompt_id: str | None = None,
+        ignore_prompt_manager_optional_params: bool | None = False,
     ):
         import litellm
 
+        self.ignore_prompt_manager_optional_params = ignore_prompt_manager_optional_params
         self.prompt_directory = prompt_directory or litellm.global_prompt_directory
         # Support for JSON-based prompts stored in memory/database
         if isinstance(prompt_data, str):

litellm/integrations/prompt_management_base.py

@@ -33,6 +33,8 @@ def resolve_prompt_manager_ignore_flags(
 
 
 class PromptManagementBase(ABC):
+    ignore_prompt_manager_optional_params: bool | None = False
+
     @property
     @abstractmethod
     def integration_name(self) -> str:
@@ -198,7 +200,8 @@ class PromptManagementBase(ABC):
         resolved_ignore_model, resolved_ignore_optional_params = resolve_prompt_manager_ignore_flags(
             prompt_spec=prompt_spec,
             ignore_prompt_manager_model=ignore_prompt_manager_model,
-            ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params,
+            ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params
+            or self.ignore_prompt_manager_optional_params,
         )
         return self.post_compile_prompt_processing(
             prompt_template=prompt_template,
@@ -245,7 +248,8 @@ class PromptManagementBase(ABC):
         resolved_ignore_model, resolved_ignore_optional_params = resolve_prompt_manager_ignore_flags(
             prompt_spec=prompt_spec,
             ignore_prompt_manager_model=ignore_prompt_manager_model,
-            ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params,
+            ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params
+            or self.ignore_prompt_manager_optional_params,
         )
         return self.post_compile_prompt_processing(
             prompt_template=prompt_template,

litellm/main.py

@@ -237,6 +237,7 @@ from .litellm_core_utils.core_helpers import (
     drops_unsupported_params,
     normalize_drop_params,
     safe_deep_copy,
+    without,
 )
 from .litellm_core_utils.fallback_utils import (
     async_completion_with_fallbacks,
@@ -807,27 +808,29 @@ async def acompletion(
         prompt_management_logger, AnthropicCacheControlHook
     )
 
-    if (
-        isinstance(litellm_logging_obj, LiteLLMLoggingObj)
+    prompt_hook_result: Final = (
+        cast(  # cast-ok: the hook returns a bare dict of request params
+            "tuple[str, list[AllMessageValues], Mapping[str, object]]",
+            await litellm_logging_obj.async_get_chat_completion_prompt(
+                model=model,
+                messages=messages,
+                non_default_params=kwargs,
+                prompt_id=prompt_id,
+                prompt_variables=kwargs.get("prompt_variables", None),
+                prompt_management_logger=prompt_management_logger,
+                tools=tools,
+                prompt_label=kwargs.get("prompt_label", None),
+                prompt_version=kwargs.get("prompt_version", None),
+                request_kwargs=kwargs,
+            ),
+        )
+        if isinstance(litellm_logging_obj, LiteLLMLoggingObj)
         and prompt_management_logger is not None
         and not each_fallback_writes_its_own_cache_dialect
-    ):
-        (
-            model,
-            messages,
-            _,
-        ) = await litellm_logging_obj.async_get_chat_completion_prompt(
-            model=model,
-            messages=messages,
-            non_default_params=kwargs,
-            prompt_id=prompt_id,
-            prompt_variables=kwargs.get("prompt_variables", None),
-            prompt_management_logger=prompt_management_logger,
-            tools=tools,
-            prompt_label=kwargs.get("prompt_label", None),
-            prompt_version=kwargs.get("prompt_version", None),
-            request_kwargs=kwargs,
-        )
+        else None
+    )
+    if prompt_hook_result is not None:
+        model, messages, _ = prompt_hook_result
         #########################################################
         # if the chat completion logging hook removed all tools,
         # set tools to None
@@ -896,8 +899,14 @@ async def acompletion(
         "shared_session": shared_session,
         "enable_json_schema_validation": enable_json_schema_validation,
     }
+    request_kwargs: Final = cast("Mapping[str, object]", kwargs)  # cast-ok: untyped request kwargs
+    prompt_params: Final[Mapping[str, object]] = prompt_hook_result[2] if prompt_hook_result is not None else {}
+    prompt_overrides: Final = {
+        key: value
+        for key, value in prompt_params.items()
+        if key not in request_kwargs or request_kwargs[key] is not value
+    }
     if custom_llm_provider is None:
-        request_kwargs: Final = cast("Mapping[str, object]", kwargs)  # cast-ok: untyped request kwargs
         _, custom_llm_provider, _, _ = get_llm_provider(
             model=model,
             custom_llm_provider=custom_llm_provider,
@@ -936,7 +945,9 @@ async def acompletion(
     try:
         # Use a partial function to pass your keyword arguments
         kwargs.pop("acompletion", None)
-        func: Final = partial(completion, **completion_kwargs, **kwargs)
+        func: Final = partial(
+            completion, **{**completion_kwargs, **prompt_overrides}, **without(request_kwargs, *prompt_overrides)
+        )
 
         # Add the context to the function
         ctx: Final = contextvars.copy_context()
@@ -977,7 +988,7 @@ async def acompletion(
                     messages=messages,
                     optional_params={
                         k: v
-                        for k, v in completion_kwargs.items()
+                        for k, v in {**completion_kwargs, **prompt_overrides}.items()
                         if v is not None
                         and k
                         not in (
@@ -1010,7 +1021,7 @@ async def acompletion(
             model=model,
             custom_llm_provider=custom_llm_provider,
             original_exception=e,
-            completion_kwargs=completion_kwargs,
+            completion_kwargs={**completion_kwargs, **prompt_overrides},
             extra_kwargs=kwargs,
         )
 

tests/unit/router_strategy/test_complexity_router.py

@@ -33,6 +33,9 @@ from litellm.constants import (
     SESSION_ID_GENERATED_METADATA_KEY,
 )
 from litellm.integrations.custom_logger import CustomLogger
+from litellm.integrations.dotprompt import prompt_initializer as dotprompt_initializer
+from litellm.integrations.langfuse.langfuse_prompt_management import langfuse_client_init
+from litellm.litellm_core_utils import litellm_logging
 from litellm.router import as_output_cap, stamped_client_ceiling
 from litellm.router_api import RoutingContext
 from tests.unit.redis.stores.router_session_pins_memory import memory_pins
@@ -88,6 +91,7 @@ from litellm.router_strategy.complexity_router.tier_predictor import (
     TrainedTierArtifact,
 )
 from litellm.types.llms.openai import ResponsesAPIResponse
+from litellm.types.prompts.init_prompts import PromptInfo, PromptLiteLLMParams, PromptSpec
 from litellm.types.router import (
     Deployment,
     LiteLLM_Params,
@@ -5524,6 +5528,143 @@ class TestRouterPreRoutingSharedAliasName:
 
         assert not any(key.startswith("_alias_marker") for key in sent)
 
+_PROMPT_TIER_API_BASE: Final = "https://prompt-tier.test/v1"
+_PROMPT_TIER_SETTINGS: Final = MappingProxyType(
+    {
+        "quick": MappingProxyType({"temperature": 0.42, "max_tokens": 31}),
+        "deep": MappingProxyType({"temperature": 0.82, "max_tokens": 63}),
+    }
+)
+
+
+@pytest.fixture
+def langfuse_prompt_env(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
+    monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-prompt-tier")
+    monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-prompt-tier")
+    monkeypatch.setenv("LANGFUSE_HOST", "https://langfuse-prompt-tier.test")
+    monkeypatch.setattr(litellm_logging, "_in_memory_loggers", [])
+    langfuse_client_init.cache_clear()
+    yield
+    langfuse_client_init.cache_clear()
+
+
+def _prompt_tier_router(integration: Literal["dotprompt", "langfuse"]) -> Router:
+    def tier_deployment(tier: str) -> Mapping[str, object]:
+        return {
+            "model_name": tier,
+            "litellm_params": {
+                "model": f"{'langfuse/' if integration == 'langfuse' else ''}openai/{tier}-tier",
+                "api_base": _PROMPT_TIER_API_BASE,
+                "api_key": "test-key",
+                **({"prompt_id": f"{tier}-prompt"} if integration == "langfuse" else {}),
+            },
+            "model_info": {"max_output_tokens": 4096},
+        }
+
+    tiers: Final = {
+        tier_name: (
+            tier
+            if integration == "langfuse"
+            else {"model_name": tier, "litellm_params": {"prompt_id": f"{tier}-prompt"}}
+        )
+        for tier_name, tier in (("SIMPLE", "quick"), ("REASONING", "deep"))
+    }
+    return Router(
+        model_list=[
+            {
+                "model_name": "smart-router",
+                "litellm_params": {
+                    "model": "auto_router/complexity_router",
+                    "complexity_router_default_model": "quick",
+                    "complexity_router_config": {
+                        "classifier_type": "heuristic",
+                        "context_compaction": False,
+                        "keyword_tier_rules": [
+                            {"keywords": ["markerquick"], "tier": "SIMPLE"},
+                            {"keywords": ["markerdeep"], "tier": "REASONING"},
+                        ],
+                        "tiers": tiers,
+                    },
+                },
+            },
+            tier_deployment("quick"),
+            tier_deployment("deep"),
+        ]
+    )
+
+
+def _registered_dotprompt(name: str, settings: Mapping[str, object], ignore_settings: bool) -> CustomLogger:
+    frontmatter: Final = "".join(f"{key}: {value}\n" for key, value in settings.items())
+    params: Final = PromptLiteLLMParams(
+        prompt_id=f"{name}-prompt",
+        prompt_integration="dotprompt",
+        dotprompt_content=f"---\n{frontmatter}---\nSystem: Prompt {name}.\n",
+        ignore_prompt_manager_optional_params=ignore_settings,
+    )
+    spec: Final = PromptSpec(
+        prompt_id=f"{name}-prompt", litellm_params=params, prompt_info=PromptInfo(prompt_type="config")
+    )
+    return dotprompt_initializer(params, spec)
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("langfuse_prompt_env")
+@pytest.mark.parametrize(
+    ("integration", "ignore_settings"),
+    [("dotprompt", False), ("dotprompt", True), ("langfuse", False)],
+    ids=["dotprompt", "dotprompt-ignoring-settings", "langfuse"],
+)
+@pytest.mark.parametrize("tier", ["quick", "deep"])
+async def test_a_tiers_selected_prompt_beats_the_tier_output_ceiling_unless_it_ignores_its_settings(
+    monkeypatch: pytest.MonkeyPatch, integration: Literal["dotprompt", "langfuse"], ignore_settings: bool, tier: str
+) -> None:
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    if integration == "dotprompt":
+        monkeypatch.setattr(
+            litellm,
+            "callbacks",
+            [
+                _registered_dotprompt(name, settings, ignore_settings)
+                for name, settings in _PROMPT_TIER_SETTINGS.items()
+            ],
+        )
+    router: Final = _prompt_tier_router(integration)
+
+    with respx.mock(assert_all_mocked=True, assert_all_called=False) as upstream:
+        for name, settings in _PROMPT_TIER_SETTINGS.items():
+            upstream.get(host="langfuse-prompt-tier.test", path=f"/api/public/v2/prompts/{name}-prompt").respond(
+                json={
+                    "type": "text",
+                    "name": f"{name}-prompt",
+                    "version": 1,
+                    "config": dict(settings),
+                    "labels": ["production"],
+                    "tags": [],
+                    "prompt": f"Prompt {name}.",
+                }
+            )
+        endpoint: Final = upstream.post(f"{_PROMPT_TIER_API_BASE}/chat/completions").respond(
+            json={
+                "id": "chatcmpl-prompt-tier",
+                "object": "chat.completion",
+                "created": 1,
+                "model": f"{tier}-tier",
+                "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
+                "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4},
+            }
+        )
+        await router.acompletion(
+            model="smart-router", messages=[{"role": "user", "content": f"marker{tier}"}], temperature=0
+        )
+
+    assert endpoint.call_count == 1
+    sent: Final = json.loads(endpoint.calls[0].request.content)
+    assert sent["model"] == f"{tier}-tier"
+    assert sent["messages"][0] == {"role": "system", "content": f"Prompt {tier}."}
+    expected: Final = {"temperature": 0, "max_tokens": 4096} if ignore_settings else dict(_PROMPT_TIER_SETTINGS[tier])
+    assert {key: sent.get(key) for key in _PROMPT_TIER_SETTINGS[tier]} == expected
+
+
 class TestAdaptiveSoftFloors:
     def test_adaptive_defaults_use_cost_weighted_cold_policy(self):
         config = ComplexityRouterConfig(

tests/unit/test_main.py

@@ -8,7 +8,7 @@ import logging
 import os
 import threading
 import urllib.parse
-from collections.abc import Awaitable, Callable, Mapping
+from collections.abc import Awaitable, Callable, Iterator, Mapping
 from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass
 from datetime import datetime
@@ -40,6 +40,9 @@ from litellm.caching.caching import Cache
 from litellm.constants import CONTROL_OPTIONS_KEY
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.integrations.custom_prompt_management import CustomPromptManagement
+from litellm.integrations.dotprompt.dotprompt_manager import DotpromptManager
+from litellm.integrations.langfuse.langfuse_prompt_management import langfuse_client_init
+from litellm.litellm_core_utils import litellm_logging
 from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs
 from litellm.litellm_core_utils.get_litellm_params import stored_control_options
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
@@ -6959,6 +6962,90 @@ async def test_an_invalid_stream_chunk_size_is_rejected_before_any_prompt_hook_r
     on_prompt.assert_not_called()
 
 
+_PROMPT_SETTINGS_API_BASE: Final = "https://prompt-settings.test/v1"
+_LANGFUSE_PROMPT_HOST: Final = "https://langfuse-prompts.test"
+
+
+@pytest.fixture
+def langfuse_prompt_env(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
+    monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-prompt-settings")
+    monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-prompt-settings")
+    monkeypatch.setenv("LANGFUSE_HOST", _LANGFUSE_PROMPT_HOST)
+    monkeypatch.setattr(litellm_logging, "_in_memory_loggers", [])
+    langfuse_client_init.cache_clear()
+    yield
+    langfuse_client_init.cache_clear()
+
+
+def _serve_langfuse_text_prompt(
+    respx_mock: respx.MockRouter, name: str, text: str, config: Mapping[str, object]
+) -> None:
+    respx_mock.get(host="langfuse-prompts.test", path=f"/api/public/v2/prompts/{name}").respond(
+        json={
+            "type": "text",
+            "name": name,
+            "version": 1,
+            "config": dict(config),
+            "labels": ["production"],
+            "tags": [],
+            "prompt": text,
+        }
+    )
+
+
+def _serve_chat_completion(respx_mock: respx.MockRouter) -> respx.Route:
+    return respx_mock.post(f"{_PROMPT_SETTINGS_API_BASE}/chat/completions").respond(
+        json={
+            "id": "chatcmpl-prompt-settings",
+            "object": "chat.completion",
+            "created": 1,
+            "model": "prompt-settings-test",
+            "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
+            "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4},
+        }
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("langfuse_prompt_env")
+@pytest.mark.parametrize("integration", ["dotprompt", "langfuse"])
+async def test_acompletion_sends_the_selected_prompts_generation_settings(
+    monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, integration: str
+) -> None:
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    template_settings: Final = {"temperature": 0.42, "max_tokens": 31}
+    if integration == "dotprompt":
+        monkeypatch.setattr(
+            litellm,
+            "callbacks",
+            [
+                DotpromptManager(
+                    prompt_data={"pinned": {"content": "System: Pinned prompt.", "metadata": template_settings}}
+                )
+            ],
+        )
+    else:
+        _serve_langfuse_text_prompt(respx_mock, "pinned", "Pinned prompt.", template_settings)
+    endpoint: Final = _serve_chat_completion(respx_mock)
+
+    await litellm.acompletion(
+        model=f"{'langfuse/' if integration == 'langfuse' else ''}openai/prompt-settings-test",
+        messages=[{"role": "user", "content": "hi"}],
+        prompt_id="pinned",
+        temperature=0,
+        top_p=0,
+        api_base=_PROMPT_SETTINGS_API_BASE,
+        api_key="test-key",
+    )
+
+    assert endpoint.call_count == 1
+    sent: Final = json.loads(endpoint.calls[0].request.content)
+    assert sent["model"] == "prompt-settings-test"
+    assert sent["messages"][0] == {"role": "system", "content": "Pinned prompt."}
+    assert sent["messages"][-1] == {"role": "user", "content": "hi"}
+    assert (sent["temperature"], sent["max_tokens"], sent["top_p"]) == (0.42, 31, 0)
+
+
 def _completion_logging_obj(call_id: str) -> LiteLLMLogging:
     return LiteLLMLogging(
         model="gpt-4.1-mini",