diff --git a/litellm/integrations/dotprompt/__init__.py b/litellm/integrations/dotprompt/__init__.py index 0a2921db6c7..ae21e65a860 100644 --- a/litellm/integrations/dotprompt/__init__.py +++ b/litellm/integrations/dotprompt/__init__.py @@ -77,6 +77,11 @@ def prompt_initializer(litellm_params: "PromptLiteLLMParams", prompt_spec: "Prom prompt_data=prompt_data, prompt_file=prompt_file, prompt_id=registration_prompt_id, + ignore_prompt_manager_optional_params=( + litellm_params.ignore_prompt_manager_optional_params + if hasattr(litellm_params, "ignore_prompt_manager_optional_params") + else False + ), ) return dot_prompt_manager diff --git a/litellm/integrations/dotprompt/dotprompt_manager.py b/litellm/integrations/dotprompt/dotprompt_manager.py index 77d70dd761c..d169f00851f 100644 --- a/litellm/integrations/dotprompt/dotprompt_manager.py +++ b/litellm/integrations/dotprompt/dotprompt_manager.py @@ -54,9 +54,11 @@ class DotpromptManager(CustomPromptManagement): prompt_file: str | None = None, prompt_data: dict | str | None = None, prompt_id: str | None = None, + ignore_prompt_manager_optional_params: bool | None = False, ): import litellm + self.ignore_prompt_manager_optional_params = ignore_prompt_manager_optional_params self.prompt_directory = prompt_directory or litellm.global_prompt_directory # Support for JSON-based prompts stored in memory/database if isinstance(prompt_data, str): diff --git a/litellm/integrations/prompt_management_base.py b/litellm/integrations/prompt_management_base.py index 3c6b5284041..6c708414859 100644 --- a/litellm/integrations/prompt_management_base.py +++ b/litellm/integrations/prompt_management_base.py @@ -33,6 +33,8 @@ def resolve_prompt_manager_ignore_flags( class PromptManagementBase(ABC): + ignore_prompt_manager_optional_params: bool | None = False + @property @abstractmethod def integration_name(self) -> str: @@ -198,7 +200,8 @@ class PromptManagementBase(ABC): resolved_ignore_model, resolved_ignore_optional_params = resolve_prompt_manager_ignore_flags( prompt_spec=prompt_spec, ignore_prompt_manager_model=ignore_prompt_manager_model, - ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params, + ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params + or self.ignore_prompt_manager_optional_params, ) return self.post_compile_prompt_processing( prompt_template=prompt_template, @@ -245,7 +248,8 @@ class PromptManagementBase(ABC): resolved_ignore_model, resolved_ignore_optional_params = resolve_prompt_manager_ignore_flags( prompt_spec=prompt_spec, ignore_prompt_manager_model=ignore_prompt_manager_model, - ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params, + ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params + or self.ignore_prompt_manager_optional_params, ) return self.post_compile_prompt_processing( prompt_template=prompt_template, diff --git a/litellm/main.py b/litellm/main.py index 35473e63852..6798242773b 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -237,6 +237,7 @@ from .litellm_core_utils.core_helpers import ( drops_unsupported_params, normalize_drop_params, safe_deep_copy, + without, ) from .litellm_core_utils.fallback_utils import ( async_completion_with_fallbacks, @@ -807,27 +808,29 @@ async def acompletion( prompt_management_logger, AnthropicCacheControlHook ) - if ( - isinstance(litellm_logging_obj, LiteLLMLoggingObj) + prompt_hook_result: Final = ( + cast( # cast-ok: the hook returns a bare dict of request params + "tuple[str, list[AllMessageValues], Mapping[str, object]]", + await litellm_logging_obj.async_get_chat_completion_prompt( + model=model, + messages=messages, + non_default_params=kwargs, + prompt_id=prompt_id, + prompt_variables=kwargs.get("prompt_variables", None), + prompt_management_logger=prompt_management_logger, + tools=tools, + prompt_label=kwargs.get("prompt_label", None), + prompt_version=kwargs.get("prompt_version", None), + request_kwargs=kwargs, + ), + ) + if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and prompt_management_logger is not None and not each_fallback_writes_its_own_cache_dialect - ): - ( - model, - messages, - _, - ) = await litellm_logging_obj.async_get_chat_completion_prompt( - model=model, - messages=messages, - non_default_params=kwargs, - prompt_id=prompt_id, - prompt_variables=kwargs.get("prompt_variables", None), - prompt_management_logger=prompt_management_logger, - tools=tools, - prompt_label=kwargs.get("prompt_label", None), - prompt_version=kwargs.get("prompt_version", None), - request_kwargs=kwargs, - ) + else None + ) + if prompt_hook_result is not None: + model, messages, _ = prompt_hook_result ######################################################### # if the chat completion logging hook removed all tools, # set tools to None @@ -896,8 +899,14 @@ async def acompletion( "shared_session": shared_session, "enable_json_schema_validation": enable_json_schema_validation, } + request_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: untyped request kwargs + prompt_params: Final[Mapping[str, object]] = prompt_hook_result[2] if prompt_hook_result is not None else {} + prompt_overrides: Final = { + key: value + for key, value in prompt_params.items() + if key not in request_kwargs or request_kwargs[key] is not value + } if custom_llm_provider is None: - request_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: untyped request kwargs _, custom_llm_provider, _, _ = get_llm_provider( model=model, custom_llm_provider=custom_llm_provider, @@ -936,7 +945,9 @@ async def acompletion( try: # Use a partial function to pass your keyword arguments kwargs.pop("acompletion", None) - func: Final = partial(completion, **completion_kwargs, **kwargs) + func: Final = partial( + completion, **{**completion_kwargs, **prompt_overrides}, **without(request_kwargs, *prompt_overrides) + ) # Add the context to the function ctx: Final = contextvars.copy_context() @@ -977,7 +988,7 @@ async def acompletion( messages=messages, optional_params={ k: v - for k, v in completion_kwargs.items() + for k, v in {**completion_kwargs, **prompt_overrides}.items() if v is not None and k not in ( @@ -1010,7 +1021,7 @@ async def acompletion( model=model, custom_llm_provider=custom_llm_provider, original_exception=e, - completion_kwargs=completion_kwargs, + completion_kwargs={**completion_kwargs, **prompt_overrides}, extra_kwargs=kwargs, ) diff --git a/tests/unit/router_strategy/test_complexity_router.py b/tests/unit/router_strategy/test_complexity_router.py index 8a990c71ed8..783d767bc68 100644 --- a/tests/unit/router_strategy/test_complexity_router.py +++ b/tests/unit/router_strategy/test_complexity_router.py @@ -33,6 +33,9 @@ from litellm.constants import ( SESSION_ID_GENERATED_METADATA_KEY, ) from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.dotprompt import prompt_initializer as dotprompt_initializer +from litellm.integrations.langfuse.langfuse_prompt_management import langfuse_client_init +from litellm.litellm_core_utils import litellm_logging from litellm.router import as_output_cap, stamped_client_ceiling from litellm.router_api import RoutingContext from tests.unit.redis.stores.router_session_pins_memory import memory_pins @@ -88,6 +91,7 @@ from litellm.router_strategy.complexity_router.tier_predictor import ( TrainedTierArtifact, ) from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.prompts.init_prompts import PromptInfo, PromptLiteLLMParams, PromptSpec from litellm.types.router import ( Deployment, LiteLLM_Params, @@ -5524,6 +5528,143 @@ class TestRouterPreRoutingSharedAliasName: assert not any(key.startswith("_alias_marker") for key in sent) +_PROMPT_TIER_API_BASE: Final = "https://prompt-tier.test/v1" +_PROMPT_TIER_SETTINGS: Final = MappingProxyType( + { + "quick": MappingProxyType({"temperature": 0.42, "max_tokens": 31}), + "deep": MappingProxyType({"temperature": 0.82, "max_tokens": 63}), + } +) + + +@pytest.fixture +def langfuse_prompt_env(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-prompt-tier") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-prompt-tier") + monkeypatch.setenv("LANGFUSE_HOST", "https://langfuse-prompt-tier.test") + monkeypatch.setattr(litellm_logging, "_in_memory_loggers", []) + langfuse_client_init.cache_clear() + yield + langfuse_client_init.cache_clear() + + +def _prompt_tier_router(integration: Literal["dotprompt", "langfuse"]) -> Router: + def tier_deployment(tier: str) -> Mapping[str, object]: + return { + "model_name": tier, + "litellm_params": { + "model": f"{'langfuse/' if integration == 'langfuse' else ''}openai/{tier}-tier", + "api_base": _PROMPT_TIER_API_BASE, + "api_key": "test-key", + **({"prompt_id": f"{tier}-prompt"} if integration == "langfuse" else {}), + }, + "model_info": {"max_output_tokens": 4096}, + } + + tiers: Final = { + tier_name: ( + tier + if integration == "langfuse" + else {"model_name": tier, "litellm_params": {"prompt_id": f"{tier}-prompt"}} + ) + for tier_name, tier in (("SIMPLE", "quick"), ("REASONING", "deep")) + } + return Router( + model_list=[ + { + "model_name": "smart-router", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_default_model": "quick", + "complexity_router_config": { + "classifier_type": "heuristic", + "context_compaction": False, + "keyword_tier_rules": [ + {"keywords": ["markerquick"], "tier": "SIMPLE"}, + {"keywords": ["markerdeep"], "tier": "REASONING"}, + ], + "tiers": tiers, + }, + }, + }, + tier_deployment("quick"), + tier_deployment("deep"), + ] + ) + + +def _registered_dotprompt(name: str, settings: Mapping[str, object], ignore_settings: bool) -> CustomLogger: + frontmatter: Final = "".join(f"{key}: {value}\n" for key, value in settings.items()) + params: Final = PromptLiteLLMParams( + prompt_id=f"{name}-prompt", + prompt_integration="dotprompt", + dotprompt_content=f"---\n{frontmatter}---\nSystem: Prompt {name}.\n", + ignore_prompt_manager_optional_params=ignore_settings, + ) + spec: Final = PromptSpec( + prompt_id=f"{name}-prompt", litellm_params=params, prompt_info=PromptInfo(prompt_type="config") + ) + return dotprompt_initializer(params, spec) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("langfuse_prompt_env") +@pytest.mark.parametrize( + ("integration", "ignore_settings"), + [("dotprompt", False), ("dotprompt", True), ("langfuse", False)], + ids=["dotprompt", "dotprompt-ignoring-settings", "langfuse"], +) +@pytest.mark.parametrize("tier", ["quick", "deep"]) +async def test_a_tiers_selected_prompt_beats_the_tier_output_ceiling_unless_it_ignores_its_settings( + monkeypatch: pytest.MonkeyPatch, integration: Literal["dotprompt", "langfuse"], ignore_settings: bool, tier: str +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + if integration == "dotprompt": + monkeypatch.setattr( + litellm, + "callbacks", + [ + _registered_dotprompt(name, settings, ignore_settings) + for name, settings in _PROMPT_TIER_SETTINGS.items() + ], + ) + router: Final = _prompt_tier_router(integration) + + with respx.mock(assert_all_mocked=True, assert_all_called=False) as upstream: + for name, settings in _PROMPT_TIER_SETTINGS.items(): + upstream.get(host="langfuse-prompt-tier.test", path=f"/api/public/v2/prompts/{name}-prompt").respond( + json={ + "type": "text", + "name": f"{name}-prompt", + "version": 1, + "config": dict(settings), + "labels": ["production"], + "tags": [], + "prompt": f"Prompt {name}.", + } + ) + endpoint: Final = upstream.post(f"{_PROMPT_TIER_API_BASE}/chat/completions").respond( + json={ + "id": "chatcmpl-prompt-tier", + "object": "chat.completion", + "created": 1, + "model": f"{tier}-tier", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}, + } + ) + await router.acompletion( + model="smart-router", messages=[{"role": "user", "content": f"marker{tier}"}], temperature=0 + ) + + assert endpoint.call_count == 1 + sent: Final = json.loads(endpoint.calls[0].request.content) + assert sent["model"] == f"{tier}-tier" + assert sent["messages"][0] == {"role": "system", "content": f"Prompt {tier}."} + expected: Final = {"temperature": 0, "max_tokens": 4096} if ignore_settings else dict(_PROMPT_TIER_SETTINGS[tier]) + assert {key: sent.get(key) for key in _PROMPT_TIER_SETTINGS[tier]} == expected + + class TestAdaptiveSoftFloors: def test_adaptive_defaults_use_cost_weighted_cold_policy(self): config = ComplexityRouterConfig( diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index ddecf43de91..e6916b05810 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -8,7 +8,7 @@ import logging import os import threading import urllib.parse -from collections.abc import Awaitable, Callable, Mapping +from collections.abc import Awaitable, Callable, Iterator, Mapping from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from datetime import datetime @@ -40,6 +40,9 @@ from litellm.caching.caching import Cache from litellm.constants import CONTROL_OPTIONS_KEY from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.custom_prompt_management import CustomPromptManagement +from litellm.integrations.dotprompt.dotprompt_manager import DotpromptManager +from litellm.integrations.langfuse.langfuse_prompt_management import langfuse_client_init +from litellm.litellm_core_utils import litellm_logging from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs from litellm.litellm_core_utils.get_litellm_params import stored_control_options from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging @@ -6959,6 +6962,90 @@ async def test_an_invalid_stream_chunk_size_is_rejected_before_any_prompt_hook_r on_prompt.assert_not_called() +_PROMPT_SETTINGS_API_BASE: Final = "https://prompt-settings.test/v1" +_LANGFUSE_PROMPT_HOST: Final = "https://langfuse-prompts.test" + + +@pytest.fixture +def langfuse_prompt_env(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-prompt-settings") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-prompt-settings") + monkeypatch.setenv("LANGFUSE_HOST", _LANGFUSE_PROMPT_HOST) + monkeypatch.setattr(litellm_logging, "_in_memory_loggers", []) + langfuse_client_init.cache_clear() + yield + langfuse_client_init.cache_clear() + + +def _serve_langfuse_text_prompt( + respx_mock: respx.MockRouter, name: str, text: str, config: Mapping[str, object] +) -> None: + respx_mock.get(host="langfuse-prompts.test", path=f"/api/public/v2/prompts/{name}").respond( + json={ + "type": "text", + "name": name, + "version": 1, + "config": dict(config), + "labels": ["production"], + "tags": [], + "prompt": text, + } + ) + + +def _serve_chat_completion(respx_mock: respx.MockRouter) -> respx.Route: + return respx_mock.post(f"{_PROMPT_SETTINGS_API_BASE}/chat/completions").respond( + json={ + "id": "chatcmpl-prompt-settings", + "object": "chat.completion", + "created": 1, + "model": "prompt-settings-test", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}, + } + ) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("langfuse_prompt_env") +@pytest.mark.parametrize("integration", ["dotprompt", "langfuse"]) +async def test_acompletion_sends_the_selected_prompts_generation_settings( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, integration: str +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + template_settings: Final = {"temperature": 0.42, "max_tokens": 31} + if integration == "dotprompt": + monkeypatch.setattr( + litellm, + "callbacks", + [ + DotpromptManager( + prompt_data={"pinned": {"content": "System: Pinned prompt.", "metadata": template_settings}} + ) + ], + ) + else: + _serve_langfuse_text_prompt(respx_mock, "pinned", "Pinned prompt.", template_settings) + endpoint: Final = _serve_chat_completion(respx_mock) + + await litellm.acompletion( + model=f"{'langfuse/' if integration == 'langfuse' else ''}openai/prompt-settings-test", + messages=[{"role": "user", "content": "hi"}], + prompt_id="pinned", + temperature=0, + top_p=0, + api_base=_PROMPT_SETTINGS_API_BASE, + api_key="test-key", + ) + + assert endpoint.call_count == 1 + sent: Final = json.loads(endpoint.calls[0].request.content) + assert sent["model"] == "prompt-settings-test" + assert sent["messages"][0] == {"role": "system", "content": "Pinned prompt."} + assert sent["messages"][-1] == {"role": "user", "content": "hi"} + assert (sent["temperature"], sent["max_tokens"], sent["top_p"]) == (0.42, 31, 0) + + def _completion_logging_obj(call_id: str) -> LiteLLMLogging: return LiteLLMLogging( model="gpt-4.1-mini",