mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-14 23:21:35 +00:00
fix(prompt_management): don't route requests without prompt_id to prompt managers that can't run them
UI-injected empty vector_store_ids/tags/guardrails on a DB model tripped the dynamic-param check, and the prompt-management fallback then handed the request to the first registered prompt manager (e.g. a saved dotprompt), whose sync path raised "prompt_id is required" as a 500 on every /chat/completions call. Empty dynamic params no longer count as a trigger, the fallback skips managers whose should_run_prompt_management declines a None prompt_id, and the sync base path returns the request unchanged for a None prompt_id like the async path.
This commit is contained in:
parent
7b574b9df6
commit
4829bb3a15
3 changed files with 105 additions and 5 deletions
|
|
@ -165,7 +165,7 @@ class PromptManagementBase(ABC):
|
|||
ignore_prompt_manager_optional_params: bool | None = False,
|
||||
) -> tuple[str, list[AllMessageValues], dict]:
|
||||
if prompt_id is None:
|
||||
raise ValueError("prompt_id is required for Prompt Management Base class")
|
||||
return model, messages, non_default_params
|
||||
if not self.should_run_prompt_management(
|
||||
prompt_id=prompt_id,
|
||||
prompt_spec=prompt_spec,
|
||||
|
|
|
|||
|
|
@ -832,8 +832,8 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
eg. AnthropicCacheControlHook and BedrockKnowledgeBaseHook both don't require a `prompt_id` to be passed in, they are triggered by dynamic params
|
||||
"""
|
||||
for param in non_default_params:
|
||||
if param in DynamicPromptManagementParamLiteral.list_all_params():
|
||||
for param in DynamicPromptManagementParamLiteral.list_all_params():
|
||||
if non_default_params.get(param):
|
||||
return True
|
||||
|
||||
#############################################################################
|
||||
|
|
@ -966,6 +966,23 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _prompt_manager_runs_without_prompt_id(
|
||||
logger: CustomLogger,
|
||||
prompt_spec: PromptSpec | None,
|
||||
dynamic_callback_params: StandardCallbackDynamicParams | None,
|
||||
) -> bool:
|
||||
if not isinstance(logger, CustomPromptManagement):
|
||||
return False
|
||||
try:
|
||||
return logger.should_run_prompt_management(
|
||||
prompt_id=None,
|
||||
prompt_spec=prompt_spec,
|
||||
dynamic_callback_params=dynamic_callback_params or StandardCallbackDynamicParams(),
|
||||
)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def get_custom_logger_for_prompt_management(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -1016,8 +1033,13 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
callback_type=CustomPromptManagement
|
||||
)
|
||||
|
||||
if prompt_management_loggers:
|
||||
logger: Final = prompt_management_loggers[0]
|
||||
for logger in prompt_management_loggers:
|
||||
if prompt_id is None and not self._prompt_manager_runs_without_prompt_id(
|
||||
logger=logger,
|
||||
prompt_spec=prompt_spec,
|
||||
dynamic_callback_params=dynamic_callback_params,
|
||||
):
|
||||
continue
|
||||
self.model_call_details["prompt_integration"] = logger.__class__.__name__
|
||||
return logger
|
||||
|
||||
|
|
|
|||
|
|
@ -5105,3 +5105,81 @@ def test_set_cost_breakdown_stores_vertex_location():
|
|||
cost_for_built_in_tools_cost_usd_dollar=0.0,
|
||||
)
|
||||
assert no_location.cost_breakdown.get("vertex_location") is None
|
||||
|
||||
|
||||
def test_prompt_hooks_skip_prompt_managers_when_no_prompt_id(logging_obj, tmp_path, monkeypatch):
|
||||
"""
|
||||
Regression for UI-injected `vector_store_ids: []` and always-on non-empty `vector_store_ids`
|
||||
with a registered prompt manager (e.g. dotprompt): requests without a prompt_id 500'd with
|
||||
"prompt_id is required for Prompt Management Base class" instead of completing normally.
|
||||
"""
|
||||
from litellm.integrations.dotprompt.dotprompt_manager import DotpromptManager
|
||||
from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import (
|
||||
VectorStorePreCallHook,
|
||||
)
|
||||
from litellm.types.vector_stores import LiteLLM_ManagedVectorStore
|
||||
from litellm.vector_stores.vector_store_registry import VectorStoreRegistry
|
||||
|
||||
(tmp_path / "stem.prompt").write_text("---\nmodel: gemini-2.5-flash\n---\nyou are a stem tutor\n")
|
||||
dotprompt_manager = DotpromptManager(prompt_directory=str(tmp_path))
|
||||
litellm.logging_callback_manager.add_litellm_callback(dotprompt_manager)
|
||||
monkeypatch.setattr(
|
||||
litellm,
|
||||
"vector_store_registry",
|
||||
VectorStoreRegistry(
|
||||
vector_stores=[LiteLLM_ManagedVectorStore(vector_store_id="vs_123", custom_llm_provider="openai")]
|
||||
),
|
||||
)
|
||||
|
||||
messages = [{"role": "user", "content": "hi"}]
|
||||
try:
|
||||
assert not logging_obj.should_run_prompt_management_hooks(
|
||||
prompt_id=None, non_default_params={"vector_store_ids": []}
|
||||
)
|
||||
|
||||
assert logging_obj.get_chat_completion_prompt(
|
||||
model="gemini-2.5-flash",
|
||||
messages=messages,
|
||||
non_default_params={"vector_store_ids": []},
|
||||
prompt_variables=None,
|
||||
prompt_id=None,
|
||||
) == ("gemini-2.5-flash", messages, {"vector_store_ids": []})
|
||||
|
||||
assert dotprompt_manager.get_chat_completion_prompt(
|
||||
model="gemini-2.5-flash",
|
||||
messages=messages,
|
||||
non_default_params={},
|
||||
prompt_id=None,
|
||||
prompt_variables=None,
|
||||
dynamic_callback_params={},
|
||||
) == ("gemini-2.5-flash", messages, {})
|
||||
|
||||
assert logging_obj.should_run_prompt_management_hooks(
|
||||
prompt_id=None, non_default_params={"vector_store_ids": ["vs_123"]}
|
||||
)
|
||||
assert isinstance(
|
||||
logging_obj.get_custom_logger_for_prompt_management(
|
||||
model="gemini-2.5-flash",
|
||||
non_default_params={"vector_store_ids": ["vs_123"]},
|
||||
prompt_id=None,
|
||||
dynamic_callback_params={},
|
||||
),
|
||||
VectorStorePreCallHook,
|
||||
)
|
||||
|
||||
assert isinstance(
|
||||
logging_obj.get_custom_logger_for_prompt_management(
|
||||
model="gemini-2.5-flash",
|
||||
non_default_params={},
|
||||
prompt_id="stem",
|
||||
dynamic_callback_params={},
|
||||
),
|
||||
DotpromptManager,
|
||||
)
|
||||
finally:
|
||||
litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm.callbacks, dotprompt_manager)
|
||||
litellm.logging_callback_manager.remove_callback_from_list_by_object(
|
||||
litellm._async_success_callback, dotprompt_manager
|
||||
)
|
||||
for hook in [cb for cb in litellm.callbacks if isinstance(cb, VectorStorePreCallHook)]:
|
||||
litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm.callbacks, hook)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue