fix(proxy): run submit-time post_call policies when a background response is polled

A background POST /v1/responses defers its post_call policy pipelines, and
the poll that reads the finished output re-matched policies from its own
request only. Policies matched at submit by body tags, the body policies
list, a wildcard or aliased model name, or the submitting key never ran.

The submit now stores its deferred pipelines in the proxy's shared cache,
keyed by the upstream response id, and every poll runs them next to its
own re-match. Also stops a router test from leaving
ResponsesAPIRequestUtils.get_model_id_from_response_id mocked for the rest
of its xdist worker.
This commit is contained in:
mateo-berri 2026-10-29 01:40:59 -07:00
parent 3b9a59348f
commit 1406342672
7 changed files with 361 additions and 173 deletions

View file

@ -2571,10 +2571,11 @@ class ProxyBaseLLMRequestProcessing:
proxy_logging_obj=proxy_logging_obj,
)
if route_type == "aget_responses":
attach_post_call_pipelines_to_retrieval(
await attach_post_call_pipelines_to_retrieval(
data=self.data,
user_api_key_dict=user_api_key_dict,
llm_router=llm_router,
submit_policies_cache=proxy_logging_obj.internal_usage_cache,
)
# Refresh AFTER pre_call_hook: guardrails (e.g. Presidio PII masking) may

View file

@ -1,11 +1,12 @@
from collections.abc import Mapping
from dataclasses import dataclass
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal, TypeAlias
from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias
from pydantic import TypeAdapter
from pydantic import BaseModel, ConfigDict, TypeAdapter
from litellm._logging import verbose_proxy_logger
from litellm.constants import MANAGED_OBJECT_STALENESS_CUTOFF_DAYS
from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket
from litellm.proxy.common_utils.callback_utils import (
add_guardrail_to_applied_guardrails_header,
@ -30,6 +31,46 @@ PolicyPipelines: TypeAlias = tuple[tuple[str, GuardrailPipeline], ...]
_POLICY_PIPELINES_ADAPTER: Final = TypeAdapter(PolicyPipelines)
_SUBMIT_POLICIES_TTL_SECONDS: Final = MANAGED_OBJECT_STALENESS_CUTOFF_DAYS * 24 * 60 * 60
class SubmitPostCallPolicies(BaseModel):
model_config = ConfigDict(frozen=True)
pipelines: PolicyPipelines
policy_sources: Mapping[str, str]
class SubmitPoliciesCache(Protocol):
async def async_set_cache(self, key: str, value: object, litellm_parent_otel_span: None, *, ttl: int) -> None: ...
async def async_get_cache(self, key: str, litellm_parent_otel_span: None) -> object: ...
def _submit_policies_cache_key(response_id: str) -> str:
upstream_response_id: Final = ResponsesAPIRequestUtils.decode_responses_api_response_id(response_id).get(
"response_id", response_id
)
return f"policy_engine:background_response_post_call_policies:{upstream_response_id}"
async def remember_submit_post_call_policies(
cache: SubmitPoliciesCache, response_id: str, submitted: SubmitPostCallPolicies
) -> None:
await cache.async_set_cache(
key=_submit_policies_cache_key(response_id),
value=submitted.model_dump(mode="json"),
litellm_parent_otel_span=None,
ttl=_SUBMIT_POLICIES_TTL_SECONDS,
)
async def _submit_post_call_policies(cache: SubmitPoliciesCache, response_id: str) -> SubmitPostCallPolicies | None:
stored: Final = await cache.async_get_cache(
key=_submit_policies_cache_key(response_id), litellm_parent_otel_span=None
)
return None if stored is None else SubmitPostCallPolicies.model_validate(stored)
@dataclass(frozen=True, slots=True)
class UngovernedRetrieval:
@ -47,18 +88,23 @@ def _model_group_for_response_id(response_id: object, llm_router: "Router | None
deployment: Final = llm_router.get_deployment(model_id)
if deployment is None:
return UngovernedRetrieval("deployment no longer in the router")
hidden_by: Final = _submit_model_hidden_by(deployment.model_name, llm_router.model_group_alias)
if hidden_by is not None:
verbose_proxy_logger.warning(
"Policy engine: background response %s re-matches policies on retrieval as model group %s (%s), "
"so a policy attached to the model name it was submitted as does not run on it",
response_id,
deployment.model_name,
hidden_by,
)
return deployment.model_name
def _warn_if_submit_model_hidden(response_id: object, model_group: str, llm_router: "Router") -> None:
hidden_by: Final = _submit_model_hidden_by(model_group, llm_router.model_group_alias)
if hidden_by is None:
return
verbose_proxy_logger.warning(
"Policy engine: background response %s has no submit-time post_call policies stored and re-matches "
"policies on retrieval as model group %s (%s), so a policy attached to the model name it was submitted as "
"does not run on it",
response_id,
model_group,
hidden_by,
)
def _submit_model_hidden_by(model_group: str, model_group_alias: Mapping[str, object]) -> str | None:
if "*" in model_group:
return "a wildcard deployment"
@ -103,23 +149,53 @@ def _post_call_pipelines_for_context(context: PolicyMatchContext) -> tuple[Polic
return post_call_pipelines, MappingProxyType({match["policy_name"]: match["matched_via"] for match in matches})
def attach_post_call_pipelines_to_retrieval(
def _retrieval_post_call_pipelines(
data: Mapping[str, object],
user_api_key_dict: "UserAPIKeyAuth",
llm_router: "Router | None",
submitted: SubmitPostCallPolicies | None,
) -> tuple[PolicyPipelines, Mapping[str, str]]:
response_id: Final = data.get("response_id")
model_group: Final = _model_group_for_response_id(response_id, llm_router)
if isinstance(model_group, UngovernedRetrieval):
if submitted is None:
verbose_proxy_logger.warning(
"Policy engine: background response %s is retrieved without its post_call policy pipelines (%s)",
response_id,
model_group.reason,
)
return (), MappingProxyType({})
if submitted is None and llm_router is not None:
_warn_if_submit_model_hidden(response_id, model_group, llm_router)
return _post_call_pipelines_for_context(_retrieval_context(data, user_api_key_dict, model_group))
async def attach_post_call_pipelines_to_retrieval(
data: dict[str, object], # mutable-ok: request-state dict the policy engine hooks all write in place
user_api_key_dict: "UserAPIKeyAuth",
llm_router: "Router | None",
submit_policies_cache: SubmitPoliciesCache,
) -> None:
if not get_policy_registry().is_initialized():
return
model_group: Final = _model_group_for_response_id(data.get("response_id"), llm_router)
if isinstance(model_group, UngovernedRetrieval):
verbose_proxy_logger.warning(
"Policy engine: background response %s is retrieved without its post_call policy pipelines (%s)",
data.get("response_id"),
model_group.reason,
)
return
context: Final = _retrieval_context(data, user_api_key_dict, model_group)
post_call_pipelines, policy_sources = _post_call_pipelines_for_context(context)
response_id: Final = data.get("response_id")
submitted: Final = (
await _submit_post_call_policies(submit_policies_cache, response_id) if isinstance(response_id, str) else None
)
retrieval_pipelines, retrieval_sources = _retrieval_post_call_pipelines(
data, user_api_key_dict, llm_router, submitted
)
submitted_pipelines: Final = submitted.pipelines if submitted is not None else ()
submitted_policy_names: Final = frozenset(policy_name for policy_name, _pipeline in submitted_pipelines)
post_call_pipelines: Final = (
*submitted_pipelines,
*(
(policy_name, pipeline)
for policy_name, pipeline in retrieval_pipelines
if policy_name not in submitted_policy_names
),
)
policy_sources: Final = {**retrieval_sources, **(submitted.policy_sources if submitted is not None else {})}
_, bucket = get_or_create_metadata_bucket(data)
already_attached: Final = _POLICY_PIPELINES_ADAPTER.validate_python(bucket.get("_guardrail_pipelines") or ())
attached_policy_names: Final = frozenset(policy_name for policy_name, _pipeline in already_attached)
@ -142,11 +218,14 @@ def attach_post_call_pipelines_to_retrieval(
add_guardrail_to_applied_guardrails_header(request_data=data, guardrail_name=step.guardrail)
add_policy_sources_to_metadata(
request_data=data,
policy_sources={policy_name: policy_sources[policy_name] for policy_name, _pipeline in added},
policy_sources={
policy_name: policy_sources[policy_name]
for policy_name, _pipeline in added
if policy_name in policy_sources
},
)
verbose_proxy_logger.debug(
"Policy engine: attached post_call pipelines to the retrieval of background response %s (model group %s): %s",
data.get("response_id"),
model_group,
"Policy engine: attached post_call pipelines to the retrieval of background response %s: %s",
response_id,
", ".join(policy_name for policy_name, _pipeline in added),
)

View file

@ -259,6 +259,10 @@ from litellm.proxy.policy_engine.pipeline_executor import (
)
from litellm.proxy.policy_engine.policy_registry import get_policy_registry
from litellm.proxy.policy_engine.policy_resolver import PolicyResolver
from litellm.proxy.policy_engine.response_retrieval import (
SubmitPostCallPolicies,
remember_submit_post_call_policies,
)
from litellm.repositories.budget_repository import BudgetRepository
from litellm.repositories.config_repository import ConfigRepository
from litellm.repositories.table_repositories import (
@ -776,9 +780,13 @@ def _post_call_pipelines(data: Mapping[str, object]) -> tuple[tuple[str, "Guardr
_PENDING_BACKGROUND_RESPONSE_STATUSES: Final = frozenset(("queued", "in_progress"))
_POLICY_SOURCES_ADAPTER: Final = TypeAdapter(Mapping[str, str])
def _is_pending_background_response(response: LLMResponseTypes) -> bool:
return isinstance(response, ResponsesAPIResponse) and response.status in _PENDING_BACKGROUND_RESPONSE_STATUSES
def _pending_background_response(response: LLMResponseTypes) -> ResponsesAPIResponse | None:
if isinstance(response, ResponsesAPIResponse) and response.status in _PENDING_BACKGROUND_RESPONSE_STATUSES:
return response
return None
def _guardrails_outside_pipeline(policy_name: str, pipeline: "GuardrailPipeline") -> frozenset[str]:
@ -841,56 +849,25 @@ def _withdraw_deferred_claims(
def _defer_post_call_pipelines(
data: dict[str, object], # mutable-ok: same request-payload shape as post_call_success_hook's data
response: ResponsesAPIResponse,
) -> None:
) -> SubmitPostCallPolicies | None:
deferred: Final = _post_call_pipelines(data)
if not deferred:
return
return None
verbose_proxy_logger.debug(
"Post_call guardrail pipelines wait for background response %s (status=%s) to be retrieved complete: %s",
response.id,
response.status,
", ".join(policy_name for policy_name, _pipeline in deferred),
)
tag_matched: Final = _tag_matched_deferrals(data, deferred)
if tag_matched:
verbose_proxy_logger.warning(
"Policy engine: background response %s matched post_call policies through a request tag at submit; "
"retrieval re-matches only the key, team, and model scopes, so a tag carried in the request body "
"does not govern the completed response: %s",
response.id,
", ".join(tag_matched),
)
body_selected: Final = _body_selected_deferrals(data, deferred)
if body_selected:
verbose_proxy_logger.warning(
"Policy engine: background response %s matched post_call policies through the request body's policies "
"list at submit; retrieval carries no request body, so those policies do not govern the completed "
"response: %s",
response.id,
", ".join(body_selected),
)
_withdraw_deferred_claims(data, deferred)
def _tag_matched_deferrals(
data: Mapping[str, object], deferred: Sequence[tuple[str, "GuardrailPipeline"]]
) -> tuple[str, ...]:
sources: Final = _policy_state_metadata(data).get("policy_sources")
if not isinstance(sources, dict):
return ()
return tuple(
policy_name
for policy_name, _pipeline in deferred
if policy_name in sources and "tag:" in str(sources[policy_name])
sources: Final = _POLICY_SOURCES_ADAPTER.validate_python(_policy_state_metadata(data).get("policy_sources") or {})
submitted: Final = SubmitPostCallPolicies(
pipelines=deferred,
policy_sources={
policy_name: sources[policy_name] for policy_name, _pipeline in deferred if policy_name in sources
},
)
def _body_selected_deferrals(
data: Mapping[str, object], deferred: Sequence[tuple[str, "GuardrailPipeline"]]
) -> tuple[str, ...]:
sources: Final = _policy_state_metadata(data).get("policy_sources")
attributed: Final = frozenset(sources) if isinstance(sources, dict) else frozenset()
return tuple(policy_name for policy_name, _pipeline in deferred if policy_name not in attributed)
_withdraw_deferred_claims(data, deferred)
return submitted
def _pipeline_unsupported_streaming_guardrails(
@ -4096,8 +4073,11 @@ class ProxyLogging:
user_api_key_dict: UserAPIKeyAuth,
response: LLMResponseTypes,
) -> LLMResponseTypes | None:
if _is_pending_background_response(response):
_defer_post_call_pipelines(data, response)
pending: Final = _pending_background_response(response)
if pending is not None:
submitted: Final = _defer_post_call_pipelines(data, pending)
if submitted is not None:
await remember_submit_post_call_policies(self.internal_usage_cache, pending.id, submitted)
return None
_, pipeline_response = await self._maybe_execute_pipelines(
data=data,

View file

@ -3,11 +3,18 @@ from collections.abc import Iterator, Mapping
import pytest
from litellm.caching.dual_cache import DualCache
from litellm.proxy.utils import InternalUsageCache
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.policy_engine.attachment_registry import get_attachment_registry
from litellm.proxy.policy_engine.policy_registry import get_policy_registry
from litellm.proxy.policy_engine.response_retrieval import attach_post_call_pipelines_to_retrieval
from litellm.proxy.policy_engine.response_retrieval import (
SubmitPostCallPolicies,
attach_post_call_pipelines_to_retrieval,
remember_submit_post_call_policies,
)
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline, PipelineStep
from litellm.types.router import Deployment, LiteLLM_Params
GOVERNED_MODEL_GROUP = "gpt-5.4-mini"
@ -46,9 +53,9 @@ def _router(model_group_alias: dict[str, object] | None = None) -> FakeRouter:
)
def _encoded_response_id(model_id: str) -> str:
def _encoded_response_id(model_id: str, upstream_response_id: str = "resp_upstream") -> str:
return ResponsesAPIRequestUtils._build_responses_api_response_id(
custom_llm_provider="openai", model_id=model_id, response_id="resp_upstream"
custom_llm_provider="openai", model_id=model_id, response_id=upstream_response_id
)
@ -97,10 +104,10 @@ def _attached_pipelines(data: Mapping[str, object]) -> tuple[tuple[str, str], ..
)
def test_attaches_model_scoped_post_call_pipeline_to_retrieval(policy_engine: None) -> None:
async def test_attaches_model_scoped_post_call_pipeline_to_retrieval(policy_engine: None) -> None:
data = _retrieval_data(GOVERNED_MODEL_ID)
attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router())
await attach_post_call_pipelines_to_retrieval(submit_policies_cache=InternalUsageCache(DualCache()), data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router())
assert _attached_pipelines(data) == (("response-governance", "output-word-filter"),)
assert data["litellm_metadata"]["_pipeline_managed_guardrails"] == frozenset({"output-word-filter"})
@ -111,50 +118,53 @@ def test_attaches_model_scoped_post_call_pipeline_to_retrieval(policy_engine: No
assert "guardrails" not in data["litellm_metadata"]
def test_key_and_team_context_also_governs_retrieval(policy_engine: None) -> None:
async def test_key_and_team_context_also_governs_retrieval(policy_engine: None) -> None:
data = _retrieval_data(UNGOVERNED_MODEL_ID)
attach_post_call_pipelines_to_retrieval(
data=data, user_api_key_dict=UserAPIKeyAuth(team_alias="governed-team"), llm_router=_router()
await attach_post_call_pipelines_to_retrieval(
data=data,
user_api_key_dict=UserAPIKeyAuth(team_alias="governed-team"),
llm_router=_router(),
submit_policies_cache=InternalUsageCache(DualCache()),
)
assert _attached_pipelines(data) == (("team-governance", "team-word-filter"),)
def test_tag_attached_policy_is_not_re_matched_when_the_retrieval_carries_no_tag(policy_engine: None) -> None:
async def test_tag_attached_policy_is_not_re_matched_when_the_retrieval_carries_no_tag(policy_engine: None) -> None:
data = _retrieval_data(GOVERNED_MODEL_ID)
attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router())
await attach_post_call_pipelines_to_retrieval(submit_policies_cache=InternalUsageCache(DualCache()), data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router())
assert _attached_pipelines(data) == (("response-governance", "output-word-filter"),)
def test_tag_attached_policy_governs_a_retrieval_whose_metadata_carries_the_tag(policy_engine: None) -> None:
async def test_tag_attached_policy_governs_a_retrieval_whose_metadata_carries_the_tag(policy_engine: None) -> None:
data: dict[str, object] = {
"response_id": _encoded_response_id(UNGOVERNED_MODEL_ID),
"litellm_metadata": {"tags": ["governed"]},
}
attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router())
await attach_post_call_pipelines_to_retrieval(submit_policies_cache=InternalUsageCache(DualCache()), data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router())
assert _attached_pipelines(data) == (("tag-governance", "tag-word-filter"),)
assert data["litellm_metadata"]["policy_sources"] == {"tag-governance": "tag:governed"}
def test_retrieval_of_an_ungoverned_model_attaches_nothing(policy_engine: None) -> None:
async def test_retrieval_of_an_ungoverned_model_attaches_nothing(policy_engine: None) -> None:
data = _retrieval_data(UNGOVERNED_MODEL_ID)
attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router())
await attach_post_call_pipelines_to_retrieval(submit_policies_cache=InternalUsageCache(DualCache()), data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router())
assert data == _retrieval_data(UNGOVERNED_MODEL_ID)
def test_already_attached_policy_is_not_attached_twice(policy_engine: None) -> None:
async def test_already_attached_policy_is_not_attached_twice(policy_engine: None) -> None:
data = _retrieval_data(GOVERNED_MODEL_ID)
router = _router()
attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=router)
await attach_post_call_pipelines_to_retrieval(submit_policies_cache=InternalUsageCache(DualCache()), data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=router)
attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=router)
await attach_post_call_pipelines_to_retrieval(submit_policies_cache=InternalUsageCache(DualCache()), data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=router)
assert _attached_pipelines(data) == (("response-governance", "output-word-filter"),)
assert data["litellm_metadata"]["applied_policies"] == ["response-governance"]
@ -168,13 +178,13 @@ def _hidden_submit_model_warnings(caplog: pytest.LogCaptureFixture) -> list[str]
]
def test_wildcard_deployment_attaches_nothing_for_the_submitted_model_and_warns(
async def test_wildcard_deployment_attaches_nothing_for_the_submitted_model_and_warns(
policy_engine: None, caplog: pytest.LogCaptureFixture
) -> None:
data = _retrieval_data(WILDCARD_MODEL_ID)
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router())
await attach_post_call_pipelines_to_retrieval(submit_policies_cache=InternalUsageCache(DualCache()), data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router())
assert data == _retrieval_data(WILDCARD_MODEL_ID)
assert [
@ -183,14 +193,14 @@ def test_wildcard_deployment_attaches_nothing_for_the_submitted_model_and_warns(
] == [True]
def test_aliased_model_group_still_attaches_its_own_policies_and_warns(
async def test_aliased_model_group_still_attaches_its_own_policies_and_warns(
policy_engine: None, caplog: pytest.LogCaptureFixture
) -> None:
data = _retrieval_data(GOVERNED_MODEL_ID)
router = _router({"gpt-mini": GOVERNED_MODEL_GROUP, "gpt-hidden": {"model": GOVERNED_MODEL_GROUP, "hidden": True}})
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=router)
await attach_post_call_pipelines_to_retrieval(submit_policies_cache=InternalUsageCache(DualCache()), data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=router)
assert _attached_pipelines(data) == (("response-governance", "output-word-filter"),)
assert [
@ -199,11 +209,12 @@ def test_aliased_model_group_still_attaches_its_own_policies_and_warns(
] == [True]
def test_plain_model_group_retrieval_does_not_warn_about_the_submitted_model(
async def test_plain_model_group_retrieval_does_not_warn_about_the_submitted_model(
policy_engine: None, caplog: pytest.LogCaptureFixture
) -> None:
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
attach_post_call_pipelines_to_retrieval(
await attach_post_call_pipelines_to_retrieval(
submit_policies_cache=InternalUsageCache(DualCache()),
data=_retrieval_data(GOVERNED_MODEL_ID),
user_api_key_dict=UserAPIKeyAuth(),
llm_router=_router({"other-alias": UNGOVERNED_MODEL_GROUP}),
@ -229,34 +240,144 @@ def _ungoverned_retrieval_warnings(caplog: pytest.LogCaptureFixture) -> list[str
(None, "response id names no deployment"),
],
)
def test_unresolvable_response_id_attaches_nothing_and_warns(
async def test_unresolvable_response_id_attaches_nothing_and_warns(
policy_engine: None, caplog: pytest.LogCaptureFixture, response_id: str, reason: str
) -> None:
data = {"response_id": response_id, "litellm_metadata": {}}
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router())
await attach_post_call_pipelines_to_retrieval(submit_policies_cache=InternalUsageCache(DualCache()), data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router())
assert data == {"response_id": response_id, "litellm_metadata": {}}
assert [message.endswith(f"({reason})") for message in _ungoverned_retrieval_warnings(caplog)] == [True]
def test_without_a_router_attaches_nothing_and_warns(policy_engine: None, caplog: pytest.LogCaptureFixture) -> None:
async def test_without_a_router_attaches_nothing_and_warns(policy_engine: None, caplog: pytest.LogCaptureFixture) -> None:
data = _retrieval_data(GOVERNED_MODEL_ID)
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=None)
await attach_post_call_pipelines_to_retrieval(submit_policies_cache=InternalUsageCache(DualCache()), data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=None)
assert data == _retrieval_data(GOVERNED_MODEL_ID)
assert [message.endswith("(no router)") for message in _ungoverned_retrieval_warnings(caplog)] == [True]
def test_without_policy_engine_attaches_nothing_quietly(caplog: pytest.LogCaptureFixture) -> None:
async def test_without_policy_engine_attaches_nothing_quietly(caplog: pytest.LogCaptureFixture) -> None:
get_policy_registry().clear()
data = _retrieval_data(GOVERNED_MODEL_ID)
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=None)
await attach_post_call_pipelines_to_retrieval(submit_policies_cache=InternalUsageCache(DualCache()), data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=None)
assert data == _retrieval_data(GOVERNED_MODEL_ID)
assert _ungoverned_retrieval_warnings(caplog) == []
def _submitted(*policies: tuple[str, str], policy_sources: Mapping[str, str] | None = None) -> SubmitPostCallPolicies:
return SubmitPostCallPolicies(
pipelines=tuple(
(
policy_name,
GuardrailPipeline(
mode="post_call", steps=[PipelineStep(guardrail=guardrail, on_pass="allow", on_fail="block")]
),
)
for policy_name, guardrail in policies
),
policy_sources=policy_sources or {},
)
async def test_retrieval_runs_the_post_call_policies_the_submit_stored_alongside_its_own_match(
policy_engine: None,
) -> None:
cache = InternalUsageCache(DualCache())
response_id = _encoded_response_id(GOVERNED_MODEL_ID)
await remember_submit_post_call_policies(
cache,
response_id,
_submitted(
("tag-governance", "tag-word-filter"),
("body-governance", "body-word-filter"),
("response-governance", "output-word-filter"),
policy_sources={"tag-governance": "tag:governed", "response-governance": "model:gpt-5.4-mini"},
),
)
data = _retrieval_data(GOVERNED_MODEL_ID)
await attach_post_call_pipelines_to_retrieval(
data=data,
user_api_key_dict=UserAPIKeyAuth(team_alias="governed-team"),
llm_router=_router(),
submit_policies_cache=cache,
)
assert _attached_pipelines(data) == (
("tag-governance", "tag-word-filter"),
("body-governance", "body-word-filter"),
("response-governance", "output-word-filter"),
("team-governance", "team-word-filter"),
)
assert data["litellm_metadata"]["applied_policies"] == [
"tag-governance",
"body-governance",
"response-governance",
"team-governance",
]
assert data["litellm_metadata"]["policy_sources"] == {
"tag-governance": "tag:governed",
"response-governance": "model:gpt-5.4-mini",
"team-governance": "team:governed-team",
}
async def test_submit_stored_policies_govern_a_retrieval_whose_deployment_left_the_router(
policy_engine: None, caplog: pytest.LogCaptureFixture
) -> None:
cache = InternalUsageCache(DualCache())
response_id = _encoded_response_id("deployment-missing-from-router")
await remember_submit_post_call_policies(
cache, response_id, _submitted(("wildcard-governance", "output-word-filter"))
)
data: dict[str, object] = {"response_id": response_id, "litellm_metadata": {}}
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
await attach_post_call_pipelines_to_retrieval(
data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router(), submit_policies_cache=cache
)
assert _attached_pipelines(data) == (("wildcard-governance", "output-word-filter"),)
assert "policy_sources" not in data["litellm_metadata"]
assert _ungoverned_retrieval_warnings(caplog) == []
async def test_submit_stored_policies_belong_to_their_own_response_id(policy_engine: None) -> None:
cache = InternalUsageCache(DualCache())
await remember_submit_post_call_policies(
cache,
_encoded_response_id(GOVERNED_MODEL_ID, upstream_response_id="resp_other_upstream"),
_submitted(("tag-governance", "tag-word-filter")),
)
data = _retrieval_data(GOVERNED_MODEL_ID)
await attach_post_call_pipelines_to_retrieval(
data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router(), submit_policies_cache=cache
)
assert _attached_pipelines(data) == (("response-governance", "output-word-filter"),)
async def test_submit_stored_policies_govern_a_poll_that_re_encodes_the_same_upstream_response(
policy_engine: None,
) -> None:
cache = InternalUsageCache(DualCache())
await remember_submit_post_call_policies(
cache, _encoded_response_id(GOVERNED_MODEL_ID), _submitted(("tag-governance", "tag-word-filter"))
)
data = _retrieval_data(UNGOVERNED_MODEL_ID)
await attach_post_call_pipelines_to_retrieval(
data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router(), submit_policies_cache=cache
)
assert _attached_pipelines(data) == (("tag-governance", "tag-word-filter"),)

View file

@ -76,7 +76,7 @@ from litellm.proxy.hooks.model_max_budget_limiter import (
model_budget_spend_cache_key,
team_member_budget_entity_id,
)
from litellm.proxy.utils import ProxyLogging
from litellm.proxy.utils import InternalUsageCache, ProxyLogging
from litellm.router import Router
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.utils import ModelResponseListIterator
@ -11816,7 +11816,12 @@ class TestBackgroundResponseRetrievalGovernance:
)
return router
async def _pre_call(self, route_type: str, monkeypatch: pytest.MonkeyPatch) -> dict[str, object]:
async def _pre_call(
self,
route_type: str,
monkeypatch: pytest.MonkeyPatch,
internal_usage_cache: InternalUsageCache | None = None,
) -> dict[str, object]:
from litellm.responses.utils import ResponsesAPIRequestUtils
client_facing_response_id = "resp_opaque-client-facing-id"
@ -11851,6 +11856,7 @@ class TestBackgroundResponseRetrievalGovernance:
)
proxy_logging_obj = MagicMock(spec=ProxyLogging)
proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=decrypting_pre_call_hook)
proxy_logging_obj.internal_usage_cache = internal_usage_cache or InternalUsageCache(DualCache())
proxy_config = MagicMock(spec=ProxyConfig)
proxy_config._get_hierarchical_router_settings = AsyncMock(return_value=None)
returned_data, _ = await processing_obj.common_processing_pre_call_logic(
@ -11878,6 +11884,47 @@ class TestBackgroundResponseRetrievalGovernance:
assert data["litellm_metadata"]["applied_policies"] == ["response-governance"]
assert data["model"] is None
@pytest.mark.asyncio
async def test_retrieving_a_background_response_runs_the_post_call_policy_its_submit_matched(
self, policy_engine: None, monkeypatch: pytest.MonkeyPatch
) -> None:
from litellm.proxy.policy_engine.response_retrieval import (
SubmitPostCallPolicies,
remember_submit_post_call_policies,
)
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline, PipelineStep
cache = InternalUsageCache(DualCache())
submit_response_id = ResponsesAPIRequestUtils._build_responses_api_response_id(
custom_llm_provider="openai", model_id="deployment-submitted", response_id="resp_upstream"
)
await remember_submit_post_call_policies(
cache,
submit_response_id,
SubmitPostCallPolicies(
pipelines=(
(
"tag-governance",
GuardrailPipeline(
mode="post_call",
steps=[PipelineStep(guardrail="pii-filter", on_pass="allow", on_fail="block")],
),
),
),
policy_sources={"tag-governance": "tag:governed"},
),
)
data = await self._pre_call("aget_responses", monkeypatch, internal_usage_cache=cache)
pipelines = data["litellm_metadata"]["_guardrail_pipelines"]
assert [(policy_name, [step.guardrail for step in pipeline.steps]) for policy_name, pipeline in pipelines] == [
("tag-governance", ["pii-filter"]),
("response-governance", ["output-word-filter"]),
]
assert data["litellm_metadata"]["applied_policies"] == ["tag-governance", "response-governance"]
@pytest.mark.asyncio
async def test_submitting_a_response_does_not_attach_pipelines_from_its_response_id(
self, policy_engine: None, monkeypatch: pytest.MonkeyPatch

View file

@ -33,6 +33,7 @@ from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.common_utils.callback_utils import add_guardrail_to_applied_guardrails_header
from litellm.llms.openai.chat.guardrail_translation.handler import OpenAIChatCompletionsHandler
from litellm.proxy.policy_engine.pipeline_executor import recorded_guardrail_information
from litellm.proxy.policy_engine.response_retrieval import attach_post_call_pipelines_to_retrieval
from litellm.proxy.utils import ProxyLogging, _streamable_post_call_pipelines, stream_gated_guardrail_names
from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ContentFilterGuardrail
from litellm.types.guardrails import BlockedWord, ContentFilterAction, GuardrailEventHooks
@ -1582,76 +1583,41 @@ async def test_pending_background_response_withdraws_the_deferred_policy_claims(
@pytest.mark.asyncio
async def test_pending_background_response_warns_when_the_deferred_policy_was_matched_through_a_tag(
@pytest.mark.parametrize(
("policy_name", "policy_source"),
[("tag-governance", "tag:governed"), ("alias-governance", "model:gpt-mini-alias"), ("body-governance", None)],
)
async def test_pending_background_response_post_call_policy_governs_its_retrieval_by_another_key(
proxy_logging: ProxyLogging,
make_user_api_key_auth: Callable[..., UserAPIKeyAuth],
monkeypatch: pytest.MonkeyPatch,
clear_policy_registry: None,
caplog: pytest.LogCaptureFixture,
policy_name: str,
policy_source: str | None,
) -> None:
monkeypatch.setattr(litellm, "callbacks", _output_blocking_callbacks({}))
monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False)
data = _claimed_post_call_pipeline_data("response-governance", policy_source="tag:governed+model:m")
await proxy_logging.post_call_success_hook(
data=_claimed_post_call_pipeline_data(policy_name, policy_source=policy_source),
response=_background_response("queued"),
user_api_key_dict=make_user_api_key_auth(key_alias="governed-key"),
)
retrieval: dict[str, object] = {"response_id": "resp_bg", "litellm_metadata": {}}
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
out = await proxy_logging.post_call_success_hook(
data=data, response=_background_response("queued"), user_api_key_dict=make_user_api_key_auth()
)
await attach_post_call_pipelines_to_retrieval(
data=retrieval,
user_api_key_dict=make_user_api_key_auth(api_key="sk-other", key_alias="other-key"),
llm_router=None,
submit_policies_cache=proxy_logging.internal_usage_cache,
)
assert out.status == "queued"
assert "policy_sources" not in data["metadata"]
assert [message for message in _warnings(caplog) if "through a request tag" in message] == [
"Policy engine: background response resp_bg matched post_call policies through a request tag at submit; "
"retrieval re-matches only the key, team, and model scopes, so a tag carried in the request body "
"does not govern the completed response: response-governance"
bucket = retrieval["litellm_metadata"]
assert isinstance(bucket, dict)
assert [(name, [step.guardrail for step in pipeline.steps]) for name, pipeline in bucket["_guardrail_pipelines"]] == [
(policy_name, ["gr-post"])
]
@pytest.mark.asyncio
async def test_pending_background_response_warns_when_the_deferred_policy_came_from_the_request_body(
proxy_logging: ProxyLogging,
make_user_api_key_auth: Callable[..., UserAPIKeyAuth],
monkeypatch: pytest.MonkeyPatch,
clear_policy_registry: None,
caplog: pytest.LogCaptureFixture,
) -> None:
monkeypatch.setattr(litellm, "callbacks", _output_blocking_callbacks({}))
monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False)
data = _claimed_post_call_pipeline_data("body-governance", policy_source=None)
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
out = await proxy_logging.post_call_success_hook(
data=data, response=_background_response("queued"), user_api_key_dict=make_user_api_key_auth()
)
assert out.status == "queued"
assert "policy_sources" not in data["metadata"]
assert _warnings(caplog) == [
"Policy engine: background response resp_bg matched post_call policies through the request body's policies "
"list at submit; retrieval carries no request body, so those policies do not govern the completed "
"response: body-governance"
]
@pytest.mark.asyncio
async def test_pending_background_response_matched_through_its_model_does_not_warn(
proxy_logging: ProxyLogging,
make_user_api_key_auth: Callable[..., UserAPIKeyAuth],
monkeypatch: pytest.MonkeyPatch,
clear_policy_registry: None,
caplog: pytest.LogCaptureFixture,
) -> None:
monkeypatch.setattr(litellm, "callbacks", _output_blocking_callbacks({}))
monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False)
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
await proxy_logging.post_call_success_hook(
data=_claimed_post_call_pipeline_data("response-governance"),
response=_background_response("queued"),
user_api_key_dict=make_user_api_key_auth(),
)
assert _warnings(caplog) == []
assert bucket["applied_policies"] == [policy_name]
assert bucket.get("policy_sources") == ({policy_name: policy_source} if policy_source is not None else None)
@pytest.mark.asyncio

View file

@ -505,11 +505,6 @@ async def test_init_responses_api_endpoints():
# Just mock the _ageneric_api_call_with_fallbacks method
router._ageneric_api_call_with_fallbacks = AsyncMock()
# Add a mock implementation of _get_model_id_from_response_id to the Router instance
ResponsesAPIRequestUtils.get_model_id_from_response_id = MagicMock(
return_value=None
)
# Call without a response_id (no model extraction should happen)
await router._init_responses_api_endpoints(
original_function=AsyncMock(), thread_id="thread_xyz"
@ -523,20 +518,19 @@ async def test_init_responses_api_endpoints():
# Reset the mock
router._ageneric_api_call_with_fallbacks.reset_mock()
# Change the return value for the second call
ResponsesAPIRequestUtils.get_model_id_from_response_id.return_value = (
"claude-3-sonnet"
response_id: Final = ResponsesAPIRequestUtils._build_responses_api_response_id(
custom_llm_provider="anthropic", model_id="claude-3-sonnet", response_id="resp_claude_123"
)
# Call with a response_id
await router._init_responses_api_endpoints(
original_function=AsyncMock(), response_id="resp_claude_123"
original_function=AsyncMock(), response_id=response_id
)
# Verify model was updated in the kwargs
second_call_kwargs = router._ageneric_api_call_with_fallbacks.call_args.kwargs
assert second_call_kwargs["model"] == "claude-3-sonnet"
assert second_call_kwargs["response_id"] == "resp_claude_123"
assert second_call_kwargs["response_id"] == response_id
@pytest.mark.asyncio