From 1406342672cafdf1d5a91193cf1bf8aad7623a86 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 01:40:59 -0700 Subject: [PATCH] fix(proxy): run submit-time post_call policies when a background response is polled A background POST /v1/responses defers its post_call policy pipelines, and the poll that reads the finished output re-matched policies from its own request only. Policies matched at submit by body tags, the body policies list, a wildcard or aliased model name, or the submitting key never ran. The submit now stores its deferred pipelines in the proxy's shared cache, keyed by the upstream response id, and every poll runs them next to its own re-match. Also stops a router test from leaving ResponsesAPIRequestUtils.get_model_id_from_response_id mocked for the rest of its xdist worker. --- litellm/proxy/common_request_processing.py | 3 +- .../proxy/policy_engine/response_retrieval.py | 131 ++++++++++--- litellm/proxy/utils.py | 70 +++---- .../policy_engine/test_response_retrieval.py | 179 +++++++++++++++--- .../proxy/test_common_request_processing.py | 51 ++++- .../proxy_logging/test_guardrail_pipeline.py | 86 +++------ .../unit/test_router/test_router_endpoints.py | 14 +- 7 files changed, 361 insertions(+), 173 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 9bb451495c8..991e0f2c668 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -2571,10 +2571,11 @@ class ProxyBaseLLMRequestProcessing: proxy_logging_obj=proxy_logging_obj, ) if route_type == "aget_responses": - attach_post_call_pipelines_to_retrieval( + await attach_post_call_pipelines_to_retrieval( data=self.data, user_api_key_dict=user_api_key_dict, llm_router=llm_router, + submit_policies_cache=proxy_logging_obj.internal_usage_cache, ) # Refresh AFTER pre_call_hook: guardrails (e.g. Presidio PII masking) may diff --git a/litellm/proxy/policy_engine/response_retrieval.py b/litellm/proxy/policy_engine/response_retrieval.py index 654b1682582..dced80a0772 100644 --- a/litellm/proxy/policy_engine/response_retrieval.py +++ b/litellm/proxy/policy_engine/response_retrieval.py @@ -1,11 +1,12 @@ from collections.abc import Mapping from dataclasses import dataclass from types import MappingProxyType -from typing import TYPE_CHECKING, Final, Literal, TypeAlias +from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias -from pydantic import TypeAdapter +from pydantic import BaseModel, ConfigDict, TypeAdapter from litellm._logging import verbose_proxy_logger +from litellm.constants import MANAGED_OBJECT_STALENESS_CUTOFF_DAYS from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket from litellm.proxy.common_utils.callback_utils import ( add_guardrail_to_applied_guardrails_header, @@ -30,6 +31,46 @@ PolicyPipelines: TypeAlias = tuple[tuple[str, GuardrailPipeline], ...] _POLICY_PIPELINES_ADAPTER: Final = TypeAdapter(PolicyPipelines) +_SUBMIT_POLICIES_TTL_SECONDS: Final = MANAGED_OBJECT_STALENESS_CUTOFF_DAYS * 24 * 60 * 60 + + +class SubmitPostCallPolicies(BaseModel): + model_config = ConfigDict(frozen=True) + + pipelines: PolicyPipelines + policy_sources: Mapping[str, str] + + +class SubmitPoliciesCache(Protocol): + async def async_set_cache(self, key: str, value: object, litellm_parent_otel_span: None, *, ttl: int) -> None: ... + + async def async_get_cache(self, key: str, litellm_parent_otel_span: None) -> object: ... + + +def _submit_policies_cache_key(response_id: str) -> str: + upstream_response_id: Final = ResponsesAPIRequestUtils.decode_responses_api_response_id(response_id).get( + "response_id", response_id + ) + return f"policy_engine:background_response_post_call_policies:{upstream_response_id}" + + +async def remember_submit_post_call_policies( + cache: SubmitPoliciesCache, response_id: str, submitted: SubmitPostCallPolicies +) -> None: + await cache.async_set_cache( + key=_submit_policies_cache_key(response_id), + value=submitted.model_dump(mode="json"), + litellm_parent_otel_span=None, + ttl=_SUBMIT_POLICIES_TTL_SECONDS, + ) + + +async def _submit_post_call_policies(cache: SubmitPoliciesCache, response_id: str) -> SubmitPostCallPolicies | None: + stored: Final = await cache.async_get_cache( + key=_submit_policies_cache_key(response_id), litellm_parent_otel_span=None + ) + return None if stored is None else SubmitPostCallPolicies.model_validate(stored) + @dataclass(frozen=True, slots=True) class UngovernedRetrieval: @@ -47,18 +88,23 @@ def _model_group_for_response_id(response_id: object, llm_router: "Router | None deployment: Final = llm_router.get_deployment(model_id) if deployment is None: return UngovernedRetrieval("deployment no longer in the router") - hidden_by: Final = _submit_model_hidden_by(deployment.model_name, llm_router.model_group_alias) - if hidden_by is not None: - verbose_proxy_logger.warning( - "Policy engine: background response %s re-matches policies on retrieval as model group %s (%s), " - "so a policy attached to the model name it was submitted as does not run on it", - response_id, - deployment.model_name, - hidden_by, - ) return deployment.model_name +def _warn_if_submit_model_hidden(response_id: object, model_group: str, llm_router: "Router") -> None: + hidden_by: Final = _submit_model_hidden_by(model_group, llm_router.model_group_alias) + if hidden_by is None: + return + verbose_proxy_logger.warning( + "Policy engine: background response %s has no submit-time post_call policies stored and re-matches " + "policies on retrieval as model group %s (%s), so a policy attached to the model name it was submitted as " + "does not run on it", + response_id, + model_group, + hidden_by, + ) + + def _submit_model_hidden_by(model_group: str, model_group_alias: Mapping[str, object]) -> str | None: if "*" in model_group: return "a wildcard deployment" @@ -103,23 +149,53 @@ def _post_call_pipelines_for_context(context: PolicyMatchContext) -> tuple[Polic return post_call_pipelines, MappingProxyType({match["policy_name"]: match["matched_via"] for match in matches}) -def attach_post_call_pipelines_to_retrieval( +def _retrieval_post_call_pipelines( + data: Mapping[str, object], + user_api_key_dict: "UserAPIKeyAuth", + llm_router: "Router | None", + submitted: SubmitPostCallPolicies | None, +) -> tuple[PolicyPipelines, Mapping[str, str]]: + response_id: Final = data.get("response_id") + model_group: Final = _model_group_for_response_id(response_id, llm_router) + if isinstance(model_group, UngovernedRetrieval): + if submitted is None: + verbose_proxy_logger.warning( + "Policy engine: background response %s is retrieved without its post_call policy pipelines (%s)", + response_id, + model_group.reason, + ) + return (), MappingProxyType({}) + if submitted is None and llm_router is not None: + _warn_if_submit_model_hidden(response_id, model_group, llm_router) + return _post_call_pipelines_for_context(_retrieval_context(data, user_api_key_dict, model_group)) + + +async def attach_post_call_pipelines_to_retrieval( data: dict[str, object], # mutable-ok: request-state dict the policy engine hooks all write in place user_api_key_dict: "UserAPIKeyAuth", llm_router: "Router | None", + submit_policies_cache: SubmitPoliciesCache, ) -> None: if not get_policy_registry().is_initialized(): return - model_group: Final = _model_group_for_response_id(data.get("response_id"), llm_router) - if isinstance(model_group, UngovernedRetrieval): - verbose_proxy_logger.warning( - "Policy engine: background response %s is retrieved without its post_call policy pipelines (%s)", - data.get("response_id"), - model_group.reason, - ) - return - context: Final = _retrieval_context(data, user_api_key_dict, model_group) - post_call_pipelines, policy_sources = _post_call_pipelines_for_context(context) + response_id: Final = data.get("response_id") + submitted: Final = ( + await _submit_post_call_policies(submit_policies_cache, response_id) if isinstance(response_id, str) else None + ) + retrieval_pipelines, retrieval_sources = _retrieval_post_call_pipelines( + data, user_api_key_dict, llm_router, submitted + ) + submitted_pipelines: Final = submitted.pipelines if submitted is not None else () + submitted_policy_names: Final = frozenset(policy_name for policy_name, _pipeline in submitted_pipelines) + post_call_pipelines: Final = ( + *submitted_pipelines, + *( + (policy_name, pipeline) + for policy_name, pipeline in retrieval_pipelines + if policy_name not in submitted_policy_names + ), + ) + policy_sources: Final = {**retrieval_sources, **(submitted.policy_sources if submitted is not None else {})} _, bucket = get_or_create_metadata_bucket(data) already_attached: Final = _POLICY_PIPELINES_ADAPTER.validate_python(bucket.get("_guardrail_pipelines") or ()) attached_policy_names: Final = frozenset(policy_name for policy_name, _pipeline in already_attached) @@ -142,11 +218,14 @@ def attach_post_call_pipelines_to_retrieval( add_guardrail_to_applied_guardrails_header(request_data=data, guardrail_name=step.guardrail) add_policy_sources_to_metadata( request_data=data, - policy_sources={policy_name: policy_sources[policy_name] for policy_name, _pipeline in added}, + policy_sources={ + policy_name: policy_sources[policy_name] + for policy_name, _pipeline in added + if policy_name in policy_sources + }, ) verbose_proxy_logger.debug( - "Policy engine: attached post_call pipelines to the retrieval of background response %s (model group %s): %s", - data.get("response_id"), - model_group, + "Policy engine: attached post_call pipelines to the retrieval of background response %s: %s", + response_id, ", ".join(policy_name for policy_name, _pipeline in added), ) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 0ddfc5ce62f..abc0527eb8b 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -259,6 +259,10 @@ from litellm.proxy.policy_engine.pipeline_executor import ( ) from litellm.proxy.policy_engine.policy_registry import get_policy_registry from litellm.proxy.policy_engine.policy_resolver import PolicyResolver +from litellm.proxy.policy_engine.response_retrieval import ( + SubmitPostCallPolicies, + remember_submit_post_call_policies, +) from litellm.repositories.budget_repository import BudgetRepository from litellm.repositories.config_repository import ConfigRepository from litellm.repositories.table_repositories import ( @@ -776,9 +780,13 @@ def _post_call_pipelines(data: Mapping[str, object]) -> tuple[tuple[str, "Guardr _PENDING_BACKGROUND_RESPONSE_STATUSES: Final = frozenset(("queued", "in_progress")) +_POLICY_SOURCES_ADAPTER: Final = TypeAdapter(Mapping[str, str]) -def _is_pending_background_response(response: LLMResponseTypes) -> bool: - return isinstance(response, ResponsesAPIResponse) and response.status in _PENDING_BACKGROUND_RESPONSE_STATUSES + +def _pending_background_response(response: LLMResponseTypes) -> ResponsesAPIResponse | None: + if isinstance(response, ResponsesAPIResponse) and response.status in _PENDING_BACKGROUND_RESPONSE_STATUSES: + return response + return None def _guardrails_outside_pipeline(policy_name: str, pipeline: "GuardrailPipeline") -> frozenset[str]: @@ -841,56 +849,25 @@ def _withdraw_deferred_claims( def _defer_post_call_pipelines( data: dict[str, object], # mutable-ok: same request-payload shape as post_call_success_hook's data response: ResponsesAPIResponse, -) -> None: +) -> SubmitPostCallPolicies | None: deferred: Final = _post_call_pipelines(data) if not deferred: - return + return None verbose_proxy_logger.debug( "Post_call guardrail pipelines wait for background response %s (status=%s) to be retrieved complete: %s", response.id, response.status, ", ".join(policy_name for policy_name, _pipeline in deferred), ) - tag_matched: Final = _tag_matched_deferrals(data, deferred) - if tag_matched: - verbose_proxy_logger.warning( - "Policy engine: background response %s matched post_call policies through a request tag at submit; " - "retrieval re-matches only the key, team, and model scopes, so a tag carried in the request body " - "does not govern the completed response: %s", - response.id, - ", ".join(tag_matched), - ) - body_selected: Final = _body_selected_deferrals(data, deferred) - if body_selected: - verbose_proxy_logger.warning( - "Policy engine: background response %s matched post_call policies through the request body's policies " - "list at submit; retrieval carries no request body, so those policies do not govern the completed " - "response: %s", - response.id, - ", ".join(body_selected), - ) - _withdraw_deferred_claims(data, deferred) - - -def _tag_matched_deferrals( - data: Mapping[str, object], deferred: Sequence[tuple[str, "GuardrailPipeline"]] -) -> tuple[str, ...]: - sources: Final = _policy_state_metadata(data).get("policy_sources") - if not isinstance(sources, dict): - return () - return tuple( - policy_name - for policy_name, _pipeline in deferred - if policy_name in sources and "tag:" in str(sources[policy_name]) + sources: Final = _POLICY_SOURCES_ADAPTER.validate_python(_policy_state_metadata(data).get("policy_sources") or {}) + submitted: Final = SubmitPostCallPolicies( + pipelines=deferred, + policy_sources={ + policy_name: sources[policy_name] for policy_name, _pipeline in deferred if policy_name in sources + }, ) - - -def _body_selected_deferrals( - data: Mapping[str, object], deferred: Sequence[tuple[str, "GuardrailPipeline"]] -) -> tuple[str, ...]: - sources: Final = _policy_state_metadata(data).get("policy_sources") - attributed: Final = frozenset(sources) if isinstance(sources, dict) else frozenset() - return tuple(policy_name for policy_name, _pipeline in deferred if policy_name not in attributed) + _withdraw_deferred_claims(data, deferred) + return submitted def _pipeline_unsupported_streaming_guardrails( @@ -4096,8 +4073,11 @@ class ProxyLogging: user_api_key_dict: UserAPIKeyAuth, response: LLMResponseTypes, ) -> LLMResponseTypes | None: - if _is_pending_background_response(response): - _defer_post_call_pipelines(data, response) + pending: Final = _pending_background_response(response) + if pending is not None: + submitted: Final = _defer_post_call_pipelines(data, pending) + if submitted is not None: + await remember_submit_post_call_policies(self.internal_usage_cache, pending.id, submitted) return None _, pipeline_response = await self._maybe_execute_pipelines( data=data, diff --git a/tests/unit/proxy/policy_engine/test_response_retrieval.py b/tests/unit/proxy/policy_engine/test_response_retrieval.py index 37849101b3a..50e67165bb2 100644 --- a/tests/unit/proxy/policy_engine/test_response_retrieval.py +++ b/tests/unit/proxy/policy_engine/test_response_retrieval.py @@ -3,11 +3,18 @@ from collections.abc import Iterator, Mapping import pytest +from litellm.caching.dual_cache import DualCache +from litellm.proxy.utils import InternalUsageCache from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.policy_engine.attachment_registry import get_attachment_registry from litellm.proxy.policy_engine.policy_registry import get_policy_registry -from litellm.proxy.policy_engine.response_retrieval import attach_post_call_pipelines_to_retrieval +from litellm.proxy.policy_engine.response_retrieval import ( + SubmitPostCallPolicies, + attach_post_call_pipelines_to_retrieval, + remember_submit_post_call_policies, +) from litellm.responses.utils import ResponsesAPIRequestUtils +from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline, PipelineStep from litellm.types.router import Deployment, LiteLLM_Params GOVERNED_MODEL_GROUP = "gpt-5.4-mini" @@ -46,9 +53,9 @@ def _router(model_group_alias: dict[str, object] | None = None) -> FakeRouter: ) -def _encoded_response_id(model_id: str) -> str: +def _encoded_response_id(model_id: str, upstream_response_id: str = "resp_upstream") -> str: return ResponsesAPIRequestUtils._build_responses_api_response_id( - custom_llm_provider="openai", model_id=model_id, response_id="resp_upstream" + custom_llm_provider="openai", model_id=model_id, response_id=upstream_response_id ) @@ -97,10 +104,10 @@ def _attached_pipelines(data: Mapping[str, object]) -> tuple[tuple[str, str], .. ) -def test_attaches_model_scoped_post_call_pipeline_to_retrieval(policy_engine: None) -> None: +async def test_attaches_model_scoped_post_call_pipeline_to_retrieval(policy_engine: None) -> None: data = _retrieval_data(GOVERNED_MODEL_ID) - attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) + await attach_post_call_pipelines_to_retrieval(submit_policies_cache=InternalUsageCache(DualCache()), data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) assert _attached_pipelines(data) == (("response-governance", "output-word-filter"),) assert data["litellm_metadata"]["_pipeline_managed_guardrails"] == frozenset({"output-word-filter"}) @@ -111,50 +118,53 @@ def test_attaches_model_scoped_post_call_pipeline_to_retrieval(policy_engine: No assert "guardrails" not in data["litellm_metadata"] -def test_key_and_team_context_also_governs_retrieval(policy_engine: None) -> None: +async def test_key_and_team_context_also_governs_retrieval(policy_engine: None) -> None: data = _retrieval_data(UNGOVERNED_MODEL_ID) - attach_post_call_pipelines_to_retrieval( - data=data, user_api_key_dict=UserAPIKeyAuth(team_alias="governed-team"), llm_router=_router() + await attach_post_call_pipelines_to_retrieval( + data=data, + user_api_key_dict=UserAPIKeyAuth(team_alias="governed-team"), + llm_router=_router(), + submit_policies_cache=InternalUsageCache(DualCache()), ) assert _attached_pipelines(data) == (("team-governance", "team-word-filter"),) -def test_tag_attached_policy_is_not_re_matched_when_the_retrieval_carries_no_tag(policy_engine: None) -> None: +async def test_tag_attached_policy_is_not_re_matched_when_the_retrieval_carries_no_tag(policy_engine: None) -> None: data = _retrieval_data(GOVERNED_MODEL_ID) - attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) + await attach_post_call_pipelines_to_retrieval(submit_policies_cache=InternalUsageCache(DualCache()), data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) assert _attached_pipelines(data) == (("response-governance", "output-word-filter"),) -def test_tag_attached_policy_governs_a_retrieval_whose_metadata_carries_the_tag(policy_engine: None) -> None: +async def test_tag_attached_policy_governs_a_retrieval_whose_metadata_carries_the_tag(policy_engine: None) -> None: data: dict[str, object] = { "response_id": _encoded_response_id(UNGOVERNED_MODEL_ID), "litellm_metadata": {"tags": ["governed"]}, } - attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) + await attach_post_call_pipelines_to_retrieval(submit_policies_cache=InternalUsageCache(DualCache()), data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) assert _attached_pipelines(data) == (("tag-governance", "tag-word-filter"),) assert data["litellm_metadata"]["policy_sources"] == {"tag-governance": "tag:governed"} -def test_retrieval_of_an_ungoverned_model_attaches_nothing(policy_engine: None) -> None: +async def test_retrieval_of_an_ungoverned_model_attaches_nothing(policy_engine: None) -> None: data = _retrieval_data(UNGOVERNED_MODEL_ID) - attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) + await attach_post_call_pipelines_to_retrieval(submit_policies_cache=InternalUsageCache(DualCache()), data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) assert data == _retrieval_data(UNGOVERNED_MODEL_ID) -def test_already_attached_policy_is_not_attached_twice(policy_engine: None) -> None: +async def test_already_attached_policy_is_not_attached_twice(policy_engine: None) -> None: data = _retrieval_data(GOVERNED_MODEL_ID) router = _router() - attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=router) + await attach_post_call_pipelines_to_retrieval(submit_policies_cache=InternalUsageCache(DualCache()), data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=router) - attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=router) + await attach_post_call_pipelines_to_retrieval(submit_policies_cache=InternalUsageCache(DualCache()), data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=router) assert _attached_pipelines(data) == (("response-governance", "output-word-filter"),) assert data["litellm_metadata"]["applied_policies"] == ["response-governance"] @@ -168,13 +178,13 @@ def _hidden_submit_model_warnings(caplog: pytest.LogCaptureFixture) -> list[str] ] -def test_wildcard_deployment_attaches_nothing_for_the_submitted_model_and_warns( +async def test_wildcard_deployment_attaches_nothing_for_the_submitted_model_and_warns( policy_engine: None, caplog: pytest.LogCaptureFixture ) -> None: data = _retrieval_data(WILDCARD_MODEL_ID) with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): - attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) + await attach_post_call_pipelines_to_retrieval(submit_policies_cache=InternalUsageCache(DualCache()), data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) assert data == _retrieval_data(WILDCARD_MODEL_ID) assert [ @@ -183,14 +193,14 @@ def test_wildcard_deployment_attaches_nothing_for_the_submitted_model_and_warns( ] == [True] -def test_aliased_model_group_still_attaches_its_own_policies_and_warns( +async def test_aliased_model_group_still_attaches_its_own_policies_and_warns( policy_engine: None, caplog: pytest.LogCaptureFixture ) -> None: data = _retrieval_data(GOVERNED_MODEL_ID) router = _router({"gpt-mini": GOVERNED_MODEL_GROUP, "gpt-hidden": {"model": GOVERNED_MODEL_GROUP, "hidden": True}}) with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): - attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=router) + await attach_post_call_pipelines_to_retrieval(submit_policies_cache=InternalUsageCache(DualCache()), data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=router) assert _attached_pipelines(data) == (("response-governance", "output-word-filter"),) assert [ @@ -199,11 +209,12 @@ def test_aliased_model_group_still_attaches_its_own_policies_and_warns( ] == [True] -def test_plain_model_group_retrieval_does_not_warn_about_the_submitted_model( +async def test_plain_model_group_retrieval_does_not_warn_about_the_submitted_model( policy_engine: None, caplog: pytest.LogCaptureFixture ) -> None: with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): - attach_post_call_pipelines_to_retrieval( + await attach_post_call_pipelines_to_retrieval( + submit_policies_cache=InternalUsageCache(DualCache()), data=_retrieval_data(GOVERNED_MODEL_ID), user_api_key_dict=UserAPIKeyAuth(), llm_router=_router({"other-alias": UNGOVERNED_MODEL_GROUP}), @@ -229,34 +240,144 @@ def _ungoverned_retrieval_warnings(caplog: pytest.LogCaptureFixture) -> list[str (None, "response id names no deployment"), ], ) -def test_unresolvable_response_id_attaches_nothing_and_warns( +async def test_unresolvable_response_id_attaches_nothing_and_warns( policy_engine: None, caplog: pytest.LogCaptureFixture, response_id: str, reason: str ) -> None: data = {"response_id": response_id, "litellm_metadata": {}} with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): - attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) + await attach_post_call_pipelines_to_retrieval(submit_policies_cache=InternalUsageCache(DualCache()), data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) assert data == {"response_id": response_id, "litellm_metadata": {}} assert [message.endswith(f"({reason})") for message in _ungoverned_retrieval_warnings(caplog)] == [True] -def test_without_a_router_attaches_nothing_and_warns(policy_engine: None, caplog: pytest.LogCaptureFixture) -> None: +async def test_without_a_router_attaches_nothing_and_warns(policy_engine: None, caplog: pytest.LogCaptureFixture) -> None: data = _retrieval_data(GOVERNED_MODEL_ID) with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): - attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=None) + await attach_post_call_pipelines_to_retrieval(submit_policies_cache=InternalUsageCache(DualCache()), data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=None) assert data == _retrieval_data(GOVERNED_MODEL_ID) assert [message.endswith("(no router)") for message in _ungoverned_retrieval_warnings(caplog)] == [True] -def test_without_policy_engine_attaches_nothing_quietly(caplog: pytest.LogCaptureFixture) -> None: +async def test_without_policy_engine_attaches_nothing_quietly(caplog: pytest.LogCaptureFixture) -> None: get_policy_registry().clear() data = _retrieval_data(GOVERNED_MODEL_ID) with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): - attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=None) + await attach_post_call_pipelines_to_retrieval(submit_policies_cache=InternalUsageCache(DualCache()), data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=None) assert data == _retrieval_data(GOVERNED_MODEL_ID) assert _ungoverned_retrieval_warnings(caplog) == [] + + +def _submitted(*policies: tuple[str, str], policy_sources: Mapping[str, str] | None = None) -> SubmitPostCallPolicies: + return SubmitPostCallPolicies( + pipelines=tuple( + ( + policy_name, + GuardrailPipeline( + mode="post_call", steps=[PipelineStep(guardrail=guardrail, on_pass="allow", on_fail="block")] + ), + ) + for policy_name, guardrail in policies + ), + policy_sources=policy_sources or {}, + ) + + +async def test_retrieval_runs_the_post_call_policies_the_submit_stored_alongside_its_own_match( + policy_engine: None, +) -> None: + cache = InternalUsageCache(DualCache()) + response_id = _encoded_response_id(GOVERNED_MODEL_ID) + await remember_submit_post_call_policies( + cache, + response_id, + _submitted( + ("tag-governance", "tag-word-filter"), + ("body-governance", "body-word-filter"), + ("response-governance", "output-word-filter"), + policy_sources={"tag-governance": "tag:governed", "response-governance": "model:gpt-5.4-mini"}, + ), + ) + data = _retrieval_data(GOVERNED_MODEL_ID) + + await attach_post_call_pipelines_to_retrieval( + data=data, + user_api_key_dict=UserAPIKeyAuth(team_alias="governed-team"), + llm_router=_router(), + submit_policies_cache=cache, + ) + + assert _attached_pipelines(data) == ( + ("tag-governance", "tag-word-filter"), + ("body-governance", "body-word-filter"), + ("response-governance", "output-word-filter"), + ("team-governance", "team-word-filter"), + ) + assert data["litellm_metadata"]["applied_policies"] == [ + "tag-governance", + "body-governance", + "response-governance", + "team-governance", + ] + assert data["litellm_metadata"]["policy_sources"] == { + "tag-governance": "tag:governed", + "response-governance": "model:gpt-5.4-mini", + "team-governance": "team:governed-team", + } + + +async def test_submit_stored_policies_govern_a_retrieval_whose_deployment_left_the_router( + policy_engine: None, caplog: pytest.LogCaptureFixture +) -> None: + cache = InternalUsageCache(DualCache()) + response_id = _encoded_response_id("deployment-missing-from-router") + await remember_submit_post_call_policies( + cache, response_id, _submitted(("wildcard-governance", "output-word-filter")) + ) + data: dict[str, object] = {"response_id": response_id, "litellm_metadata": {}} + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + await attach_post_call_pipelines_to_retrieval( + data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router(), submit_policies_cache=cache + ) + + assert _attached_pipelines(data) == (("wildcard-governance", "output-word-filter"),) + assert "policy_sources" not in data["litellm_metadata"] + assert _ungoverned_retrieval_warnings(caplog) == [] + + +async def test_submit_stored_policies_belong_to_their_own_response_id(policy_engine: None) -> None: + cache = InternalUsageCache(DualCache()) + await remember_submit_post_call_policies( + cache, + _encoded_response_id(GOVERNED_MODEL_ID, upstream_response_id="resp_other_upstream"), + _submitted(("tag-governance", "tag-word-filter")), + ) + data = _retrieval_data(GOVERNED_MODEL_ID) + + await attach_post_call_pipelines_to_retrieval( + data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router(), submit_policies_cache=cache + ) + + assert _attached_pipelines(data) == (("response-governance", "output-word-filter"),) + + +async def test_submit_stored_policies_govern_a_poll_that_re_encodes_the_same_upstream_response( + policy_engine: None, +) -> None: + cache = InternalUsageCache(DualCache()) + await remember_submit_post_call_policies( + cache, _encoded_response_id(GOVERNED_MODEL_ID), _submitted(("tag-governance", "tag-word-filter")) + ) + data = _retrieval_data(UNGOVERNED_MODEL_ID) + + await attach_post_call_pipelines_to_retrieval( + data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router(), submit_policies_cache=cache + ) + + assert _attached_pipelines(data) == (("tag-governance", "tag-word-filter"),) diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index fdcbcbfcd05..f8b5db7cf1a 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -76,7 +76,7 @@ from litellm.proxy.hooks.model_max_budget_limiter import ( model_budget_spend_cache_key, team_member_budget_entity_id, ) -from litellm.proxy.utils import ProxyLogging +from litellm.proxy.utils import InternalUsageCache, ProxyLogging from litellm.router import Router from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.utils import ModelResponseListIterator @@ -11816,7 +11816,12 @@ class TestBackgroundResponseRetrievalGovernance: ) return router - async def _pre_call(self, route_type: str, monkeypatch: pytest.MonkeyPatch) -> dict[str, object]: + async def _pre_call( + self, + route_type: str, + monkeypatch: pytest.MonkeyPatch, + internal_usage_cache: InternalUsageCache | None = None, + ) -> dict[str, object]: from litellm.responses.utils import ResponsesAPIRequestUtils client_facing_response_id = "resp_opaque-client-facing-id" @@ -11851,6 +11856,7 @@ class TestBackgroundResponseRetrievalGovernance: ) proxy_logging_obj = MagicMock(spec=ProxyLogging) proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=decrypting_pre_call_hook) + proxy_logging_obj.internal_usage_cache = internal_usage_cache or InternalUsageCache(DualCache()) proxy_config = MagicMock(spec=ProxyConfig) proxy_config._get_hierarchical_router_settings = AsyncMock(return_value=None) returned_data, _ = await processing_obj.common_processing_pre_call_logic( @@ -11878,6 +11884,47 @@ class TestBackgroundResponseRetrievalGovernance: assert data["litellm_metadata"]["applied_policies"] == ["response-governance"] assert data["model"] is None + @pytest.mark.asyncio + async def test_retrieving_a_background_response_runs_the_post_call_policy_its_submit_matched( + self, policy_engine: None, monkeypatch: pytest.MonkeyPatch + ) -> None: + from litellm.proxy.policy_engine.response_retrieval import ( + SubmitPostCallPolicies, + remember_submit_post_call_policies, + ) + from litellm.responses.utils import ResponsesAPIRequestUtils + from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline, PipelineStep + + cache = InternalUsageCache(DualCache()) + submit_response_id = ResponsesAPIRequestUtils._build_responses_api_response_id( + custom_llm_provider="openai", model_id="deployment-submitted", response_id="resp_upstream" + ) + await remember_submit_post_call_policies( + cache, + submit_response_id, + SubmitPostCallPolicies( + pipelines=( + ( + "tag-governance", + GuardrailPipeline( + mode="post_call", + steps=[PipelineStep(guardrail="pii-filter", on_pass="allow", on_fail="block")], + ), + ), + ), + policy_sources={"tag-governance": "tag:governed"}, + ), + ) + + data = await self._pre_call("aget_responses", monkeypatch, internal_usage_cache=cache) + + pipelines = data["litellm_metadata"]["_guardrail_pipelines"] + assert [(policy_name, [step.guardrail for step in pipeline.steps]) for policy_name, pipeline in pipelines] == [ + ("tag-governance", ["pii-filter"]), + ("response-governance", ["output-word-filter"]), + ] + assert data["litellm_metadata"]["applied_policies"] == ["tag-governance", "response-governance"] + @pytest.mark.asyncio async def test_submitting_a_response_does_not_attach_pipelines_from_its_response_id( self, policy_engine: None, monkeypatch: pytest.MonkeyPatch diff --git a/tests/unit/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/unit/proxy/utils/proxy_logging/test_guardrail_pipeline.py index acc1463b072..1932d82ddda 100644 --- a/tests/unit/proxy/utils/proxy_logging/test_guardrail_pipeline.py +++ b/tests/unit/proxy/utils/proxy_logging/test_guardrail_pipeline.py @@ -33,6 +33,7 @@ from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.callback_utils import add_guardrail_to_applied_guardrails_header from litellm.llms.openai.chat.guardrail_translation.handler import OpenAIChatCompletionsHandler from litellm.proxy.policy_engine.pipeline_executor import recorded_guardrail_information +from litellm.proxy.policy_engine.response_retrieval import attach_post_call_pipelines_to_retrieval from litellm.proxy.utils import ProxyLogging, _streamable_post_call_pipelines, stream_gated_guardrail_names from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ContentFilterGuardrail from litellm.types.guardrails import BlockedWord, ContentFilterAction, GuardrailEventHooks @@ -1582,76 +1583,41 @@ async def test_pending_background_response_withdraws_the_deferred_policy_claims( @pytest.mark.asyncio -async def test_pending_background_response_warns_when_the_deferred_policy_was_matched_through_a_tag( +@pytest.mark.parametrize( + ("policy_name", "policy_source"), + [("tag-governance", "tag:governed"), ("alias-governance", "model:gpt-mini-alias"), ("body-governance", None)], +) +async def test_pending_background_response_post_call_policy_governs_its_retrieval_by_another_key( proxy_logging: ProxyLogging, make_user_api_key_auth: Callable[..., UserAPIKeyAuth], monkeypatch: pytest.MonkeyPatch, clear_policy_registry: None, - caplog: pytest.LogCaptureFixture, + policy_name: str, + policy_source: str | None, ) -> None: monkeypatch.setattr(litellm, "callbacks", _output_blocking_callbacks({})) monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) - data = _claimed_post_call_pipeline_data("response-governance", policy_source="tag:governed+model:m") + await proxy_logging.post_call_success_hook( + data=_claimed_post_call_pipeline_data(policy_name, policy_source=policy_source), + response=_background_response("queued"), + user_api_key_dict=make_user_api_key_auth(key_alias="governed-key"), + ) + retrieval: dict[str, object] = {"response_id": "resp_bg", "litellm_metadata": {}} - with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): - out = await proxy_logging.post_call_success_hook( - data=data, response=_background_response("queued"), user_api_key_dict=make_user_api_key_auth() - ) + await attach_post_call_pipelines_to_retrieval( + data=retrieval, + user_api_key_dict=make_user_api_key_auth(api_key="sk-other", key_alias="other-key"), + llm_router=None, + submit_policies_cache=proxy_logging.internal_usage_cache, + ) - assert out.status == "queued" - assert "policy_sources" not in data["metadata"] - assert [message for message in _warnings(caplog) if "through a request tag" in message] == [ - "Policy engine: background response resp_bg matched post_call policies through a request tag at submit; " - "retrieval re-matches only the key, team, and model scopes, so a tag carried in the request body " - "does not govern the completed response: response-governance" + bucket = retrieval["litellm_metadata"] + assert isinstance(bucket, dict) + assert [(name, [step.guardrail for step in pipeline.steps]) for name, pipeline in bucket["_guardrail_pipelines"]] == [ + (policy_name, ["gr-post"]) ] - - -@pytest.mark.asyncio -async def test_pending_background_response_warns_when_the_deferred_policy_came_from_the_request_body( - proxy_logging: ProxyLogging, - make_user_api_key_auth: Callable[..., UserAPIKeyAuth], - monkeypatch: pytest.MonkeyPatch, - clear_policy_registry: None, - caplog: pytest.LogCaptureFixture, -) -> None: - monkeypatch.setattr(litellm, "callbacks", _output_blocking_callbacks({})) - monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) - data = _claimed_post_call_pipeline_data("body-governance", policy_source=None) - - with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): - out = await proxy_logging.post_call_success_hook( - data=data, response=_background_response("queued"), user_api_key_dict=make_user_api_key_auth() - ) - - assert out.status == "queued" - assert "policy_sources" not in data["metadata"] - assert _warnings(caplog) == [ - "Policy engine: background response resp_bg matched post_call policies through the request body's policies " - "list at submit; retrieval carries no request body, so those policies do not govern the completed " - "response: body-governance" - ] - - -@pytest.mark.asyncio -async def test_pending_background_response_matched_through_its_model_does_not_warn( - proxy_logging: ProxyLogging, - make_user_api_key_auth: Callable[..., UserAPIKeyAuth], - monkeypatch: pytest.MonkeyPatch, - clear_policy_registry: None, - caplog: pytest.LogCaptureFixture, -) -> None: - monkeypatch.setattr(litellm, "callbacks", _output_blocking_callbacks({})) - monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) - - with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): - await proxy_logging.post_call_success_hook( - data=_claimed_post_call_pipeline_data("response-governance"), - response=_background_response("queued"), - user_api_key_dict=make_user_api_key_auth(), - ) - - assert _warnings(caplog) == [] + assert bucket["applied_policies"] == [policy_name] + assert bucket.get("policy_sources") == ({policy_name: policy_source} if policy_source is not None else None) @pytest.mark.asyncio diff --git a/tests/unit/test_router/test_router_endpoints.py b/tests/unit/test_router/test_router_endpoints.py index 815b32b33f7..f8c13beb55e 100644 --- a/tests/unit/test_router/test_router_endpoints.py +++ b/tests/unit/test_router/test_router_endpoints.py @@ -505,11 +505,6 @@ async def test_init_responses_api_endpoints(): # Just mock the _ageneric_api_call_with_fallbacks method router._ageneric_api_call_with_fallbacks = AsyncMock() - # Add a mock implementation of _get_model_id_from_response_id to the Router instance - ResponsesAPIRequestUtils.get_model_id_from_response_id = MagicMock( - return_value=None - ) - # Call without a response_id (no model extraction should happen) await router._init_responses_api_endpoints( original_function=AsyncMock(), thread_id="thread_xyz" @@ -523,20 +518,19 @@ async def test_init_responses_api_endpoints(): # Reset the mock router._ageneric_api_call_with_fallbacks.reset_mock() - # Change the return value for the second call - ResponsesAPIRequestUtils.get_model_id_from_response_id.return_value = ( - "claude-3-sonnet" + response_id: Final = ResponsesAPIRequestUtils._build_responses_api_response_id( + custom_llm_provider="anthropic", model_id="claude-3-sonnet", response_id="resp_claude_123" ) # Call with a response_id await router._init_responses_api_endpoints( - original_function=AsyncMock(), response_id="resp_claude_123" + original_function=AsyncMock(), response_id=response_id ) # Verify model was updated in the kwargs second_call_kwargs = router._ageneric_api_call_with_fallbacks.call_args.kwargs assert second_call_kwargs["model"] == "claude-3-sonnet" - assert second_call_kwargs["response_id"] == "resp_claude_123" + assert second_call_kwargs["response_id"] == response_id @pytest.mark.asyncio