diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 9bb451495c8..991e0f2c668 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -2571,10 +2571,11 @@ class ProxyBaseLLMRequestProcessing: proxy_logging_obj=proxy_logging_obj, ) if route_type == "aget_responses": - attach_post_call_pipelines_to_retrieval( + await attach_post_call_pipelines_to_retrieval( data=self.data, user_api_key_dict=user_api_key_dict, llm_router=llm_router, + submit_policies_cache=proxy_logging_obj.internal_usage_cache, ) # Refresh AFTER pre_call_hook: guardrails (e.g. Presidio PII masking) may diff --git a/litellm/proxy/policy_engine/response_retrieval.py b/litellm/proxy/policy_engine/response_retrieval.py index 654b1682582..dced80a0772 100644 --- a/litellm/proxy/policy_engine/response_retrieval.py +++ b/litellm/proxy/policy_engine/response_retrieval.py @@ -1,11 +1,12 @@ from collections.abc import Mapping from dataclasses import dataclass from types import MappingProxyType -from typing import TYPE_CHECKING, Final, Literal, TypeAlias +from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias -from pydantic import TypeAdapter +from pydantic import BaseModel, ConfigDict, TypeAdapter from litellm._logging import verbose_proxy_logger +from litellm.constants import MANAGED_OBJECT_STALENESS_CUTOFF_DAYS from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket from litellm.proxy.common_utils.callback_utils import ( add_guardrail_to_applied_guardrails_header, @@ -30,6 +31,46 @@ PolicyPipelines: TypeAlias = tuple[tuple[str, GuardrailPipeline], ...] _POLICY_PIPELINES_ADAPTER: Final = TypeAdapter(PolicyPipelines) +_SUBMIT_POLICIES_TTL_SECONDS: Final = MANAGED_OBJECT_STALENESS_CUTOFF_DAYS * 24 * 60 * 60 + + +class SubmitPostCallPolicies(BaseModel): + model_config = ConfigDict(frozen=True) + + pipelines: PolicyPipelines + policy_sources: Mapping[str, str] + + +class SubmitPoliciesCache(Protocol): + async def async_set_cache(self, key: str, value: object, litellm_parent_otel_span: None, *, ttl: int) -> None: ... + + async def async_get_cache(self, key: str, litellm_parent_otel_span: None) -> object: ... + + +def _submit_policies_cache_key(response_id: str) -> str: + upstream_response_id: Final = ResponsesAPIRequestUtils.decode_responses_api_response_id(response_id).get( + "response_id", response_id + ) + return f"policy_engine:background_response_post_call_policies:{upstream_response_id}" + + +async def remember_submit_post_call_policies( + cache: SubmitPoliciesCache, response_id: str, submitted: SubmitPostCallPolicies +) -> None: + await cache.async_set_cache( + key=_submit_policies_cache_key(response_id), + value=submitted.model_dump(mode="json"), + litellm_parent_otel_span=None, + ttl=_SUBMIT_POLICIES_TTL_SECONDS, + ) + + +async def _submit_post_call_policies(cache: SubmitPoliciesCache, response_id: str) -> SubmitPostCallPolicies | None: + stored: Final = await cache.async_get_cache( + key=_submit_policies_cache_key(response_id), litellm_parent_otel_span=None + ) + return None if stored is None else SubmitPostCallPolicies.model_validate(stored) + @dataclass(frozen=True, slots=True) class UngovernedRetrieval: @@ -47,18 +88,23 @@ def _model_group_for_response_id(response_id: object, llm_router: "Router | None deployment: Final = llm_router.get_deployment(model_id) if deployment is None: return UngovernedRetrieval("deployment no longer in the router") - hidden_by: Final = _submit_model_hidden_by(deployment.model_name, llm_router.model_group_alias) - if hidden_by is not None: - verbose_proxy_logger.warning( - "Policy engine: background response %s re-matches policies on retrieval as model group %s (%s), " - "so a policy attached to the model name it was submitted as does not run on it", - response_id, - deployment.model_name, - hidden_by, - ) return deployment.model_name +def _warn_if_submit_model_hidden(response_id: object, model_group: str, llm_router: "Router") -> None: + hidden_by: Final = _submit_model_hidden_by(model_group, llm_router.model_group_alias) + if hidden_by is None: + return + verbose_proxy_logger.warning( + "Policy engine: background response %s has no submit-time post_call policies stored and re-matches " + "policies on retrieval as model group %s (%s), so a policy attached to the model name it was submitted as " + "does not run on it", + response_id, + model_group, + hidden_by, + ) + + def _submit_model_hidden_by(model_group: str, model_group_alias: Mapping[str, object]) -> str | None: if "*" in model_group: return "a wildcard deployment" @@ -103,23 +149,53 @@ def _post_call_pipelines_for_context(context: PolicyMatchContext) -> tuple[Polic return post_call_pipelines, MappingProxyType({match["policy_name"]: match["matched_via"] for match in matches}) -def attach_post_call_pipelines_to_retrieval( +def _retrieval_post_call_pipelines( + data: Mapping[str, object], + user_api_key_dict: "UserAPIKeyAuth", + llm_router: "Router | None", + submitted: SubmitPostCallPolicies | None, +) -> tuple[PolicyPipelines, Mapping[str, str]]: + response_id: Final = data.get("response_id") + model_group: Final = _model_group_for_response_id(response_id, llm_router) + if isinstance(model_group, UngovernedRetrieval): + if submitted is None: + verbose_proxy_logger.warning( + "Policy engine: background response %s is retrieved without its post_call policy pipelines (%s)", + response_id, + model_group.reason, + ) + return (), MappingProxyType({}) + if submitted is None and llm_router is not None: + _warn_if_submit_model_hidden(response_id, model_group, llm_router) + return _post_call_pipelines_for_context(_retrieval_context(data, user_api_key_dict, model_group)) + + +async def attach_post_call_pipelines_to_retrieval( data: dict[str, object], # mutable-ok: request-state dict the policy engine hooks all write in place user_api_key_dict: "UserAPIKeyAuth", llm_router: "Router | None", + submit_policies_cache: SubmitPoliciesCache, ) -> None: if not get_policy_registry().is_initialized(): return - model_group: Final = _model_group_for_response_id(data.get("response_id"), llm_router) - if isinstance(model_group, UngovernedRetrieval): - verbose_proxy_logger.warning( - "Policy engine: background response %s is retrieved without its post_call policy pipelines (%s)", - data.get("response_id"), - model_group.reason, - ) - return - context: Final = _retrieval_context(data, user_api_key_dict, model_group) - post_call_pipelines, policy_sources = _post_call_pipelines_for_context(context) + response_id: Final = data.get("response_id") + submitted: Final = ( + await _submit_post_call_policies(submit_policies_cache, response_id) if isinstance(response_id, str) else None + ) + retrieval_pipelines, retrieval_sources = _retrieval_post_call_pipelines( + data, user_api_key_dict, llm_router, submitted + ) + submitted_pipelines: Final = submitted.pipelines if submitted is not None else () + submitted_policy_names: Final = frozenset(policy_name for policy_name, _pipeline in submitted_pipelines) + post_call_pipelines: Final = ( + *submitted_pipelines, + *( + (policy_name, pipeline) + for policy_name, pipeline in retrieval_pipelines + if policy_name not in submitted_policy_names + ), + ) + policy_sources: Final = {**retrieval_sources, **(submitted.policy_sources if submitted is not None else {})} _, bucket = get_or_create_metadata_bucket(data) already_attached: Final = _POLICY_PIPELINES_ADAPTER.validate_python(bucket.get("_guardrail_pipelines") or ()) attached_policy_names: Final = frozenset(policy_name for policy_name, _pipeline in already_attached) @@ -142,11 +218,14 @@ def attach_post_call_pipelines_to_retrieval( add_guardrail_to_applied_guardrails_header(request_data=data, guardrail_name=step.guardrail) add_policy_sources_to_metadata( request_data=data, - policy_sources={policy_name: policy_sources[policy_name] for policy_name, _pipeline in added}, + policy_sources={ + policy_name: policy_sources[policy_name] + for policy_name, _pipeline in added + if policy_name in policy_sources + }, ) verbose_proxy_logger.debug( - "Policy engine: attached post_call pipelines to the retrieval of background response %s (model group %s): %s", - data.get("response_id"), - model_group, + "Policy engine: attached post_call pipelines to the retrieval of background response %s: %s", + response_id, ", ".join(policy_name for policy_name, _pipeline in added), ) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 0ddfc5ce62f..abc0527eb8b 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -259,6 +259,10 @@ from litellm.proxy.policy_engine.pipeline_executor import ( ) from litellm.proxy.policy_engine.policy_registry import get_policy_registry from litellm.proxy.policy_engine.policy_resolver import PolicyResolver +from litellm.proxy.policy_engine.response_retrieval import ( + SubmitPostCallPolicies, + remember_submit_post_call_policies, +) from litellm.repositories.budget_repository import BudgetRepository from litellm.repositories.config_repository import ConfigRepository from litellm.repositories.table_repositories import ( @@ -776,9 +780,13 @@ def _post_call_pipelines(data: Mapping[str, object]) -> tuple[tuple[str, "Guardr _PENDING_BACKGROUND_RESPONSE_STATUSES: Final = frozenset(("queued", "in_progress")) +_POLICY_SOURCES_ADAPTER: Final = TypeAdapter(Mapping[str, str]) -def _is_pending_background_response(response: LLMResponseTypes) -> bool: - return isinstance(response, ResponsesAPIResponse) and response.status in _PENDING_BACKGROUND_RESPONSE_STATUSES + +def _pending_background_response(response: LLMResponseTypes) -> ResponsesAPIResponse | None: + if isinstance(response, ResponsesAPIResponse) and response.status in _PENDING_BACKGROUND_RESPONSE_STATUSES: + return response + return None def _guardrails_outside_pipeline(policy_name: str, pipeline: "GuardrailPipeline") -> frozenset[str]: @@ -841,56 +849,25 @@ def _withdraw_deferred_claims( def _defer_post_call_pipelines( data: dict[str, object], # mutable-ok: same request-payload shape as post_call_success_hook's data response: ResponsesAPIResponse, -) -> None: +) -> SubmitPostCallPolicies | None: deferred: Final = _post_call_pipelines(data) if not deferred: - return + return None verbose_proxy_logger.debug( "Post_call guardrail pipelines wait for background response %s (status=%s) to be retrieved complete: %s", response.id, response.status, ", ".join(policy_name for policy_name, _pipeline in deferred), ) - tag_matched: Final = _tag_matched_deferrals(data, deferred) - if tag_matched: - verbose_proxy_logger.warning( - "Policy engine: background response %s matched post_call policies through a request tag at submit; " - "retrieval re-matches only the key, team, and model scopes, so a tag carried in the request body " - "does not govern the completed response: %s", - response.id, - ", ".join(tag_matched), - ) - body_selected: Final = _body_selected_deferrals(data, deferred) - if body_selected: - verbose_proxy_logger.warning( - "Policy engine: background response %s matched post_call policies through the request body's policies " - "list at submit; retrieval carries no request body, so those policies do not govern the completed " - "response: %s", - response.id, - ", ".join(body_selected), - ) - _withdraw_deferred_claims(data, deferred) - - -def _tag_matched_deferrals( - data: Mapping[str, object], deferred: Sequence[tuple[str, "GuardrailPipeline"]] -) -> tuple[str, ...]: - sources: Final = _policy_state_metadata(data).get("policy_sources") - if not isinstance(sources, dict): - return () - return tuple( - policy_name - for policy_name, _pipeline in deferred - if policy_name in sources and "tag:" in str(sources[policy_name]) + sources: Final = _POLICY_SOURCES_ADAPTER.validate_python(_policy_state_metadata(data).get("policy_sources") or {}) + submitted: Final = SubmitPostCallPolicies( + pipelines=deferred, + policy_sources={ + policy_name: sources[policy_name] for policy_name, _pipeline in deferred if policy_name in sources + }, ) - - -def _body_selected_deferrals( - data: Mapping[str, object], deferred: Sequence[tuple[str, "GuardrailPipeline"]] -) -> tuple[str, ...]: - sources: Final = _policy_state_metadata(data).get("policy_sources") - attributed: Final = frozenset(sources) if isinstance(sources, dict) else frozenset() - return tuple(policy_name for policy_name, _pipeline in deferred if policy_name not in attributed) + _withdraw_deferred_claims(data, deferred) + return submitted def _pipeline_unsupported_streaming_guardrails( @@ -4096,8 +4073,11 @@ class ProxyLogging: user_api_key_dict: UserAPIKeyAuth, response: LLMResponseTypes, ) -> LLMResponseTypes | None: - if _is_pending_background_response(response): - _defer_post_call_pipelines(data, response) + pending: Final = _pending_background_response(response) + if pending is not None: + submitted: Final = _defer_post_call_pipelines(data, pending) + if submitted is not None: + await remember_submit_post_call_policies(self.internal_usage_cache, pending.id, submitted) return None _, pipeline_response = await self._maybe_execute_pipelines( data=data, diff --git a/tests/unit/proxy/policy_engine/test_response_retrieval.py b/tests/unit/proxy/policy_engine/test_response_retrieval.py index 37849101b3a..50e67165bb2 100644 --- a/tests/unit/proxy/policy_engine/test_response_retrieval.py +++ b/tests/unit/proxy/policy_engine/test_response_retrieval.py @@ -3,11 +3,18 @@ from collections.abc import Iterator, Mapping import pytest +from litellm.caching.dual_cache import DualCache +from litellm.proxy.utils import InternalUsageCache from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.policy_engine.attachment_registry import get_attachment_registry from litellm.proxy.policy_engine.policy_registry import get_policy_registry -from litellm.proxy.policy_engine.response_retrieval import attach_post_call_pipelines_to_retrieval +from litellm.proxy.policy_engine.response_retrieval import ( + SubmitPostCallPolicies, + attach_post_call_pipelines_to_retrieval, + remember_submit_post_call_policies, +) from litellm.responses.utils import ResponsesAPIRequestUtils +from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline, PipelineStep from litellm.types.router import Deployment, LiteLLM_Params GOVERNED_MODEL_GROUP = "gpt-5.4-mini" @@ -46,9 +53,9 @@ def _router(model_group_alias: dict[str, object] | None = None) -> FakeRouter: ) -def _encoded_response_id(model_id: str) -> str: +def _encoded_response_id(model_id: str, upstream_response_id: str = "resp_upstream") -> str: return ResponsesAPIRequestUtils._build_responses_api_response_id( - custom_llm_provider="openai", model_id=model_id, response_id="resp_upstream" + custom_llm_provider="openai", model_id=model_id, response_id=upstream_response_id ) @@ -97,10 +104,10 @@ def _attached_pipelines(data: Mapping[str, object]) -> tuple[tuple[str, str], .. ) -def test_attaches_model_scoped_post_call_pipeline_to_retrieval(policy_engine: None) -> None: +async def test_attaches_model_scoped_post_call_pipeline_to_retrieval(policy_engine: None) -> None: data = _retrieval_data(GOVERNED_MODEL_ID) - attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) + await attach_post_call_pipelines_to_retrieval(submit_policies_cache=InternalUsageCache(DualCache()), data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) assert _attached_pipelines(data) == (("response-governance", "output-word-filter"),) assert data["litellm_metadata"]["_pipeline_managed_guardrails"] == frozenset({"output-word-filter"}) @@ -111,50 +118,53 @@ def test_attaches_model_scoped_post_call_pipeline_to_retrieval(policy_engine: No assert "guardrails" not in data["litellm_metadata"] -def test_key_and_team_context_also_governs_retrieval(policy_engine: None) -> None: +async def test_key_and_team_context_also_governs_retrieval(policy_engine: None) -> None: data = _retrieval_data(UNGOVERNED_MODEL_ID) - attach_post_call_pipelines_to_retrieval( - data=data, user_api_key_dict=UserAPIKeyAuth(team_alias="governed-team"), llm_router=_router() + await attach_post_call_pipelines_to_retrieval( + data=data, + user_api_key_dict=UserAPIKeyAuth(team_alias="governed-team"), + llm_router=_router(), + submit_policies_cache=InternalUsageCache(DualCache()), ) assert _attached_pipelines(data) == (("team-governance", "team-word-filter"),) -def test_tag_attached_policy_is_not_re_matched_when_the_retrieval_carries_no_tag(policy_engine: None) -> None: +async def test_tag_attached_policy_is_not_re_matched_when_the_retrieval_carries_no_tag(policy_engine: None) -> None: data = _retrieval_data(GOVERNED_MODEL_ID) - attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) + await attach_post_call_pipelines_to_retrieval(submit_policies_cache=InternalUsageCache(DualCache()), data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) assert _attached_pipelines(data) == (("response-governance", "output-word-filter"),) -def test_tag_attached_policy_governs_a_retrieval_whose_metadata_carries_the_tag(policy_engine: None) -> None: +async def test_tag_attached_policy_governs_a_retrieval_whose_metadata_carries_the_tag(policy_engine: None) -> None: data: dict[str, object] = { "response_id": _encoded_response_id(UNGOVERNED_MODEL_ID), "litellm_metadata": {"tags": ["governed"]}, } - attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) + await attach_post_call_pipelines_to_retrieval(submit_policies_cache=InternalUsageCache(DualCache()), data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) assert _attached_pipelines(data) == (("tag-governance", "tag-word-filter"),) assert data["litellm_metadata"]["policy_sources"] == {"tag-governance": "tag:governed"} -def test_retrieval_of_an_ungoverned_model_attaches_nothing(policy_engine: None) -> None: +async def test_retrieval_of_an_ungoverned_model_attaches_nothing(policy_engine: None) -> None: data = _retrieval_data(UNGOVERNED_MODEL_ID) - attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) + await attach_post_call_pipelines_to_retrieval(submit_policies_cache=InternalUsageCache(DualCache()), data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) assert data == _retrieval_data(UNGOVERNED_MODEL_ID) -def test_already_attached_policy_is_not_attached_twice(policy_engine: None) -> None: +async def test_already_attached_policy_is_not_attached_twice(policy_engine: None) -> None: data = _retrieval_data(GOVERNED_MODEL_ID) router = _router() - attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=router) + await attach_post_call_pipelines_to_retrieval(submit_policies_cache=InternalUsageCache(DualCache()), data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=router) - attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=router) + await attach_post_call_pipelines_to_retrieval(submit_policies_cache=InternalUsageCache(DualCache()), data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=router) assert _attached_pipelines(data) == (("response-governance", "output-word-filter"),) assert data["litellm_metadata"]["applied_policies"] == ["response-governance"] @@ -168,13 +178,13 @@ def _hidden_submit_model_warnings(caplog: pytest.LogCaptureFixture) -> list[str] ] -def test_wildcard_deployment_attaches_nothing_for_the_submitted_model_and_warns( +async def test_wildcard_deployment_attaches_nothing_for_the_submitted_model_and_warns( policy_engine: None, caplog: pytest.LogCaptureFixture ) -> None: data = _retrieval_data(WILDCARD_MODEL_ID) with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): - attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) + await attach_post_call_pipelines_to_retrieval(submit_policies_cache=InternalUsageCache(DualCache()), data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) assert data == _retrieval_data(WILDCARD_MODEL_ID) assert [ @@ -183,14 +193,14 @@ def test_wildcard_deployment_attaches_nothing_for_the_submitted_model_and_warns( ] == [True] -def test_aliased_model_group_still_attaches_its_own_policies_and_warns( +async def test_aliased_model_group_still_attaches_its_own_policies_and_warns( policy_engine: None, caplog: pytest.LogCaptureFixture ) -> None: data = _retrieval_data(GOVERNED_MODEL_ID) router = _router({"gpt-mini": GOVERNED_MODEL_GROUP, "gpt-hidden": {"model": GOVERNED_MODEL_GROUP, "hidden": True}}) with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): - attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=router) + await attach_post_call_pipelines_to_retrieval(submit_policies_cache=InternalUsageCache(DualCache()), data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=router) assert _attached_pipelines(data) == (("response-governance", "output-word-filter"),) assert [ @@ -199,11 +209,12 @@ def test_aliased_model_group_still_attaches_its_own_policies_and_warns( ] == [True] -def test_plain_model_group_retrieval_does_not_warn_about_the_submitted_model( +async def test_plain_model_group_retrieval_does_not_warn_about_the_submitted_model( policy_engine: None, caplog: pytest.LogCaptureFixture ) -> None: with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): - attach_post_call_pipelines_to_retrieval( + await attach_post_call_pipelines_to_retrieval( + submit_policies_cache=InternalUsageCache(DualCache()), data=_retrieval_data(GOVERNED_MODEL_ID), user_api_key_dict=UserAPIKeyAuth(), llm_router=_router({"other-alias": UNGOVERNED_MODEL_GROUP}), @@ -229,34 +240,144 @@ def _ungoverned_retrieval_warnings(caplog: pytest.LogCaptureFixture) -> list[str (None, "response id names no deployment"), ], ) -def test_unresolvable_response_id_attaches_nothing_and_warns( +async def test_unresolvable_response_id_attaches_nothing_and_warns( policy_engine: None, caplog: pytest.LogCaptureFixture, response_id: str, reason: str ) -> None: data = {"response_id": response_id, "litellm_metadata": {}} with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): - attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) + await attach_post_call_pipelines_to_retrieval(submit_policies_cache=InternalUsageCache(DualCache()), data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) assert data == {"response_id": response_id, "litellm_metadata": {}} assert [message.endswith(f"({reason})") for message in _ungoverned_retrieval_warnings(caplog)] == [True] -def test_without_a_router_attaches_nothing_and_warns(policy_engine: None, caplog: pytest.LogCaptureFixture) -> None: +async def test_without_a_router_attaches_nothing_and_warns(policy_engine: None, caplog: pytest.LogCaptureFixture) -> None: data = _retrieval_data(GOVERNED_MODEL_ID) with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): - attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=None) + await attach_post_call_pipelines_to_retrieval(submit_policies_cache=InternalUsageCache(DualCache()), data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=None) assert data == _retrieval_data(GOVERNED_MODEL_ID) assert [message.endswith("(no router)") for message in _ungoverned_retrieval_warnings(caplog)] == [True] -def test_without_policy_engine_attaches_nothing_quietly(caplog: pytest.LogCaptureFixture) -> None: +async def test_without_policy_engine_attaches_nothing_quietly(caplog: pytest.LogCaptureFixture) -> None: get_policy_registry().clear() data = _retrieval_data(GOVERNED_MODEL_ID) with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): - attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=None) + await attach_post_call_pipelines_to_retrieval(submit_policies_cache=InternalUsageCache(DualCache()), data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=None) assert data == _retrieval_data(GOVERNED_MODEL_ID) assert _ungoverned_retrieval_warnings(caplog) == [] + + +def _submitted(*policies: tuple[str, str], policy_sources: Mapping[str, str] | None = None) -> SubmitPostCallPolicies: + return SubmitPostCallPolicies( + pipelines=tuple( + ( + policy_name, + GuardrailPipeline( + mode="post_call", steps=[PipelineStep(guardrail=guardrail, on_pass="allow", on_fail="block")] + ), + ) + for policy_name, guardrail in policies + ), + policy_sources=policy_sources or {}, + ) + + +async def test_retrieval_runs_the_post_call_policies_the_submit_stored_alongside_its_own_match( + policy_engine: None, +) -> None: + cache = InternalUsageCache(DualCache()) + response_id = _encoded_response_id(GOVERNED_MODEL_ID) + await remember_submit_post_call_policies( + cache, + response_id, + _submitted( + ("tag-governance", "tag-word-filter"), + ("body-governance", "body-word-filter"), + ("response-governance", "output-word-filter"), + policy_sources={"tag-governance": "tag:governed", "response-governance": "model:gpt-5.4-mini"}, + ), + ) + data = _retrieval_data(GOVERNED_MODEL_ID) + + await attach_post_call_pipelines_to_retrieval( + data=data, + user_api_key_dict=UserAPIKeyAuth(team_alias="governed-team"), + llm_router=_router(), + submit_policies_cache=cache, + ) + + assert _attached_pipelines(data) == ( + ("tag-governance", "tag-word-filter"), + ("body-governance", "body-word-filter"), + ("response-governance", "output-word-filter"), + ("team-governance", "team-word-filter"), + ) + assert data["litellm_metadata"]["applied_policies"] == [ + "tag-governance", + "body-governance", + "response-governance", + "team-governance", + ] + assert data["litellm_metadata"]["policy_sources"] == { + "tag-governance": "tag:governed", + "response-governance": "model:gpt-5.4-mini", + "team-governance": "team:governed-team", + } + + +async def test_submit_stored_policies_govern_a_retrieval_whose_deployment_left_the_router( + policy_engine: None, caplog: pytest.LogCaptureFixture +) -> None: + cache = InternalUsageCache(DualCache()) + response_id = _encoded_response_id("deployment-missing-from-router") + await remember_submit_post_call_policies( + cache, response_id, _submitted(("wildcard-governance", "output-word-filter")) + ) + data: dict[str, object] = {"response_id": response_id, "litellm_metadata": {}} + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + await attach_post_call_pipelines_to_retrieval( + data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router(), submit_policies_cache=cache + ) + + assert _attached_pipelines(data) == (("wildcard-governance", "output-word-filter"),) + assert "policy_sources" not in data["litellm_metadata"] + assert _ungoverned_retrieval_warnings(caplog) == [] + + +async def test_submit_stored_policies_belong_to_their_own_response_id(policy_engine: None) -> None: + cache = InternalUsageCache(DualCache()) + await remember_submit_post_call_policies( + cache, + _encoded_response_id(GOVERNED_MODEL_ID, upstream_response_id="resp_other_upstream"), + _submitted(("tag-governance", "tag-word-filter")), + ) + data = _retrieval_data(GOVERNED_MODEL_ID) + + await attach_post_call_pipelines_to_retrieval( + data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router(), submit_policies_cache=cache + ) + + assert _attached_pipelines(data) == (("response-governance", "output-word-filter"),) + + +async def test_submit_stored_policies_govern_a_poll_that_re_encodes_the_same_upstream_response( + policy_engine: None, +) -> None: + cache = InternalUsageCache(DualCache()) + await remember_submit_post_call_policies( + cache, _encoded_response_id(GOVERNED_MODEL_ID), _submitted(("tag-governance", "tag-word-filter")) + ) + data = _retrieval_data(UNGOVERNED_MODEL_ID) + + await attach_post_call_pipelines_to_retrieval( + data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router(), submit_policies_cache=cache + ) + + assert _attached_pipelines(data) == (("tag-governance", "tag-word-filter"),) diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index fdcbcbfcd05..f8b5db7cf1a 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -76,7 +76,7 @@ from litellm.proxy.hooks.model_max_budget_limiter import ( model_budget_spend_cache_key, team_member_budget_entity_id, ) -from litellm.proxy.utils import ProxyLogging +from litellm.proxy.utils import InternalUsageCache, ProxyLogging from litellm.router import Router from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.utils import ModelResponseListIterator @@ -11816,7 +11816,12 @@ class TestBackgroundResponseRetrievalGovernance: ) return router - async def _pre_call(self, route_type: str, monkeypatch: pytest.MonkeyPatch) -> dict[str, object]: + async def _pre_call( + self, + route_type: str, + monkeypatch: pytest.MonkeyPatch, + internal_usage_cache: InternalUsageCache | None = None, + ) -> dict[str, object]: from litellm.responses.utils import ResponsesAPIRequestUtils client_facing_response_id = "resp_opaque-client-facing-id" @@ -11851,6 +11856,7 @@ class TestBackgroundResponseRetrievalGovernance: ) proxy_logging_obj = MagicMock(spec=ProxyLogging) proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=decrypting_pre_call_hook) + proxy_logging_obj.internal_usage_cache = internal_usage_cache or InternalUsageCache(DualCache()) proxy_config = MagicMock(spec=ProxyConfig) proxy_config._get_hierarchical_router_settings = AsyncMock(return_value=None) returned_data, _ = await processing_obj.common_processing_pre_call_logic( @@ -11878,6 +11884,47 @@ class TestBackgroundResponseRetrievalGovernance: assert data["litellm_metadata"]["applied_policies"] == ["response-governance"] assert data["model"] is None + @pytest.mark.asyncio + async def test_retrieving_a_background_response_runs_the_post_call_policy_its_submit_matched( + self, policy_engine: None, monkeypatch: pytest.MonkeyPatch + ) -> None: + from litellm.proxy.policy_engine.response_retrieval import ( + SubmitPostCallPolicies, + remember_submit_post_call_policies, + ) + from litellm.responses.utils import ResponsesAPIRequestUtils + from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline, PipelineStep + + cache = InternalUsageCache(DualCache()) + submit_response_id = ResponsesAPIRequestUtils._build_responses_api_response_id( + custom_llm_provider="openai", model_id="deployment-submitted", response_id="resp_upstream" + ) + await remember_submit_post_call_policies( + cache, + submit_response_id, + SubmitPostCallPolicies( + pipelines=( + ( + "tag-governance", + GuardrailPipeline( + mode="post_call", + steps=[PipelineStep(guardrail="pii-filter", on_pass="allow", on_fail="block")], + ), + ), + ), + policy_sources={"tag-governance": "tag:governed"}, + ), + ) + + data = await self._pre_call("aget_responses", monkeypatch, internal_usage_cache=cache) + + pipelines = data["litellm_metadata"]["_guardrail_pipelines"] + assert [(policy_name, [step.guardrail for step in pipeline.steps]) for policy_name, pipeline in pipelines] == [ + ("tag-governance", ["pii-filter"]), + ("response-governance", ["output-word-filter"]), + ] + assert data["litellm_metadata"]["applied_policies"] == ["tag-governance", "response-governance"] + @pytest.mark.asyncio async def test_submitting_a_response_does_not_attach_pipelines_from_its_response_id( self, policy_engine: None, monkeypatch: pytest.MonkeyPatch diff --git a/tests/unit/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/unit/proxy/utils/proxy_logging/test_guardrail_pipeline.py index acc1463b072..1932d82ddda 100644 --- a/tests/unit/proxy/utils/proxy_logging/test_guardrail_pipeline.py +++ b/tests/unit/proxy/utils/proxy_logging/test_guardrail_pipeline.py @@ -33,6 +33,7 @@ from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.callback_utils import add_guardrail_to_applied_guardrails_header from litellm.llms.openai.chat.guardrail_translation.handler import OpenAIChatCompletionsHandler from litellm.proxy.policy_engine.pipeline_executor import recorded_guardrail_information +from litellm.proxy.policy_engine.response_retrieval import attach_post_call_pipelines_to_retrieval from litellm.proxy.utils import ProxyLogging, _streamable_post_call_pipelines, stream_gated_guardrail_names from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ContentFilterGuardrail from litellm.types.guardrails import BlockedWord, ContentFilterAction, GuardrailEventHooks @@ -1582,76 +1583,41 @@ async def test_pending_background_response_withdraws_the_deferred_policy_claims( @pytest.mark.asyncio -async def test_pending_background_response_warns_when_the_deferred_policy_was_matched_through_a_tag( +@pytest.mark.parametrize( + ("policy_name", "policy_source"), + [("tag-governance", "tag:governed"), ("alias-governance", "model:gpt-mini-alias"), ("body-governance", None)], +) +async def test_pending_background_response_post_call_policy_governs_its_retrieval_by_another_key( proxy_logging: ProxyLogging, make_user_api_key_auth: Callable[..., UserAPIKeyAuth], monkeypatch: pytest.MonkeyPatch, clear_policy_registry: None, - caplog: pytest.LogCaptureFixture, + policy_name: str, + policy_source: str | None, ) -> None: monkeypatch.setattr(litellm, "callbacks", _output_blocking_callbacks({})) monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) - data = _claimed_post_call_pipeline_data("response-governance", policy_source="tag:governed+model:m") + await proxy_logging.post_call_success_hook( + data=_claimed_post_call_pipeline_data(policy_name, policy_source=policy_source), + response=_background_response("queued"), + user_api_key_dict=make_user_api_key_auth(key_alias="governed-key"), + ) + retrieval: dict[str, object] = {"response_id": "resp_bg", "litellm_metadata": {}} - with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): - out = await proxy_logging.post_call_success_hook( - data=data, response=_background_response("queued"), user_api_key_dict=make_user_api_key_auth() - ) + await attach_post_call_pipelines_to_retrieval( + data=retrieval, + user_api_key_dict=make_user_api_key_auth(api_key="sk-other", key_alias="other-key"), + llm_router=None, + submit_policies_cache=proxy_logging.internal_usage_cache, + ) - assert out.status == "queued" - assert "policy_sources" not in data["metadata"] - assert [message for message in _warnings(caplog) if "through a request tag" in message] == [ - "Policy engine: background response resp_bg matched post_call policies through a request tag at submit; " - "retrieval re-matches only the key, team, and model scopes, so a tag carried in the request body " - "does not govern the completed response: response-governance" + bucket = retrieval["litellm_metadata"] + assert isinstance(bucket, dict) + assert [(name, [step.guardrail for step in pipeline.steps]) for name, pipeline in bucket["_guardrail_pipelines"]] == [ + (policy_name, ["gr-post"]) ] - - -@pytest.mark.asyncio -async def test_pending_background_response_warns_when_the_deferred_policy_came_from_the_request_body( - proxy_logging: ProxyLogging, - make_user_api_key_auth: Callable[..., UserAPIKeyAuth], - monkeypatch: pytest.MonkeyPatch, - clear_policy_registry: None, - caplog: pytest.LogCaptureFixture, -) -> None: - monkeypatch.setattr(litellm, "callbacks", _output_blocking_callbacks({})) - monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) - data = _claimed_post_call_pipeline_data("body-governance", policy_source=None) - - with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): - out = await proxy_logging.post_call_success_hook( - data=data, response=_background_response("queued"), user_api_key_dict=make_user_api_key_auth() - ) - - assert out.status == "queued" - assert "policy_sources" not in data["metadata"] - assert _warnings(caplog) == [ - "Policy engine: background response resp_bg matched post_call policies through the request body's policies " - "list at submit; retrieval carries no request body, so those policies do not govern the completed " - "response: body-governance" - ] - - -@pytest.mark.asyncio -async def test_pending_background_response_matched_through_its_model_does_not_warn( - proxy_logging: ProxyLogging, - make_user_api_key_auth: Callable[..., UserAPIKeyAuth], - monkeypatch: pytest.MonkeyPatch, - clear_policy_registry: None, - caplog: pytest.LogCaptureFixture, -) -> None: - monkeypatch.setattr(litellm, "callbacks", _output_blocking_callbacks({})) - monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) - - with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): - await proxy_logging.post_call_success_hook( - data=_claimed_post_call_pipeline_data("response-governance"), - response=_background_response("queued"), - user_api_key_dict=make_user_api_key_auth(), - ) - - assert _warnings(caplog) == [] + assert bucket["applied_policies"] == [policy_name] + assert bucket.get("policy_sources") == ({policy_name: policy_source} if policy_source is not None else None) @pytest.mark.asyncio diff --git a/tests/unit/test_router/test_router_endpoints.py b/tests/unit/test_router/test_router_endpoints.py index 815b32b33f7..f8c13beb55e 100644 --- a/tests/unit/test_router/test_router_endpoints.py +++ b/tests/unit/test_router/test_router_endpoints.py @@ -505,11 +505,6 @@ async def test_init_responses_api_endpoints(): # Just mock the _ageneric_api_call_with_fallbacks method router._ageneric_api_call_with_fallbacks = AsyncMock() - # Add a mock implementation of _get_model_id_from_response_id to the Router instance - ResponsesAPIRequestUtils.get_model_id_from_response_id = MagicMock( - return_value=None - ) - # Call without a response_id (no model extraction should happen) await router._init_responses_api_endpoints( original_function=AsyncMock(), thread_id="thread_xyz" @@ -523,20 +518,19 @@ async def test_init_responses_api_endpoints(): # Reset the mock router._ageneric_api_call_with_fallbacks.reset_mock() - # Change the return value for the second call - ResponsesAPIRequestUtils.get_model_id_from_response_id.return_value = ( - "claude-3-sonnet" + response_id: Final = ResponsesAPIRequestUtils._build_responses_api_response_id( + custom_llm_provider="anthropic", model_id="claude-3-sonnet", response_id="resp_claude_123" ) # Call with a response_id await router._init_responses_api_endpoints( - original_function=AsyncMock(), response_id="resp_claude_123" + original_function=AsyncMock(), response_id=response_id ) # Verify model was updated in the kwargs second_call_kwargs = router._ageneric_api_call_with_fallbacks.call_args.kwargs assert second_call_kwargs["model"] == "claude-3-sonnet" - assert second_call_kwargs["response_id"] == "resp_claude_123" + assert second_call_kwargs["response_id"] == response_id @pytest.mark.asyncio