From dd8d12dfb88e33a06149fa1d6f53b0ff1e07d761 Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Mon, 5 Oct 2026 20:13:55 -0700 Subject: [PATCH 1/6] fix(proxy): build proxy-only failure logs from an allowlist and ignore request-time GCS credentials Backport of #26 to rc/1.105.0. Cherry-picked from 9c2877c4be (main). --- .../gcs_bucket/gcs_bucket_base.py | 49 ++--------- litellm/proxy/utils.py | 11 ++- .../gcs_bucket/test_gcs_bucket_base.py | 18 ++++ .../files/test_vertex_ai_files_handler.py | 84 ------------------- ..._utils_model_creation_and_error_logging.py | 23 +++++ 5 files changed, 58 insertions(+), 127 deletions(-) diff --git a/litellm/integrations/gcs_bucket/gcs_bucket_base.py b/litellm/integrations/gcs_bucket/gcs_bucket_base.py index 0856a420d6f..a7211e38fee 100644 --- a/litellm/integrations/gcs_bucket/gcs_bucket_base.py +++ b/litellm/integrations/gcs_bucket/gcs_bucket_base.py @@ -18,7 +18,7 @@ from litellm.llms.custom_httpx.http_handler import ( httpxSpecialProvider, ) from litellm.types.integrations.gcs_bucket import * -from litellm.types.utils import StandardCallbackDynamicParams, StandardLoggingPayload +from litellm.types.utils import StandardLoggingPayload if TYPE_CHECKING: from litellm.llms.vertex_ai.vertex_llm_base import VertexBase @@ -137,49 +137,16 @@ class GCSBucketBase(CustomBatchLogger): async def get_gcs_logging_config(self, kwargs: dict[str, Any] | None = {}) -> GCSLoggingConfig: """ - This function is used to get the GCS logging config for the GCS Bucket Logger. - It checks if the dynamic parameters are provided in the kwargs and uses them to get the GCS logging config. - If no dynamic parameters are provided, it uses the default values. + Returns the bucket and service account this logger was configured with. """ - if kwargs is None: - kwargs = {} - - standard_callback_dynamic_params: Final[StandardCallbackDynamicParams | None] = kwargs.get( - "standard_callback_dynamic_params", None - ) - - bucket_name: str - path_service_account: str | None - if standard_callback_dynamic_params is not None: - verbose_logger.debug("Using dynamic GCS logging") - verbose_logger.debug("standard_callback_dynamic_params: %s", standard_callback_dynamic_params) - - _bucket_name: str | None = standard_callback_dynamic_params.get("gcs_bucket_name", None) or self.BUCKET_NAME - _path_service_account: Final[str | None] = ( - standard_callback_dynamic_params.get("gcs_path_service_account", None) or self.path_service_account_json + if self.BUCKET_NAME is None: + raise ValueError( + "GCS_BUCKET_NAME is not set in the environment, but GCS Bucket is being used as a logging callback. Please set 'GCS_BUCKET_NAME' in the environment." ) - - if _bucket_name is None: - raise ValueError( - "GCS_BUCKET_NAME is not set in the environment, but GCS Bucket is being used as a logging callback. Please set 'GCS_BUCKET_NAME' in the environment." - ) - bucket_name = _bucket_name - path_service_account = _path_service_account - vertex_instance = await self.get_or_create_vertex_instance(credentials=path_service_account) - else: - # If no dynamic parameters, use the default instance - if self.BUCKET_NAME is None: - raise ValueError( - "GCS_BUCKET_NAME is not set in the environment, but GCS Bucket is being used as a logging callback. Please set 'GCS_BUCKET_NAME' in the environment." - ) - bucket_name = self.BUCKET_NAME - path_service_account = self.path_service_account_json - vertex_instance = await self.get_or_create_vertex_instance(credentials=path_service_account) - return GCSLoggingConfig( - bucket_name=bucket_name, - vertex_instance=vertex_instance, - path_service_account=path_service_account, + bucket_name=self.BUCKET_NAME, + vertex_instance=await self.get_or_create_vertex_instance(credentials=self.path_service_account_json), + path_service_account=self.path_service_account_json, ) async def get_or_create_vertex_instance(self, credentials: str | None) -> VertexBase: diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 2b1259eb469..d1a09b0b065 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1328,6 +1328,13 @@ def _call_type_for_route(route: str | None) -> str | None: _PROXY_ONLY_LLM_API_ERRORS: Final = (HTTPException, ProxyException, GuardrailRaisedException) _LOG_DB_METRICS_CALL_TYPES: Final = frozenset(("get_data", "insert_data", "update_data", "delete_data")) +_PROXY_ONLY_ERROR_LOGGED_FIELDS: Final = frozenset( + {"model", "messages", "input", "prompt", "user", "stream", "metadata", "litellm_metadata", "litellm_call_id"} +) + + +def _proxy_only_error_log_fields(request_data: Mapping[str, _T]) -> Mapping[str, _T]: + return MappingProxyType({k: v for k, v in request_data.items() if k in _PROXY_ONLY_ERROR_LOGGED_FIELDS}) _EXCEPTION_CHAIN_MAX_DEPTH: Final = 16 @@ -3714,7 +3721,7 @@ class ProxyLogging: original_function=route or "IGNORE_THIS", rules_obj=litellm.utils.Rules(), start_time=datetime.now(), - **request_data, + **_proxy_only_error_log_fields(request_data), ) request_data["litellm_logging_obj"] = litellm_logging_obj # rebind-ok: lifted then popped by the caller if "metadata" not in request_data: @@ -3727,7 +3734,7 @@ class ProxyLogging: _litellm_params: Final = {} litellm_param_keys: Final = LoggedLiteLLMParams.__annotations__.keys() - for k, v in request_data.items(): + for k, v in _proxy_only_error_log_fields(request_data).items(): if k in litellm_param_keys: _litellm_params[k] = v elif k not in ("model", "user", "litellm_logging_obj"): diff --git a/tests/unit/integrations/gcs_bucket/test_gcs_bucket_base.py b/tests/unit/integrations/gcs_bucket/test_gcs_bucket_base.py index 781b56cae14..d9d017c1893 100644 --- a/tests/unit/integrations/gcs_bucket/test_gcs_bucket_base.py +++ b/tests/unit/integrations/gcs_bucket/test_gcs_bucket_base.py @@ -91,6 +91,24 @@ class TestGCSBucketBase: custom_llm_provider="vertex_ai", ) + @pytest.mark.asyncio + async def test_gcs_logging_config_ignores_request_time_bucket_and_credentials(self, monkeypatch): + monkeypatch.setenv("GCS_PATH_SERVICE_ACCOUNT", "/admin/sa.json") + handler = GCSBucketBase(bucket_name="admin-bucket") + + with patch.object(handler, "get_or_create_vertex_instance", new_callable=AsyncMock) as mock_vertex: + config = await handler.get_gcs_logging_config( + { + "standard_callback_dynamic_params": { + "gcs_bucket_name": "caller-bucket", + "gcs_path_service_account": "/caller/sa.json", + } + } + ) + + assert (config["bucket_name"], config["path_service_account"]) == ("admin-bucket", "/admin/sa.json") + mock_vertex.assert_awaited_once_with(credentials="/admin/sa.json") + @pytest.mark.asyncio async def test_log_json_data_on_gcs_url_encodes_object_name(self): handler = GCSBucketBase(bucket_name="test-bucket") diff --git a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_handler.py b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_handler.py index 9b7cd127b83..39310836bf2 100644 --- a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_handler.py +++ b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_handler.py @@ -260,87 +260,3 @@ class TestVertexAIFilesHandler: ) assert service_account == '{"type": "service_account", "project_id": "p"}' - - @pytest.mark.asyncio - async def test_afile_content_honors_per_model_bucket_over_env(self, monkeypatch): - """ - Regression for #32640: a batch output written to a per-model gcs_bucket_name must be - readable even when the global GCS_BUCKET_NAME points at a different bucket. Before the - fix the read path resolved the bucket from env only and raised - "file_id bucket does not match the configured storage bucket". - """ - monkeypatch.setenv("GCS_BUCKET_NAME", "env-default-bucket") - monkeypatch.delenv("GCS_PATH_SERVICE_ACCOUNT", raising=False) - - file_id = "gs%3A%2F%2Fmy-model-bucket%2Flitellm-vertex-files%2Fuploads%2Fabc-batch-output.jsonl" - file_content_request = FileContentRequest(file_id=file_id, extra_headers=None, extra_body=None) - - with ( - patch.object(self.handler, "download_gcs_object", new_callable=AsyncMock) as mock_download, - patch.object( - self.handler, - "get_or_create_vertex_instance", - new_callable=AsyncMock, - return_value=object(), - ), - ): - mock_download.return_value = b"batch output" - - result = await self.handler.afile_content( - file_content_request=file_content_request, - vertex_credentials="/model/sa.json", - vertex_project="test-project", - vertex_location="us-central1", - timeout=60.0, - max_retries=0, - litellm_params={ - "gcs_bucket_name": "my-model-bucket", - "vertex_credentials": "/model/sa.json", - }, - ) - - assert isinstance(result, HttpxBinaryResponseContent) - assert result.response.content == b"batch output" - - dynamic_params = mock_download.call_args.kwargs["standard_callback_dynamic_params"] - assert dynamic_params["gcs_bucket_name"] == "my-model-bucket" - assert dynamic_params["gcs_path_service_account"] == "/model/sa.json" - assert mock_download.call_args.kwargs["object_name"] == "litellm-vertex-files/uploads/abc-batch-output.jsonl" - - @pytest.mark.asyncio - async def test_afile_content_reads_without_global_env_bucket(self, monkeypatch): - """ - Regression for #32640: with no global GCS_BUCKET_NAME set, a model-group-level - deployment (per-model gcs_bucket_name) must still be readable. Before the fix the read - path raised "GCS_BUCKET_NAME is not set in the environment". - """ - monkeypatch.delenv("GCS_BUCKET_NAME", raising=False) - monkeypatch.delenv("GCS_PATH_SERVICE_ACCOUNT", raising=False) - - file_id = "gs%3A%2F%2Fmy-model-bucket%2Flitellm-vertex-files%2Fuploads%2Fabc-batch-output.jsonl" - file_content_request = FileContentRequest(file_id=file_id, extra_headers=None, extra_body=None) - - with ( - patch.object(self.handler, "download_gcs_object", new_callable=AsyncMock) as mock_download, - patch.object( - self.handler, - "get_or_create_vertex_instance", - new_callable=AsyncMock, - return_value=object(), - ), - ): - mock_download.return_value = b"batch output" - - result = await self.handler.afile_content( - file_content_request=file_content_request, - vertex_credentials="/model/sa.json", - vertex_project="test-project", - vertex_location="us-central1", - timeout=60.0, - max_retries=0, - litellm_params={"gcs_bucket_name": "my-model-bucket"}, - ) - - assert isinstance(result, HttpxBinaryResponseContent) - dynamic_params = mock_download.call_args.kwargs["standard_callback_dynamic_params"] - assert dynamic_params["gcs_bucket_name"] == "my-model-bucket" diff --git a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py index d577ef354f0..29f99bb76da 100644 --- a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py +++ b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py @@ -163,6 +163,29 @@ async def test_proxy_only_error_log_keeps_litellm_metadata_in_litellm_params(): assert "litellm_metadata" not in captured["optional_params"] +@pytest.mark.asyncio +async def test_proxy_only_error_log_drops_caller_logging_fields_and_keeps_key_details(): + request_data: Final[dict[str, object]] = { + "model": "gpt-4o", + "messages": [{"role": "user", "content": "hi"}], + "litellm_metadata": {"tags": ["t"]}, + "langsmith_base_url": "https://caller.invalid", + "standard_callback_dynamic_params": {"gcs_path_service_account": "/caller/sa.json"}, + } + + await ProxyLogging(user_api_key_cache=DualCache())._handle_logging_proxy_only_error( + request_data=request_data, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-rejected", team_id="team-a", request_route="/v1/chat/completions"), + route="/v1/chat/completions", + original_exception=Exception("bad key"), + ) + + details: Final = request_data["litellm_logging_obj"].model_call_details + assert "caller" not in str(details["standard_callback_dynamic_params"]) + assert "langsmith_base_url" not in details + assert details["litellm_params"]["metadata"]["user_api_key_team_id"] == "team-a" + + @pytest.mark.asyncio async def test_proxy_only_error_log_keeps_the_request_litellm_call_id(monkeypatch: pytest.MonkeyPatch): """LIT-7836: a route that already stamped the caller's litellm_call_id must From 97474e4ae7b99494aa841ff25160729012b81fb1 Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Tue, 6 Oct 2026 01:58:29 -0700 Subject: [PATCH 2/6] fix(proxy): drop caller metadata from proxy-only failure logs metadata and litellm_metadata leave the proxy-only failure-log allowlist, so a rejected request can no longer point its failure log at its own Langsmith or Langfuse host through nested metadata. (cherry picked from commit 30f6715f480ee7e2854514fe53b91d1b16e1be03) --- litellm/proxy/utils.py | 2 +- ..._utils_model_creation_and_error_logging.py | 55 ++----------------- .../test_post_call_failure_hook.py | 8 +-- 3 files changed, 9 insertions(+), 56 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index d1a09b0b065..58b103ec739 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1329,7 +1329,7 @@ def _call_type_for_route(route: str | None) -> str | None: _PROXY_ONLY_LLM_API_ERRORS: Final = (HTTPException, ProxyException, GuardrailRaisedException) _LOG_DB_METRICS_CALL_TYPES: Final = frozenset(("get_data", "insert_data", "update_data", "delete_data")) _PROXY_ONLY_ERROR_LOGGED_FIELDS: Final = frozenset( - {"model", "messages", "input", "prompt", "user", "stream", "metadata", "litellm_metadata", "litellm_call_id"} + {"model", "messages", "input", "prompt", "user", "stream", "litellm_call_id"} ) diff --git a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py index 29f99bb76da..4121dca8ae1 100644 --- a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py +++ b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py @@ -116,59 +116,12 @@ async def test_proxy_only_error_log_marks_no_upstream_llm_call(): @pytest.mark.asyncio -async def test_proxy_only_error_log_keeps_litellm_metadata_in_litellm_params(): - """Responses API requests carry guardrail info under ``litellm_metadata`` - (not ``metadata``). It must land in litellm_params so - ``merge_litellm_metadata`` can surface ``guardrail_information`` in the - spend-log failure row, matching the chat completions path.""" - from litellm.proxy._types import UserAPIKeyAuth - - proxy_logging_obj = ProxyLogging(user_api_key_cache=DualCache()) - captured = {} - guardrail_info = [{"guardrail_name": "test-guard", "guardrail_status": "blocked"}] - - def fake_update_environment_variables(self, *args, **kwargs): - captured["litellm_params"] = kwargs.get("litellm_params") - captured["optional_params"] = kwargs.get("optional_params") - - from litellm.litellm_core_utils.litellm_logging import Logging - - orig_update_env = Logging.update_environment_variables - orig_pre_call = Logging.pre_call - orig_async_failure = Logging.async_failure_handler - - async def _noop_async_failure(self, *args, **kwargs): - return None - - Logging.update_environment_variables = fake_update_environment_variables - Logging.pre_call = lambda self, *args, **kwargs: None - Logging.async_failure_handler = _noop_async_failure - try: - await proxy_logging_obj._handle_logging_proxy_only_error( - request_data={ - "model": "gpt-4o", - "input": "blocked prompt", - "litellm_metadata": {"standard_logging_guardrail_information": guardrail_info}, - }, - user_api_key_dict=UserAPIKeyAuth(api_key="sk-9876", request_route="/v1/responses"), - route="/v1/responses", - original_exception=HTTPException(status_code=400, detail="blocked"), - ) - finally: - Logging.update_environment_variables = orig_update_env - Logging.pre_call = orig_pre_call - Logging.async_failure_handler = orig_async_failure - - assert captured["litellm_params"]["litellm_metadata"]["standard_logging_guardrail_information"] == guardrail_info - assert "litellm_metadata" not in captured["optional_params"] - - -@pytest.mark.asyncio -async def test_proxy_only_error_log_drops_caller_logging_fields_and_keeps_key_details(): +@pytest.mark.parametrize("metadata_field", ["metadata", "litellm_metadata"]) +async def test_proxy_only_error_log_drops_caller_logging_fields_and_metadata(metadata_field: str): request_data: Final[dict[str, object]] = { "model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}], - "litellm_metadata": {"tags": ["t"]}, + metadata_field: {"langsmith_base_url": "https://caller.invalid", "langsmith_api_key": "caller-key"}, "langsmith_base_url": "https://caller.invalid", "standard_callback_dynamic_params": {"gcs_path_service_account": "/caller/sa.json"}, } @@ -183,7 +136,7 @@ async def test_proxy_only_error_log_drops_caller_logging_fields_and_keeps_key_de details: Final = request_data["litellm_logging_obj"].model_call_details assert "caller" not in str(details["standard_callback_dynamic_params"]) assert "langsmith_base_url" not in details - assert details["litellm_params"]["metadata"]["user_api_key_team_id"] == "team-a" + assert "caller" not in str(details["litellm_params"]) @pytest.mark.asyncio diff --git a/tests/unit/proxy/utils/proxy_logging/test_post_call_failure_hook.py b/tests/unit/proxy/utils/proxy_logging/test_post_call_failure_hook.py index 0e007429789..a9b399c550a 100644 --- a/tests/unit/proxy/utils/proxy_logging/test_post_call_failure_hook.py +++ b/tests/unit/proxy/utils/proxy_logging/test_post_call_failure_hook.py @@ -529,9 +529,9 @@ async def test_post_call_failure_hook_omits_provider_when_a_deployment_does_not_ async def test_handle_logging_proxy_only_path_attributes_with_read_only_metadata( proxy_logging, make_user_api_key_auth, monkeypatch ): - """With a logging object already on the request, its metadata is taken as given; - a read-only mapping there must not crash the stamp, and the failure handler - still receives the provider attribution.""" + """With a logging object already on the request, a read-only metadata mapping + must not crash the stamp, the failure handler still receives the provider + attribution, and the request's metadata is not copied into the update.""" from litellm.proxy import proxy_server monkeypatch.setattr( @@ -568,7 +568,7 @@ async def test_handle_logging_proxy_only_path_attributes_with_read_only_metadata update_kwargs = logging_obj.update_environment_variables.call_args.kwargs assert update_kwargs["custom_llm_provider"] == "openai" assert update_kwargs["litellm_params"]["custom_llm_provider"] == "openai" - assert update_kwargs["litellm_params"]["metadata"] == {"user_api_key_alias": "frozen"} + assert "user_api_key_alias" not in update_kwargs["litellm_params"]["metadata"] @pytest.mark.asyncio From 116dd0504c9c77fd5c7c0930a2b03d6f37b7603b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 8 Oct 2026 07:26:14 -0700 Subject: [PATCH 3/6] fix(proxy): keep allowlisted metadata in proxy-only failure logs Backport of #50 to rc/1.105.0. Cherry-picked from 9722ca937e (main). --- litellm/litellm_core_utils/litellm_logging.py | 20 ++- litellm/proxy/utils.py | 96 ++++++++++++-- .../test_litellm_logging.py | 15 +++ ..._utils_model_creation_and_error_logging.py | 108 +++++++++++++++- .../test_post_call_failure_hook.py | 122 +++++++++++++++++- 5 files changed, 329 insertions(+), 32 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 2b2f6e9b41b..30f1604d152 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -6478,19 +6478,17 @@ class StandardLoggingPayloadSetup: litellm_metadata: Final = cast( # cast-ok: request metadata is caller-provided Mapping[str, object], litellm_params.get("litellm_metadata") or {} ) - if metadata.get("tags", []): - request_tags = cast(list[str], metadata.get("tags", [])).copy() # cast-ok: tags are caller-provided - elif litellm_metadata.get("tags", []): - request_tags = cast(list[str], litellm_metadata.get("tags", [])).copy() # cast-ok: tags are caller-provided - else: - request_tags = [] + request_tags: Final = next( + ( + cast(list[str], tags) # cast-ok: tags are caller-provided + for tags in (metadata.get("tags"), litellm_metadata.get("tags")) + if isinstance(tags, list) and tags + ), + [], + ) user_agent_tags: Final = StandardLoggingPayloadSetup._get_user_agent_tags(proxy_server_request) additional_header_tags: Final = StandardLoggingPayloadSetup._get_extra_header_tags(proxy_server_request) - if user_agent_tags is not None: - request_tags.extend(user_agent_tags) - if additional_header_tags is not None: - request_tags.extend(additional_header_tags) - return request_tags + return [*request_tags, *(user_agent_tags or ()), *(additional_header_tags or ())] _get_request_tags = get_request_tags diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 58b103ec739..bd9d714f74a 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1237,7 +1237,8 @@ def _stamp_deployment_attribution( callbacks key off it and must not count a proxy-side reject against the deployment. A failure after the provider handoff keeps the metadata the router stamped; a request that never reached a provider is flagged ``PROXY_REJECTED_BEFORE_ROUTING_KEY`` (deployment metrics key off it) whatever - its metadata says, since ``metadata.model_info`` can be caller supplied.""" + its metadata says, since ``metadata.model_info`` can be caller supplied. Its ``deployment`` and + ``model_group`` always come from the router config, never from the caller's metadata.""" attribution: Final = _deployment_attribution_for_model_group(model_group, team_id) if "custom_llm_provider" in attribution: litellm_params["custom_llm_provider"] = attribution["custom_llm_provider"] @@ -1252,9 +1253,13 @@ def _stamp_deployment_attribution( if not isinstance(metadata, dict): return attribution metadata.setdefault("model_info", attribution["model_info"]) - metadata.setdefault("deployment", attribution["deployment"]) - if isinstance(model_group, str): - metadata.setdefault("model_group", model_group) + routing: Final = MappingProxyType( + {"deployment": attribution["deployment"], **({"model_group": model_group} if model_group else {})} + ) + metadata.update(routing) + litellm_metadata: Final = litellm_params.get("litellm_metadata") + if isinstance(litellm_metadata, dict): + litellm_metadata.update(routing) return attribution @@ -1333,8 +1338,50 @@ _PROXY_ONLY_ERROR_LOGGED_FIELDS: Final = frozenset( ) -def _proxy_only_error_log_fields(request_data: Mapping[str, _T]) -> Mapping[str, _T]: - return MappingProxyType({k: v for k, v in request_data.items() if k in _PROXY_ONLY_ERROR_LOGGED_FIELDS}) +def _as_str_mapping(value: object) -> Mapping[str, object]: + if not isinstance(value, Mapping): + return MappingProxyType({}) + return cast( # cast-ok: the runtime Mapping check validates the container shape + Mapping[str, object], value + ) + + +def _proxy_only_error_logged_metadata_keys(*, authenticated: bool, past_pre_call: bool) -> frozenset[str]: + return frozenset( + key + for key, logged in (("tags", authenticated), ("standard_logging_guardrail_information", past_pre_call)) + if logged + ) + + +def _is_logged_metadata_value(key: str, value: object) -> bool: + return key != "tags" or (isinstance(value, list) and all(isinstance(item, str) for item in value)) + + +def _proxy_only_error_log_metadata(value: object, logged_keys: frozenset[str]) -> dict[str, object] | None: + if not isinstance(value, Mapping): + return None + metadata: Final = _as_str_mapping(value) + return { + key: value for key, value in metadata.items() if key in logged_keys and _is_logged_metadata_value(key, value) + } + + +def _proxy_only_error_log_fields( + request_data: Mapping[str, object], logged_metadata_keys: frozenset[str] +) -> Mapping[str, object]: + metadata_fields: Final = { + key: filtered_metadata + for key in ("metadata", "litellm_metadata") + if (filtered_metadata := _proxy_only_error_log_metadata(request_data.get(key), logged_metadata_keys)) + is not None + } + return MappingProxyType( + { + **{key: value for key, value in request_data.items() if key in _PROXY_ONLY_ERROR_LOGGED_FIELDS}, + **metadata_fields, + } + ) _EXCEPTION_CHAIN_MAX_DEPTH: Final = 16 @@ -3614,6 +3661,7 @@ class ProxyLogging: user_api_key_dict=user_api_key_dict, route=route, original_exception=original_exception, + error_type=error_type, ) request_data.update( @@ -3702,6 +3750,7 @@ class ProxyLogging: user_api_key_dict: UserAPIKeyAuth, route: str | None = None, original_exception: Exception | None = None, + error_type: ProxyErrorTypes | None = None, ): """ Handle logging for proxy only errors by calling `litellm_logging_obj.async_failure_handler` @@ -3709,19 +3758,22 @@ class ProxyLogging: Is triggered when self._is_proxy_only_error() returns True """ litellm_logging_obj: Logging | None = request_data.get("litellm_logging_obj", None) + logged_metadata_keys: Final = _proxy_only_error_logged_metadata_keys( + authenticated=error_type != ProxyErrorTypes.auth_error, past_pre_call=litellm_logging_obj is not None + ) + user_api_key_logged_metadata: Final = LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key( + user_api_key_dict=user_api_key_dict + ) if litellm_logging_obj is None: from litellm._uuid import uuid request_data.setdefault("litellm_call_id", str(uuid.uuid4())) - user_api_key_logged_metadata: Final = LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key( - user_api_key_dict=user_api_key_dict - ) litellm_logging_obj, data = litellm.utils.function_setup( original_function=route or "IGNORE_THIS", rules_obj=litellm.utils.Rules(), start_time=datetime.now(), - **_proxy_only_error_log_fields(request_data), + **_proxy_only_error_log_fields(request_data, logged_metadata_keys), ) request_data["litellm_logging_obj"] = litellm_logging_obj # rebind-ok: lifted then popped by the caller if "metadata" not in request_data: @@ -3734,14 +3786,32 @@ class ProxyLogging: _litellm_params: Final = {} litellm_param_keys: Final = LoggedLiteLLMParams.__annotations__.keys() - for k, v in _proxy_only_error_log_fields(request_data).items(): + for k, v in _proxy_only_error_log_fields(request_data, logged_metadata_keys).items(): if k in litellm_param_keys: _litellm_params[k] = v elif k not in ("model", "user", "litellm_logging_obj"): _optional_params[k] = v + existing: Final = _as_str_mapping(litellm_logging_obj.litellm_params) + litellm_params: Final = { + **_litellm_params, + "metadata": { + **_as_str_mapping(existing.get("metadata")), + **_as_str_mapping(_litellm_params.get("metadata")), + **user_api_key_logged_metadata, + }, + **{ + key: { + **_as_str_mapping(existing.get(key)), + **_as_str_mapping(value), + } + for key, value in _litellm_params.items() + if key == "litellm_metadata" + }, + } + attribution: Final = _stamp_deployment_attribution( - _litellm_params, + litellm_params, request_data.get("model"), user_api_key_dict.team_id, dispatched=_reached_deployment(litellm_logging_obj), @@ -3751,7 +3821,7 @@ class ProxyLogging: model=request_data.get("model", ""), user=request_data.get("user", ""), optional_params=_optional_params, - litellm_params=_litellm_params, + litellm_params=litellm_params, **( {"custom_llm_provider": attribution["custom_llm_provider"]} if "custom_llm_provider" in attribution diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 346f54622f3..6ddea8c95e6 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -3140,6 +3140,21 @@ def test_get_request_tags_from_metadata_and_litellm_metadata(): assert "User-Agent: litellm/1.0.0" in tags +@pytest.mark.parametrize("unusable_metadata_tags", [123, "tag", {"tag": True}, []]) +def test_get_request_tags_falls_back_to_litellm_metadata_when_metadata_tags_are_unusable(unusable_metadata_tags): + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + + tags = StandardLoggingPayloadSetup._get_request_tags( + litellm_params={ + "metadata": {"tags": unusable_metadata_tags}, + "litellm_metadata": {"tags": ["litellm-metadata-tag"]}, + }, + proxy_server_request={}, + ) + + assert tags == ["litellm-metadata-tag"] + + def test_get_request_tags_does_not_mutate_original_tags(): """ Test that _get_request_tags does not mutate the original tags list in metadata. diff --git a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py index 4121dca8ae1..33b6718893d 100644 --- a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py +++ b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py @@ -117,11 +117,58 @@ async def test_proxy_only_error_log_marks_no_upstream_llm_call(): @pytest.mark.asyncio @pytest.mark.parametrize("metadata_field", ["metadata", "litellm_metadata"]) -async def test_proxy_only_error_log_drops_caller_logging_fields_and_metadata(metadata_field: str): +async def test_proxy_only_error_log_drops_caller_guardrail_information_without_logging_object( + metadata_field: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "callbacks", []) + monkeypatch.setattr(litellm, "input_callback", []) + monkeypatch.setattr(litellm, "failure_callback", []) + monkeypatch.setattr(litellm, "_async_failure_callback", []) + guardrail_info: Final = [ + { + "guardrail_name": "caller-guardrail", + "guardrail_status": "guardrail_intervened", + "guardrail_cost": 123.45, + } + ] request_data: Final[dict[str, object]] = { "model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}], - metadata_field: {"langsmith_base_url": "https://caller.invalid", "langsmith_api_key": "caller-key"}, + metadata_field: {"tags": ["t"], "standard_logging_guardrail_information": guardrail_info}, + } + + await ProxyLogging(user_api_key_cache=DualCache())._handle_logging_proxy_only_error( + request_data=request_data, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-rejected", team_id="team-a", request_route="/v1/chat/completions"), + route="/v1/chat/completions", + original_exception=HTTPException(status_code=400, detail="blocked"), + ) + + details: Final = request_data["litellm_logging_obj"].model_call_details + assert details["standard_logging_object"] is not None + assert details["litellm_params"][metadata_field]["tags"] == ["t"] + assert "standard_logging_guardrail_information" not in details["litellm_params"][metadata_field] + assert "guardrail_cost" not in str(details["standard_logging_object"]) + assert "litellm_metadata" not in details["optional_params"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("metadata_field", ["metadata", "litellm_metadata"]) +async def test_proxy_only_error_log_keeps_tags_and_drops_caller_logging_fields( + metadata_field: str, +) -> None: + request_data: Final[dict[str, object]] = { + "model": "gpt-4o", + "messages": [{"role": "user", "content": "hi"}], + metadata_field: { + "langsmith_base_url": "https://caller.invalid", + "langsmith_api_key": "caller-key", + "tags": ["t"], + "user_api_key_team_id": "caller-team", + "user_api_key_hash": "caller-hash", + "user_api_key_auth_metadata": {"caller": "metadata"}, + }, "langsmith_base_url": "https://caller.invalid", "standard_callback_dynamic_params": {"gcs_path_service_account": "/caller/sa.json"}, } @@ -137,6 +184,63 @@ async def test_proxy_only_error_log_drops_caller_logging_fields_and_metadata(met assert "caller" not in str(details["standard_callback_dynamic_params"]) assert "langsmith_base_url" not in details assert "caller" not in str(details["litellm_params"]) + assert details["litellm_params"][metadata_field]["tags"] == ["t"] + assert details["litellm_params"]["metadata"]["user_api_key_team_id"] == "team-a" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("metadata_field", ["metadata", "litellm_metadata"]) +async def test_proxy_only_error_log_drops_malformed_metadata_tags(metadata_field: str) -> None: + guardrail_info: Final = [{"guardrail_name": "g", "guardrail_status": "guardrail_intervened"}] + request_data: Final[dict[str, object]] = { + "model": "gpt-4o", + "messages": [{"role": "user", "content": "hi"}], + metadata_field: {"tags": 123, "standard_logging_guardrail_information": guardrail_info}, + } + + await ProxyLogging(user_api_key_cache=DualCache())._handle_logging_proxy_only_error( + request_data=request_data, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-rejected", team_id="team-a", request_route="/v1/chat/completions"), + route="/v1/chat/completions", + original_exception=Exception("bad key"), + ) + + details: Final = request_data["litellm_logging_obj"].model_call_details + assert details["standard_logging_object"] is not None + assert "tags" not in details["litellm_params"][metadata_field] + assert "standard_logging_guardrail_information" not in details["litellm_params"][metadata_field] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("metadata_field", ["metadata", "litellm_metadata"]) +async def test_proxy_only_error_log_builds_payload_when_logging_object_has_malformed_tags( + metadata_field: str, +) -> None: + request_data: Final[dict[str, object]] = { + "model": "gpt-4o", + "messages": [{"role": "user", "content": "hi"}], + "litellm_call_id": "malformed-tags-call", + metadata_field: {"tags": 123}, + } + logging_obj, _ = litellm.utils.function_setup( + original_function="acompletion", + rules_obj=litellm.utils.Rules(), + start_time=real_datetime.datetime.now(), + **request_data, + ) + + await ProxyLogging(user_api_key_cache=DualCache())._handle_logging_proxy_only_error( + request_data={**request_data, "litellm_logging_obj": logging_obj}, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-blocked", team_id="team-a", request_route="/v1/chat/completions"), + route="/v1/chat/completions", + original_exception=HTTPException(status_code=400, detail="Content blocked"), + ) + + standard_logging_object: Final = logging_obj.model_call_details["standard_logging_object"] + assert standard_logging_object is not None + assert standard_logging_object["model"] == request_data["model"] + assert standard_logging_object["metadata"]["user_api_key_team_id"] == "team-a" + assert 123 not in standard_logging_object["request_tags"] @pytest.mark.asyncio diff --git a/tests/unit/proxy/utils/proxy_logging/test_post_call_failure_hook.py b/tests/unit/proxy/utils/proxy_logging/test_post_call_failure_hook.py index a9b399c550a..3a11877c99f 100644 --- a/tests/unit/proxy/utils/proxy_logging/test_post_call_failure_hook.py +++ b/tests/unit/proxy/utils/proxy_logging/test_post_call_failure_hook.py @@ -357,6 +357,103 @@ async def test_post_call_failure_hook_flags_pre_routing_reject_despite_caller_mo assert kwargs["litellm_params"]["custom_llm_provider"] == "openai" +@pytest.mark.asyncio +@pytest.mark.parametrize("metadata_field", ["metadata", "litellm_metadata"]) +async def test_pre_routing_reject_logs_the_configured_deployment_over_caller_metadata( + proxy_logging, make_user_api_key_auth, monkeypatch, metadata_field: str +): + from litellm.proxy import proxy_server + from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload + + recorded: list[dict] = [] + + class _RecordingLogger(CustomLogger): + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + recorded.append(kwargs) + + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "internal-model", + "litellm_params": {"model": "openai/gpt-4.1", "api_key": "sk-test"}, + } + ] + ), + ) + monkeypatch.setattr(litellm, "callbacks", [_RecordingLogger()]) + proxy_logging.alert_types = [] + request_data: Final[dict[str, object]] = { + "model": "internal-model", + "messages": [{"role": "user", "content": "hi"}], + "litellm_call_id": "caller-routing-metadata", + metadata_field: {"deployment": "openai/caller-chosen", "model_group": "caller-group"}, + } + logging_obj, _ = litellm.utils.function_setup( + original_function="acompletion", rules_obj=litellm.utils.Rules(), start_time=datetime.now(), **request_data + ) + + await proxy_logging.post_call_failure_hook( + request_data={**request_data, "litellm_logging_obj": logging_obj}, + original_exception=HTTPException(status_code=400, detail="Content blocked"), + user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions"), + route="/chat/completions", + ) + + assert len(recorded) == 1 + now: Final = datetime.now() + payload: Final = get_logging_payload( + kwargs={**recorded[0], "completion_start_time": now}, response_obj=None, start_time=now, end_time=now + ) + standard_logging_object: Final = recorded[0]["standard_logging_object"] + assert { + "logged_model": standard_logging_object["model"], + "logged_model_group": standard_logging_object["model_group"], + "spend_model": payload["model"], + "spend_model_group": payload["model_group"], + } == { + "logged_model": "openai/gpt-4.1", + "logged_model_group": "internal-model", + "spend_model": "openai/gpt-4.1", + "spend_model_group": "internal-model", + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("error_type", "logged_tags"), [(ProxyErrorTypes.auth_error, []), (None, ["checkout"])]) +async def test_proxy_only_failure_log_keeps_caller_tags_only_after_auth( + proxy_logging, monkeypatch, error_type: ProxyErrorTypes | None, logged_tags: list[str] +): + from litellm.proxy import proxy_server + + recorded: list[dict] = [] + + class _RecordingLogger(CustomLogger): + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + recorded.append(kwargs) + + monkeypatch.setattr(proxy_server, "llm_router", None) + monkeypatch.setattr(litellm, "callbacks", [_RecordingLogger()]) + proxy_logging.alert_types = [] + + await proxy_logging.post_call_failure_hook( + request_data={ + "model": "internal-model", + "messages": [{"role": "user", "content": "hi"}], + "metadata": {"tags": ["checkout"]}, + }, + original_exception=HTTPException(status_code=401, detail="Invalid proxy server token passed"), + user_api_key_dict=UserAPIKeyAuth(request_route="/chat/completions"), + error_type=error_type, + route="/chat/completions", + ) + + assert len(recorded) == 1 + assert recorded[0]["standard_logging_object"]["request_tags"] == logged_tags + + @pytest.mark.asyncio async def test_post_call_failure_hook_attribution_does_not_count_against_the_deployment( proxy_logging, make_user_api_key_auth, monkeypatch @@ -529,9 +626,8 @@ async def test_post_call_failure_hook_omits_provider_when_a_deployment_does_not_ async def test_handle_logging_proxy_only_path_attributes_with_read_only_metadata( proxy_logging, make_user_api_key_auth, monkeypatch ): - """With a logging object already on the request, a read-only metadata mapping - must not crash the stamp, the failure handler still receives the provider - attribution, and the request's metadata is not copied into the update.""" + """With a logging object already on the request, allowlisted metadata is kept, + a read-only mapping must not crash the stamp, and the failure handler receives provider attribution.""" from litellm.proxy import proxy_server monkeypatch.setattr( @@ -550,16 +646,26 @@ async def test_handle_logging_proxy_only_path_attributes_with_read_only_metadata logging_obj = MagicMock() logging_obj.call_type = "acompletion" logging_obj.model_call_details = {} + logging_obj.litellm_params = {"metadata": {"preexisting": "logged"}} logging_obj.async_failure_handler = AsyncMock() + guardrail_info: Final = [ + { + "guardrail_name": "local-block", + "guardrail_status": "guardrail_intervened", + "guardrail_cost": 0.25, + } + ] await proxy_logging._handle_logging_proxy_only_error( request_data={ "litellm_logging_obj": logging_obj, "model": "internal-model", "messages": [{"role": "user", "content": "hi"}], - "metadata": MappingProxyType({"user_api_key_alias": "frozen"}), + "metadata": MappingProxyType( + {"tags": ["checkout"], "standard_logging_guardrail_information": guardrail_info} + ), }, - user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions"), + user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions", key_alias="frozen"), route="/chat/completions", original_exception=HTTPException(status_code=403, detail="blocked"), ) @@ -568,7 +674,11 @@ async def test_handle_logging_proxy_only_path_attributes_with_read_only_metadata update_kwargs = logging_obj.update_environment_variables.call_args.kwargs assert update_kwargs["custom_llm_provider"] == "openai" assert update_kwargs["litellm_params"]["custom_llm_provider"] == "openai" - assert "user_api_key_alias" not in update_kwargs["litellm_params"]["metadata"] + assert { + key: update_kwargs["litellm_params"]["metadata"][key] for key in ("preexisting", "user_api_key_alias") + } == {"preexisting": "logged", "user_api_key_alias": "frozen"} + assert update_kwargs["litellm_params"]["metadata"]["tags"] == ["checkout"] + assert update_kwargs["litellm_params"]["metadata"]["standard_logging_guardrail_information"] == guardrail_info @pytest.mark.asyncio From 6566067117d7488e63265d6a1e2323a7c5e81bc3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 8 Oct 2026 07:34:00 -0700 Subject: [PATCH 4/6] fix(proxy): always strip client guardrail info and mirror failure attribution into litellm_metadata Backport of #59 to rc/1.105.0. Cherry-picked from 79e48b9b19 (main). This line has no client pricing strip on pass-through, so only the guardrail strip is ported there. --- litellm/proxy/litellm_pre_call_utils.py | 19 ++++++++--- .../pass_through_endpoints.py | 8 +++-- litellm/proxy/utils.py | 1 + .../test_pass_through_endpoints.py | 33 +++++++++++++++++++ tests/unit/proxy/test_pricing_field_strip.py | 31 +++++++++++++++-- ..._utils_model_creation_and_error_logging.py | 1 + 6 files changed, 84 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 3af0f51bd23..332f08d03b3 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -402,10 +402,7 @@ _ALLOW_CLIENT_MESSAGE_REDACTION_OPT_OUT_METADATA_KEY: Final = "allow_client_mess _CLIENT_PRICING_CONTROL_FIELDS: Final = frozenset(CustomPricingLiteLLMParams.model_fields.keys()) | {"base_model"} # ``model_info`` carries the same pricing fields when read by # ``use_custom_pricing_for_model``; strip from metadata for the same reason. -# ``standard_logging_guardrail_information`` is proxy-written telemetry summed -# into response_cost and spend; a client seeding it forges (even negative) -# guardrail cost. -_CLIENT_PRICING_METADATA_FIELDS: Final = frozenset({"model_info", "standard_logging_guardrail_information"}) +_CLIENT_PRICING_METADATA_FIELDS: Final = frozenset({"model_info"}) # ``attempted_fallbacks`` and ``original_model_group`` are written by the router # and read by spend logs as fact; a client value has no legitimate meaning and no # key or team setting keeps it, so the strip is never gated. @@ -661,6 +658,19 @@ def _strip_client_pricing_overrides(data: dict[str, object]) -> None: ) +def strip_client_guardrail_information(data: dict[str, object]) -> None: + """Guardrail hooks write this field and its cost is billed, so no key or team setting keeps a caller's value.""" + for metadata_key in ("metadata", "litellm_metadata"): + if ( + isinstance(metadata := data.get(metadata_key), dict) + and "standard_logging_guardrail_information" in metadata + ): + metadata.pop("standard_logging_guardrail_information") + verbose_proxy_logger.debug( + "Stripped client-supplied guardrail information from request body: %s", metadata_key + ) + + def _strip_router_reserved_metadata( data: dict[str, object], # mutable-ok: strips in place on the request body the pre-call pipeline threads through ) -> None: @@ -2364,6 +2374,7 @@ async def add_litellm_data_to_request( # would silently skip the field. if not _key_or_team_allows_client_pricing_override(user_api_key_dict): _strip_client_pricing_overrides(data) + strip_client_guardrail_information(data) _strip_router_reserved_metadata(data) # Same reason as the strips above: runs after the metadata string-to-dict parse diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index ee958679c1f..b2a7ade4f50 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -117,6 +117,7 @@ from litellm.proxy.litellm_pre_call_utils import ( # noqa: F401 # legacy modul _key_or_team_allows_client_pricing_override, # pyright: ignore[reportPrivateUsage] # reuse the proxy's pricing trust policy _strip_client_pricing_overrides, # pyright: ignore[reportPrivateUsage] # sanitize before trusted hooks add guardrail costs get_dynamic_logging_metadata, # pyright: ignore[reportPrivateUsage] # shared proxy helper, same import style as _read_request_body above + strip_client_guardrail_information, ) from litellm.proxy.route_llm_request import ProxyModelNotFoundError from litellm.proxy.utils import normalize_route_for_root_path @@ -1186,10 +1187,11 @@ async def pass_through_request( _parsed_body, ) + client_body: Final = TypeAdapter(dict[str, object]).validate_python(_parsed_body) + strip_client_guardrail_information(client_body) if not _key_or_team_allows_client_pricing_override(user_api_key_dict): - pricing_body: Final = TypeAdapter(dict[str, object]).validate_python(_parsed_body) - _strip_client_pricing_overrides(pricing_body) - _parsed_body = pricing_body + _strip_client_pricing_overrides(client_body) + _parsed_body = client_body if custom_llm_provider in ("laya", "bespoke"): decision_request: Final = TypeAdapter(Mapping[str, object]).validate_python(_parsed_body) checkpoint: Final = validate_oss_request(custom_llm_provider, decision_request) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index bd9d714f74a..b8eb52bfacd 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -3804,6 +3804,7 @@ class ProxyLogging: key: { **_as_str_mapping(existing.get(key)), **_as_str_mapping(value), + **user_api_key_logged_metadata, } for key, value in _litellm_params.items() if key == "litellm_metadata" diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 7be56cb9162..e73783b0272 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -1383,6 +1383,39 @@ async def test_pass_through_request_uses_resolved_timeout(): assert mock_get_client.call_args[1]["params"]["timeout"] == 1500 +@pytest.mark.asyncio +@pytest.mark.parametrize("allow_client_pricing_override", [True, False]) +async def test_pass_through_request_strips_client_guardrail_information(allow_client_pricing_override: bool): + mock_request = MagicMock(spec=Request) + mock_request.method = "POST" + mock_request.headers = Headers({}) + mock_request.query_params = QueryParams({}) + + with patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging: + mock_proxy_logging.pre_call_hook = AsyncMock(side_effect=RuntimeError("stop before upstream")) + mock_proxy_logging.post_call_failure_hook = AsyncMock() + with pytest.raises(ProxyException): + await pass_through_request( + request=mock_request, + target="http://test.com", + custom_headers={}, + user_api_key_dict=UserAPIKeyAuth( + metadata={"allow_client_pricing_override": allow_client_pricing_override} + ), + custom_body={ + "model": "gpt-4o", + "input_cost_per_token": 0.5, + "metadata": { + "trace_id": "keep-me", + "standard_logging_guardrail_information": [{"guardrail_name": "forged", "guardrail_cost": 5.0}], + }, + }, + ) + + hooked_body: Final = mock_proxy_logging.pre_call_hook.call_args.kwargs["data"] + assert hooked_body["metadata"] == {"trace_id": "keep-me"} + + @pytest.mark.asyncio async def test_create_pass_through_route_forwards_timeout(): unique_path = "/test/path/unique/timeout" diff --git a/tests/unit/proxy/test_pricing_field_strip.py b/tests/unit/proxy/test_pricing_field_strip.py index 3e9bee9217d..c4a14b39f54 100644 --- a/tests/unit/proxy/test_pricing_field_strip.py +++ b/tests/unit/proxy/test_pricing_field_strip.py @@ -22,6 +22,7 @@ from litellm.proxy.litellm_pre_call_utils import ( _CLIENT_PRICING_METADATA_FIELDS, _strip_client_pricing_overrides, add_litellm_data_to_request, + strip_client_guardrail_information, ) from litellm.types.utils import CustomPricingLiteLLMParams @@ -133,7 +134,7 @@ class TestStripClientPricingOverrides: "standard_logging_guardrail_information": [{"guardrail_cost": 5.0}], }, } - _strip_client_pricing_overrides(data) + strip_client_guardrail_information(data) assert data["metadata"] == {"user_session": "keep-me"} assert data["litellm_metadata"] == {} @@ -164,7 +165,6 @@ class TestStripClientPricingOverrides: def test_metadata_field_set_contains_model_info(self): assert "model_info" in _CLIENT_PRICING_METADATA_FIELDS - assert "standard_logging_guardrail_information" in _CLIENT_PRICING_METADATA_FIELDS def test_strip_emits_debug_log_listing_dropped_fields(self, caplog): # Operators need a paper trail so they can diagnose why a previously @@ -315,6 +315,33 @@ async def test_add_litellm_data_to_request_skips_strip_with_key_opt_in(): assert updated["base_model"] == "gpt-4o-mini" +@pytest.mark.asyncio +@pytest.mark.parametrize("opt_in_field", ["metadata", "team_metadata"]) +async def test_add_litellm_data_to_request_strips_guardrail_information_despite_pricing_opt_in(opt_in_field): + data = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "hi"}], + "input_cost_per_token": 0.0001, + "metadata": { + "user_session": "keep-me", + "standard_logging_guardrail_information": [{"guardrail_name": "forged", "guardrail_cost": 5.0}], + }, + } + + updated = await add_litellm_data_to_request( + data=data, + request=_make_request_mock(), + user_api_key_dict=_user_api_key_auth(**{opt_in_field: {"allow_client_pricing_override": True}}), + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + assert updated["input_cost_per_token"] == 0.0001 + assert updated["metadata"]["user_session"] == "keep-me" + assert "standard_logging_guardrail_information" not in updated["metadata"] + + @pytest.mark.asyncio async def test_add_litellm_data_to_request_strips_json_string_litellm_metadata(): """``litellm_metadata`` may arrive as a JSON-encoded string (multipart/ diff --git a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py index 33b6718893d..3d647baa640 100644 --- a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py +++ b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py @@ -186,6 +186,7 @@ async def test_proxy_only_error_log_keeps_tags_and_drops_caller_logging_fields( assert "caller" not in str(details["litellm_params"]) assert details["litellm_params"][metadata_field]["tags"] == ["t"] assert details["litellm_params"]["metadata"]["user_api_key_team_id"] == "team-a" + assert details["litellm_params"][metadata_field]["user_api_key_team_id"] == "team-a" @pytest.mark.asyncio From fb0506db578f79bfb8543b4b5c21cc7536205d17 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 8 Oct 2026 09:26:37 -0700 Subject: [PATCH 5/6] fix(proxy): merge only dict client metadata into pass-through logging Backport adaptation for rc/1.105.0. main merges a pass-through caller's metadata and litellm_metadata into the logging metadata only when they are dicts. This line merged any value, so a list of pairs could carry standard_logging_guardrail_information past the guardrail strip from #59. --- .../test_pass_through_endpoints.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index e73783b0272..2d81656fdb1 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -1416,6 +1416,32 @@ async def test_pass_through_request_strips_client_guardrail_information(allow_cl assert hooked_body["metadata"] == {"trace_id": "keep-me"} +@pytest.mark.parametrize("metadata_key", ["litellm_metadata", "metadata"]) +def test_pass_through_logging_ignores_client_guardrail_information_sent_as_pairs(metadata_key: str): + mock_request = MagicMock(spec=Request) + mock_request.method = "POST" + mock_request.url = httpx.URL("http://0.0.0.0:4000/openai/v1/chat/completions") + mock_request.headers = Headers({}) + mock_request.scope = {} + + kwargs: Final = HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint( + request=mock_request, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + passthrough_logging_payload=MagicMock(), + logging_obj=MagicMock(), + _parsed_body={ + metadata_key: [ + ["trace_id", "from-pairs"], + ["standard_logging_guardrail_information", [{"guardrail_name": "forged", "guardrail_cost": 5.0}]], + ], + }, + ) + + logged_metadata: Final = kwargs["litellm_params"]["metadata"] + assert "standard_logging_guardrail_information" not in logged_metadata + assert "trace_id" not in logged_metadata + + @pytest.mark.asyncio async def test_create_pass_through_route_forwards_timeout(): unique_path = "/test/path/unique/timeout" From 5197c9733f15c991f4b980bd77a5c0054163fdfa Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Tue, 27 Oct 2026 10:45:45 -0700 Subject: [PATCH 6/6] bump: version 1.106.5 --- pyproject.toml | 4 ++-- uv.lock | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b5fed81c0c4..34b7361b640 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm" -version = "1.106.4" +version = "1.106.5" description = "Library to easily interface with LLM API providers" readme = "README.md" requires-python = ">=3.10, <3.15" @@ -379,7 +379,7 @@ litellm-enterprise = { workspace = true } members = ["enterprise", "litellm-proxy-extras"] [tool.commitizen] -version = "1.106.4" +version = "1.106.5" version_files = [ "pyproject.toml:^version", ] diff --git a/uv.lock b/uv.lock index 253ae49c8bd..1770cbf2a28 100644 --- a/uv.lock +++ b/uv.lock @@ -4540,7 +4540,7 @@ wheels = [ [[package]] name = "litellm" -version = "1.106.4" +version = "1.106.5" source = { editable = "." } dependencies = [ { name = "aiohttp" },