From d437cd662be2d781c63c8a14adf6af95c0ad9ff1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 19 Sep 2026 03:56:44 -0700 Subject: [PATCH] fix(proxy): placeholder the metadata copied into a placeholdered row's stored request body --- .../spend_tracking/spend_tracking_utils.py | 48 +++++++++++++-- .../test_spend_tracking_utils.py | 59 +++++++++++++++++++ 2 files changed, 102 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 055e128e0c4..9756844b587 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -758,7 +758,9 @@ def get_logging_payload( proxy_server_request=_get_proxy_server_request_for_spend_logs_payload( metadata=metadata, litellm_params=( - _placeholder_stored_request_body_model(litellm_params) if model_is_placeholdered else litellm_params + _placeholder_stored_request_body(litellm_params, persisted_model_group, raw_model) + if model_is_placeholdered + else litellm_params ), kwargs=kwargs, ), @@ -1066,7 +1068,7 @@ def _sanitize_request_body_for_spend_logs_payload( visited.add(obj_id) def _sanitize_value(value: object) -> object: - if isinstance(value, dict): + if isinstance(value, Mapping): return _sanitize_request_body_for_spend_logs_payload(value, visited, max_string_length_prompt_in_db) elif isinstance(value, list): return [_sanitize_value(item) for item in value] @@ -1420,20 +1422,56 @@ def _convert_mapping_to_json_serializable(obj: Mapping[str, object]) -> dict[str return dict(obj) -def _placeholder_stored_request_body_model(litellm_params: Mapping[str, object]) -> Mapping[str, object]: +def _placeholder_stored_request_body_metadata( + request_body: Mapping[str, object], persisted_model_group: str, raw_model: str +) -> Mapping[str, object]: + body_metadata: Final = request_body.get("metadata") + if not isinstance(body_metadata, Mapping): + return request_body + error_information: Final = body_metadata.get("error_information") + placeholdered_fields: Final = MappingProxyType( + { + "model_group": persisted_model_group, + "error_information": _scrub_raw_model_from_error_information( + cast(StandardLoggingPayloadErrorInformation, error_information), raw_model + ) + if isinstance(error_information, Mapping) + else error_information, + } + ) + return MappingProxyType( + { + **request_body, + "metadata": MappingProxyType( + {key: placeholdered_fields.get(key, value) for key, value in body_metadata.items()} + ), + } + ) + + +def _placeholder_stored_request_body( + litellm_params: Mapping[str, object], persisted_model_group: str, raw_model: str +) -> Mapping[str, object]: proxy_server_request: Final = litellm_params.get("proxy_server_request") if not isinstance(proxy_server_request, Mapping): return litellm_params request_body: Final = proxy_server_request.get("body") - if not isinstance(request_body, Mapping) or "model" not in request_body: + if not isinstance(request_body, Mapping): return litellm_params + model_placeholdered: Final = ( + MappingProxyType({**request_body, "model": UNKNOWN_MODEL_SPEND_LOG_MODEL}) + if "model" in request_body + else request_body + ) return MappingProxyType( { **litellm_params, "proxy_server_request": MappingProxyType( { **proxy_server_request, - "body": MappingProxyType({**request_body, "model": UNKNOWN_MODEL_SPEND_LOG_MODEL}), + "body": _placeholder_stored_request_body_metadata( + model_placeholdered, persisted_model_group, raw_model + ), } ), } diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 50f4d2dcf5a..0004711954a 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -1151,6 +1151,65 @@ def test_get_logging_payload_placeholders_the_stored_request_body_model_only_whe assert stored_request_body["model"] == expected_stored_model +@pytest.mark.parametrize( + ("deployment_info", "expected_stored_model_group", "expected_stored_error_message"), + [ + ({}, "", f"Invalid value for 'model' = {UNKNOWN_MODEL_SPEND_LOG_MODEL}"), + ( + {"model_info": {"id": "routed-deployment"}}, + _RAW_MODEL_WITH_PROMPT, + f"Invalid value for 'model' = {_RAW_MODEL_WITH_PROMPT}", + ), + ], +) +def test_get_logging_payload_placeholders_the_metadata_copied_into_the_stored_request_body( + monkeypatch: pytest.MonkeyPatch, + deployment_info: dict[str, object], + expected_stored_model_group: str, + expected_stored_error_message: str, +): + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "general_settings", {"store_prompts_in_spend_logs": True}) + metadata: Final = { + "user_api_key": "sk-test", + "status": "failure", + "model_group": _RAW_MODEL_WITH_PROMPT, + "error_information": { + "error_code": "400", + "error_class": "BadRequestError", + "llm_provider": "openai", + "error_message": f"Invalid value for 'model' = {_RAW_MODEL_WITH_PROMPT}", + "traceback": "", + }, + **deployment_info, + } + kwargs: Final = { + "model": _RAW_MODEL_WITH_PROMPT, + "call_type": "amoderation", + "litellm_params": { + "metadata": metadata, + "proxy_server_request": { + "url": "http://localhost:4000/v1/moderations", + "body": {"input": "hi", "model": _RAW_MODEL_WITH_PROMPT, "metadata": metadata}, + }, + }, + } + + payload: Final = get_logging_payload( + kwargs=kwargs, + response_obj=ValueError("Invalid value for 'model'"), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + + stored_request_body: Final = json.loads(payload["proxy_server_request"]) + assert stored_request_body["metadata"]["model_group"] == expected_stored_model_group + assert stored_request_body["metadata"]["error_information"]["error_message"] == expected_stored_error_message + assert stored_request_body["metadata"]["user_api_key"] == "sk-test" + assert ("medical records" in payload["proxy_server_request"]) == bool(deployment_info) + + _WHITESPACE_MODEL_GROUP: Final = "Broken GPT Mini" _WHITESPACE_MODEL_GROUP_ALIAS: Final = "Broken GPT Alias" _COOLDOWN_ERROR_MESSAGE: Final = (