scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-21 00:21:49 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 74

fix(proxy): placeholder the metadata copied into a placeholdered row's stored request body

Browse source
This commit is contained in:
mateo-berri 2026-09-19 03:56:44 -07:00
parent a49fbc6272
commit d437cd662b
2 changed files with 102 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

48
litellm/proxy/spend_tracking/spend_tracking_utils.py
View file

@ -758,7 +758,9 @@ def get_logging_payload(
proxy_server_request=_get_proxy_server_request_for_spend_logs_payload(
metadata=metadata,
litellm_params=(
_placeholder_stored_request_body_model(litellm_params) if model_is_placeholdered else litellm_params
_placeholder_stored_request_body(litellm_params, persisted_model_group, raw_model)
if model_is_placeholdered
else litellm_params
),
kwargs=kwargs,
),
@ -1066,7 +1068,7 @@ def _sanitize_request_body_for_spend_logs_payload(
visited.add(obj_id)
def _sanitize_value(value: object) -> object:
if isinstance(value, dict):
if isinstance(value, Mapping):
return _sanitize_request_body_for_spend_logs_payload(value, visited, max_string_length_prompt_in_db)
elif isinstance(value, list):
return [_sanitize_value(item) for item in value]
@ -1420,20 +1422,56 @@ def _convert_mapping_to_json_serializable(obj: Mapping[str, object]) -> dict[str
return dict(obj)
def _placeholder_stored_request_body_model(litellm_params: Mapping[str, object]) -> Mapping[str, object]:
def _placeholder_stored_request_body_metadata(
request_body: Mapping[str, object], persisted_model_group: str, raw_model: str
) -> Mapping[str, object]:
body_metadata: Final = request_body.get("metadata")
if not isinstance(body_metadata, Mapping):
return request_body
error_information: Final = body_metadata.get("error_information")
placeholdered_fields: Final = MappingProxyType(
{
"model_group": persisted_model_group,
"error_information": _scrub_raw_model_from_error_information(
cast(StandardLoggingPayloadErrorInformation, error_information), raw_model
)
if isinstance(error_information, Mapping)
else error_information,
}
)
return MappingProxyType(
{
**request_body,
"metadata": MappingProxyType(
{key: placeholdered_fields.get(key, value) for key, value in body_metadata.items()}
),
}
)
def _placeholder_stored_request_body(
litellm_params: Mapping[str, object], persisted_model_group: str, raw_model: str
) -> Mapping[str, object]:
proxy_server_request: Final = litellm_params.get("proxy_server_request")
if not isinstance(proxy_server_request, Mapping):
return litellm_params
request_body: Final = proxy_server_request.get("body")
if not isinstance(request_body, Mapping) or "model" not in request_body:
if not isinstance(request_body, Mapping):
return litellm_params
model_placeholdered: Final = (
MappingProxyType({**request_body, "model": UNKNOWN_MODEL_SPEND_LOG_MODEL})
if "model" in request_body
else request_body
)
return MappingProxyType(
{
**litellm_params,
"proxy_server_request": MappingProxyType(
{
**proxy_server_request,
"body": MappingProxyType({**request_body, "model": UNKNOWN_MODEL_SPEND_LOG_MODEL}),
"body": _placeholder_stored_request_body_metadata(
model_placeholdered, persisted_model_group, raw_model
),
}
),
}

59
tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
View file

@ -1151,6 +1151,65 @@ def test_get_logging_payload_placeholders_the_stored_request_body_model_only_whe
assert stored_request_body["model"] == expected_stored_model
@pytest.mark.parametrize(
("deployment_info", "expected_stored_model_group", "expected_stored_error_message"),
[
({}, "", f"Invalid value for 'model' = {UNKNOWN_MODEL_SPEND_LOG_MODEL}"),
(
{"model_info": {"id": "routed-deployment"}},
_RAW_MODEL_WITH_PROMPT,
f"Invalid value for 'model' = {_RAW_MODEL_WITH_PROMPT}",
),
],
)
def test_get_logging_payload_placeholders_the_metadata_copied_into_the_stored_request_body(
monkeypatch: pytest.MonkeyPatch,
deployment_info: dict[str, object],
expected_stored_model_group: str,
expected_stored_error_message: str,
):
from litellm.proxy import proxy_server
monkeypatch.setattr(proxy_server, "general_settings", {"store_prompts_in_spend_logs": True})
metadata: Final = {
"user_api_key": "sk-test",
"status": "failure",
"model_group": _RAW_MODEL_WITH_PROMPT,
"error_information": {
"error_code": "400",
"error_class": "BadRequestError",
"llm_provider": "openai",
"error_message": f"Invalid value for 'model' = {_RAW_MODEL_WITH_PROMPT}",
"traceback": "",
},
**deployment_info,
}
kwargs: Final = {
"model": _RAW_MODEL_WITH_PROMPT,
"call_type": "amoderation",
"litellm_params": {
"metadata": metadata,
"proxy_server_request": {
"url": "http://localhost:4000/v1/moderations",
"body": {"input": "hi", "model": _RAW_MODEL_WITH_PROMPT, "metadata": metadata},
},
},
}
payload: Final = get_logging_payload(
kwargs=kwargs,
response_obj=ValueError("Invalid value for 'model'"),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
stored_request_body: Final = json.loads(payload["proxy_server_request"])
assert stored_request_body["metadata"]["model_group"] == expected_stored_model_group
assert stored_request_body["metadata"]["error_information"]["error_message"] == expected_stored_error_message
assert stored_request_body["metadata"]["user_api_key"] == "sk-test"
assert ("medical records" in payload["proxy_server_request"]) == bool(deployment_info)
_WHITESPACE_MODEL_GROUP: Final = "Broken GPT Mini"
_WHITESPACE_MODEL_GROUP_ALIAS: Final = "Broken GPT Alias"
_COOLDOWN_ERROR_MESSAGE: Final = (