mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-21 00:21:49 +00:00
fix(spend_tracking): carry the request path in collector events that omit bodies
The collector event dropped proxy_server_request whenever spend logs do not store bodies, so the sidecar could not tell a file or response read from a minted response and re-keyed a second read into its own row. The event now always carries the request's method and path, and nothing else of the request: no host, query, headers or body.
This commit is contained in:
parent
b140dffd7e
commit
c6b62515af
2 changed files with 29 additions and 4 deletions
|
|
@ -6,7 +6,7 @@ callback's ``kwargs`` into the projection ``_PROXY_track_cost_callback`` and
|
||||||
``DBSpendUpdateWriter.update_database`` actually read: identities and metadata, timings, usage, the
|
``DBSpendUpdateWriter.update_database`` actually read: identities and metadata, timings, usage, the
|
||||||
standard logging payload without its prompt/response bodies, and the tool names. The request
|
standard logging payload without its prompt/response bodies, and the tool names. The request
|
||||||
messages, the raw ``proxy_server_request`` body and the full response travel only when spend logs
|
messages, the raw ``proxy_server_request`` body and the full response travel only when spend logs
|
||||||
are configured to store prompts and responses. The cache key is the preset key the caching layer
|
are configured to store prompts and responses; the request's method and path always travel. The cache key is the preset key the caching layer
|
||||||
already computed, never a fresh hash over the request body.
|
already computed, never a fresh hash over the request body.
|
||||||
|
|
||||||
``spend_event_callback_args`` rebuilds the ``(kwargs, response_obj, start_time, end_time)`` tuple
|
``spend_event_callback_args`` rebuilds the ``(kwargs, response_obj, start_time, end_time)`` tuple
|
||||||
|
|
@ -18,6 +18,7 @@ from dataclasses import dataclass
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from types import MappingProxyType
|
from types import MappingProxyType
|
||||||
from typing import Final, Literal, TypeAlias
|
from typing import Final, Literal, TypeAlias
|
||||||
|
from urllib.parse import urlsplit
|
||||||
|
|
||||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
|
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
|
||||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||||
|
|
@ -247,6 +248,19 @@ def _metadata_for_event(
|
||||||
return MappingProxyType({**kept, "user_api_key_budget_reservation": budget_reservation})
|
return MappingProxyType({**kept, "user_api_key_budget_reservation": budget_reservation})
|
||||||
|
|
||||||
|
|
||||||
|
def _request_route_for_event(proxy_server_request: object) -> ObjectMapping | None:
|
||||||
|
"""The request's method and path alone: what the spend row needs to tell a read of a stored
|
||||||
|
object (its id in the path) from a response minted for this call, with no host, query,
|
||||||
|
headers or body."""
|
||||||
|
request: Final = _mapping_or_none(proxy_server_request)
|
||||||
|
if request is None:
|
||||||
|
return None
|
||||||
|
url: Final = request.get("url")
|
||||||
|
if not isinstance(url, str):
|
||||||
|
return None
|
||||||
|
return MappingProxyType({"url": urlsplit(url).path, "method": request.get("method")})
|
||||||
|
|
||||||
|
|
||||||
def _litellm_params_for_event(
|
def _litellm_params_for_event(
|
||||||
litellm_params: _LitellmParams, cache_key: str | None, store_bodies: bool
|
litellm_params: _LitellmParams, cache_key: str | None, store_bodies: bool
|
||||||
) -> _LitellmParams:
|
) -> _LitellmParams:
|
||||||
|
|
@ -267,7 +281,11 @@ def _litellm_params_for_event(
|
||||||
"user_api_key_end_user_id": litellm_params.get("user_api_key_end_user_id"),
|
"user_api_key_end_user_id": litellm_params.get("user_api_key_end_user_id"),
|
||||||
"metadata": _metadata_for_event(metadata, budget_reservation),
|
"metadata": _metadata_for_event(metadata, budget_reservation),
|
||||||
"litellm_metadata": _metadata_for_event(litellm_metadata, budget_reservation),
|
"litellm_metadata": _metadata_for_event(litellm_metadata, budget_reservation),
|
||||||
"proxy_server_request": litellm_params.get("proxy_server_request") if store_bodies else None,
|
"proxy_server_request": (
|
||||||
|
litellm_params.get("proxy_server_request")
|
||||||
|
if store_bodies
|
||||||
|
else _request_route_for_event(litellm_params.get("proxy_server_request"))
|
||||||
|
),
|
||||||
"preset_cache_key": cache_key,
|
"preset_cache_key": cache_key,
|
||||||
}
|
}
|
||||||
return projected
|
return projected
|
||||||
|
|
|
||||||
|
|
@ -63,7 +63,12 @@ def _success_kwargs(preset_cache_key: str | None = "preset-key") -> dict:
|
||||||
"litellm_params": {
|
"litellm_params": {
|
||||||
"api_base": "https://api.openai.com",
|
"api_base": "https://api.openai.com",
|
||||||
"preset_cache_key": preset_cache_key,
|
"preset_cache_key": preset_cache_key,
|
||||||
"proxy_server_request": {"body": {"messages": [{"role": "user", "content": _BIG_PROMPT}]}},
|
"proxy_server_request": {
|
||||||
|
"url": "http://litellm:4000/v1/chat/completions?api-version=2026-01-01",
|
||||||
|
"method": "POST",
|
||||||
|
"headers": {"authorization": "Bearer sk-secret"},
|
||||||
|
"body": {"messages": [{"role": "user", "content": _BIG_PROMPT}]},
|
||||||
|
},
|
||||||
"metadata": {
|
"metadata": {
|
||||||
"user_api_key": "hash-1",
|
"user_api_key": "hash-1",
|
||||||
"user_api_key_user_id": "user-1",
|
"user_api_key_user_id": "user-1",
|
||||||
|
|
@ -110,7 +115,9 @@ def test_event_is_compact_and_omits_bodies_by_default():
|
||||||
decoded: Final = json.loads(line)
|
decoded: Final = json.loads(line)
|
||||||
assert "messages" not in decoded["standard_logging_object"]
|
assert "messages" not in decoded["standard_logging_object"]
|
||||||
assert "response" not in decoded["standard_logging_object"]
|
assert "response" not in decoded["standard_logging_object"]
|
||||||
assert decoded["litellm_params"]["proxy_server_request"] is None
|
assert decoded["litellm_params"]["proxy_server_request"] == {"url": "/v1/chat/completions", "method": "POST"}
|
||||||
|
assert b"sk-secret" not in line
|
||||||
|
assert b"api-version" not in line
|
||||||
|
|
||||||
|
|
||||||
def test_event_carries_bodies_when_spend_logs_store_them():
|
def test_event_carries_bodies_when_spend_logs_store_them():
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue