scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-06 08:16:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 78

fix(spend_tracking): key /v1/messages spend rows on the msg_ id the client received

Browse source 
POST /v1/messages returns an Anthropic-shaped body whose `id` is the only
request id the caller ever sees, but the spend row was written with a
`chatcmpl-<uuid>` (non-streaming) or the bare `litellm_call_id` (streaming and
the /anthropic/v1/messages passthrough), so
GET /spend/logs?request_id=msg_... returned [].

The logging conversion now carries the provider's response id through:
_handle_anthropic_messages_response_logging seeds the ModelResponse it builds
with the Anthropic id, and the passthrough logging handler prefers the id it
read off the response body or the message_start chunk over litellm_call_id.
get_spend_logs_id already prefers response_obj["id"], so the spend row and
standard_logging_object["id"] now both carry the id the client holds.
This commit is contained in:
mateo-berri 2026-09-03 00:26:18 -07:00
parent ff17e8b987
commit 46d7e92845
3 changed files with 190 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
litellm/litellm_core_utils/litellm_logging.py
View file

@ -414,6 +414,11 @@ def _resolve_vertex_location_for_cost(
return VertexBase.get_vertex_region(configured_location, model)
def _anthropic_response_id(source: object) -> str | None:
candidate: Final = source.get("id") if isinstance(source, dict) else getattr(source, "id", None)
return candidate if isinstance(candidate, str) and candidate else None
class Logging(LiteLLMLoggingBaseClass):
global \
supabaseClient, \
@ -3832,11 +3837,12 @@ class Logging(LiteLLMLoggingBaseClass):
if isinstance(result, ResponsesAPIResponse):
return self._translate_responses_api_response_to_model_response(result)
anthropic_response_id: Final = _anthropic_response_id(result)
httpx_response: Final = self.model_call_details.get("httpx_response", None)
if httpx_response and isinstance(httpx_response, httpx.Response):
result = litellm.AnthropicConfig().transform_response(
raw_response=httpx_response,
model_response=litellm.ModelResponse(),
model_response=litellm.ModelResponse(id=anthropic_response_id),
model=self.model,
messages=[],
logging_obj=self,
@ -3859,7 +3865,7 @@ class Logging(LiteLLMLoggingBaseClass):
status_code=200,
headers={},
),
model_response=litellm.ModelResponse(),
model_response=litellm.ModelResponse(id=anthropic_response_id),
json_mode=None,
speed=self.optional_params.get("speed") if self.optional_params else None,
)

27
litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py
View file

@ -107,6 +107,7 @@ class AnthropicPassthroughLoggingHandler:
start_time=start_time,
end_time=end_time,
logging_obj=logging_obj,
response_id=optional_str(response_body.get("id")),
)
return {
@ -148,8 +149,9 @@ class AnthropicPassthroughLoggingHandler:
return model
@staticmethod
def _extract_model_from_anthropic_chunks(
def _extract_message_start_field(
all_chunks: Sequence[str | bytes],
field: str,
) -> str | None:
for raw in all_chunks:
text = raw.decode("utf-8") if isinstance(raw, bytes) else raw
@ -163,11 +165,23 @@ class AnthropicPassthroughLoggingHandler:
if not isinstance(data, dict):
continue
if data.get("type") == "message_start":
model = (data.get("message") or {}).get("model")
if model:
return model
value = (data.get("message") or {}).get(field)
if isinstance(value, str) and value:
return value
return None
@staticmethod
def _extract_model_from_anthropic_chunks(
all_chunks: Sequence[str | bytes],
) -> str | None:
return AnthropicPassthroughLoggingHandler._extract_message_start_field(all_chunks, "model")
@staticmethod
def _extract_response_id_from_anthropic_chunks(
all_chunks: Sequence[str | bytes],
) -> str | None:
return AnthropicPassthroughLoggingHandler._extract_message_start_field(all_chunks, "id")
@staticmethod
def _stream_was_interrupted(
all_chunks: Sequence[str | bytes],
@ -251,6 +265,7 @@ class AnthropicPassthroughLoggingHandler:
start_time: datetime,
end_time: datetime,
logging_obj: LiteLLMLoggingObj,
response_id: str | None = None,
):
"""
Create the standard logging object for Anthropic passthrough
@ -312,8 +327,7 @@ class AnthropicPassthroughLoggingHandler:
json.dumps(kwargs, indent=4, default=str),
)
# set litellm_call_id to logging response object
litellm_model_response.id = logging_obj.litellm_call_id
litellm_model_response.id = response_id or logging_obj.litellm_call_id
litellm_model_response.model = model
logging_obj.model_call_details["model"] = model
if not logging_obj.model_call_details.get("custom_llm_provider"):
@ -413,6 +427,7 @@ class AnthropicPassthroughLoggingHandler:
start_time=start_time,
end_time=end_time,
logging_obj=litellm_logging_obj,
response_id=AnthropicPassthroughLoggingHandler._extract_response_id_from_anthropic_chunks(all_chunks),
)
return {

161
tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
View file

@ -4025,3 +4025,164 @@ def test_caller_forged_router_metadata_is_discarded(bucket):
)
metadata = json.loads(payload["metadata"])
assert metadata["router_metadata"] is None
ANTHROPIC_MESSAGES_RESPONSE: Final = {
"id": "msg_01Lit6806NonStreaming",
"type": "message",
"role": "assistant",
"model": "claude-haiku-4-5",
"content": [{"type": "text", "text": "epsilon"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 14, "output_tokens": 4},
}
ANTHROPIC_MESSAGES_SSE_CHUNKS: Final = (
'event: message_start\ndata: {"type":"message_start","message":{"id":"msg_01Lit6806Streaming",'
'"type":"message","role":"assistant","model":"claude-haiku-4-5","content":[],'
'"usage":{"input_tokens":14,"output_tokens":1}}}\n\n',
'event: content_block_start\ndata: {"type":"content_block_start","index":0,'
'"content_block":{"type":"text","text":""}}\n\n',
'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,'
'"delta":{"type":"text_delta","text":"epsilon"}}\n\n',
'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}\n\n',
'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn"},'
'"usage":{"output_tokens":4}}\n\n',
"event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n",
)
def _anthropic_messages_logging_obj(*, stream: bool) -> Any:
from litellm.litellm_core_utils.litellm_logging import Logging
logging_obj = Logging(
model="claude-haiku-4-5",
messages=[{"role": "user", "content": "hi"}],
stream=stream,
call_type="anthropic_messages",
start_time=datetime.datetime.now(timezone.utc),
litellm_call_id="6806cafe-0000-4000-8000-000000000001",
function_id="1234",
)
logging_obj.optional_params = {}
logging_obj.model_call_details["custom_llm_provider"] = "anthropic"
return logging_obj
def _spend_log_request_id(response_obj: Any, kwargs: dict) -> str:
payload = get_logging_payload(
kwargs=kwargs,
response_obj=response_obj,
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
return payload["request_id"]
def test_spend_log_request_id_is_the_message_id_a_non_streaming_messages_caller_received():
"""
POST /v1/messages hands the caller `id: msg_...`, the only request id they ever see, so
GET /spend/logs?request_id=msg_... has to find the row.
"""
logging_obj = _anthropic_messages_logging_obj(stream=False)
logged_response = logging_obj._handle_anthropic_messages_response_logging(
result=ANTHROPIC_MESSAGES_RESPONSE
)
assert logged_response.id == "msg_01Lit6806NonStreaming"
assert (
_spend_log_request_id(
response_obj=logged_response,
kwargs={
"call_type": "anthropic_messages",
"model": "claude-haiku-4-5",
"litellm_call_id": "6806cafe-0000-4000-8000-000000000001",
"litellm_params": {"metadata": {"user_api_key": "test-key"}},
},
)
== "msg_01Lit6806NonStreaming"
)
def test_spend_log_request_id_is_the_message_id_a_streaming_messages_caller_received():
"""
The streaming leg of /v1/messages logs through the Anthropic passthrough handler, which used
to stamp litellm_call_id over the msg_ id carried by the message_start event.
"""
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import (
AnthropicPassthroughLoggingHandler,
)
from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType
logging_obj = _anthropic_messages_logging_obj(stream=True)
logging_obj.model_call_details["stream"] = True
logged = AnthropicPassthroughLoggingHandler._handle_logging_anthropic_collected_chunks(
litellm_logging_obj=logging_obj,
passthrough_success_handler_obj=MagicMock(),
url_route="/v1/messages",
request_body={"model": "claude-haiku-4-5"},
endpoint_type=EndpointType.ANTHROPIC,
start_time=datetime.datetime.now(timezone.utc),
all_chunks=list(ANTHROPIC_MESSAGES_SSE_CHUNKS),
end_time=datetime.datetime.now(timezone.utc),
)
assert logged["result"].id == "msg_01Lit6806Streaming"
assert (
_spend_log_request_id(
response_obj=logged["result"],
kwargs={
**logged["kwargs"],
"call_type": "anthropic_messages",
"litellm_call_id": "6806cafe-0000-4000-8000-000000000001",
"litellm_params": {"metadata": {"user_api_key": "test-key"}},
},
)
== "msg_01Lit6806Streaming"
)
def test_spend_log_request_id_still_falls_back_to_litellm_call_id_without_a_provider_id():
"""
Anthropic-compatible upstreams that omit `id` must keep landing on litellm_call_id rather
than on a fresh chatcmpl- uuid nobody can look up.
"""
logging_obj = _anthropic_messages_logging_obj(stream=True)
logging_obj.model_call_details["stream"] = True
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import (
AnthropicPassthroughLoggingHandler,
)
AnthropicPassthroughLoggingHandler._create_anthropic_response_logging_payload(
litellm_model_response=litellm.ModelResponse(id="chatcmpl-generated"),
model="claude-haiku-4-5",
kwargs={},
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
logging_obj=logging_obj,
)
assert logging_obj.model_call_details["complete_streaming_response"].id == (
"6806cafe-0000-4000-8000-000000000001"
)
def test_spend_log_request_id_for_chat_completions_is_untouched():
"""
/v1/chat/completions callers look their rows up by the chatcmpl- id in the response body.
"""
assert (
_spend_log_request_id(
response_obj=litellm.ModelResponse(id="chatcmpl-EJvWIw3DAhuKYuwp3jJI4Pnhp2vjv", choices=[]),
kwargs={
"call_type": "acompletion",
"model": "gpt-5.6",
"litellm_call_id": "6806cafe-0000-4000-8000-000000000002",
"litellm_params": {"metadata": {"user_api_key": "test-key"}},
},
)
== "chatcmpl-EJvWIw3DAhuKYuwp3jJI4Pnhp2vjv"
)