scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-11 22:51:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 124

[Feature] UI - Logs: Show retry count for requests

Browse source 
Add attempted_retries and max_retries fields to SpendLogsMetadata so the
Logs page can display how many retries occurred for each request. The
router now injects retry tracking metadata before each make_call, which
flows through the logging pipeline into the spend logs metadata JSON.

The UI shows "Not Retried" when the first attempt succeeded, and
"N / M" (attempted / max) when retries occurred. The field is hidden
for requests that did not go through the router.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
yuneng-jiang 2026-02-20 13:51:20 -08:00
parent d5fa49fbb7
commit de1517411f
7 changed files with 277 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
litellm/proxy/_types.py
View file

@ -3044,6 +3044,8 @@ class SpendLogsMetadata(TypedDict):
str
] # S3/GCS object key for cold storage retrieval
litellm_overhead_time_ms: Optional[float] # LiteLLM overhead time in milliseconds
attempted_retries: Optional[int] # Number of retries attempted (0 = first attempt succeeded)
max_retries: Optional[int] # Max retries configured for this request
cost_breakdown: Optional[
CostBreakdown
] # Detailed cost breakdown (input_cost, output_cost, margin, discount, etc.)

2
litellm/proxy/spend_tracking/spend_tracking_utils.py
View file

@ -86,6 +86,8 @@ def _get_spend_logs_metadata(
guardrail_information=None,
cold_storage_object_key=cold_storage_object_key,
litellm_overhead_time_ms=None,
attempted_retries=None,
max_retries=None,
cost_breakdown=None,
)
verbose_proxy_logger.debug(

6
litellm/router.py
View file

@ -5128,6 +5128,9 @@ class Router:
verbose_router_logger.debug(
f"async function w/ retries: original_function - {original_function}, num_retries - {num_retries}"
)
## ADD RETRY TRACKING TO METADATA - used for spend logs retry tracking
_metadata["attempted_retries"] = 0
_metadata["max_retries"] = num_retries
try:
self._handle_mock_testing_rate_limit_error(
model_group=model_group, kwargs=kwargs
@ -5215,6 +5218,9 @@ class Router:
for current_attempt in range(num_retries):
try:
# Update retry tracking metadata before each retry attempt
_metadata["attempted_retries"] = current_attempt + 1
_metadata["max_retries"] = num_retries
# if the function call is successful, no exception will be raised and we'll break out of the loop
response = await self.make_call(original_function, *args, **kwargs)
if coroutine_checker.is_async_callable(

201
tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
View file

@ -1031,3 +1031,204 @@ def test_get_logging_payload_guardrail_info_when_no_standard_logging_payload():
metadata_result = json.loads(payload["metadata"])
assert metadata_result["guardrail_information"] == guardrail_info
@patch("litellm.proxy.proxy_server.master_key", None)
@patch("litellm.proxy.proxy_server.general_settings", {})
def test_get_logging_payload_includes_retry_info_in_spend_logs_metadata():
"""
Test that retry info (attempted_retries, max_retries) from metadata
is included in the spend logs metadata JSON.
"""
kwargs = {
"model": "gpt-3.5-turbo",
"litellm_params": {
"metadata": {
"user_api_key": "sk-test-key",
"attempted_retries": 2,
"max_retries": 3,
}
},
"standard_logging_object": StandardLoggingPayload(
id="test-retry-123",
call_type="completion",
stream=False,
response_cost=0.001,
status="success",
total_tokens=100,
prompt_tokens=50,
completion_tokens=50,
startTime=1234567890.0,
endTime=1234567891.0,
completionStartTime=None,
model_map_information=StandardLoggingModelInformation(
model_map_key="gpt-3.5-turbo", model_map_value=None
),
model="gpt-3.5-turbo",
model_id="model-123",
model_group="openai",
custom_llm_provider="openai",
api_base="https://api.openai.com",
metadata=StandardLoggingMetadata(
user_api_key_hash="test_hash",
user_api_key_alias=None,
user_api_key_team_id=None,
user_api_key_org_id=None,
user_api_key_user_id=None,
user_api_key_team_alias=None,
spend_logs_metadata=None,
requester_ip_address=None,
requester_metadata=None,
user_api_key_end_user_id=None,
),
cache_hit=False,
cache_key=None,
saved_cache_cost=0.0,
request_tags=[],
end_user=None,
requester_ip_address=None,
messages=[],
response={},
error_str=None,
model_parameters={},
hidden_params=StandardLoggingHiddenParams(
model_id="model-123",
cache_key=None,
api_base="https://api.openai.com",
response_cost="0.001",
litellm_overhead_time_ms=None,
additional_headers=None,
batch_models=None,
litellm_model_name=None,
usage_object=None,
),
),
}
response_obj = {
"id": "test-response-retry",
"choices": [{"message": {"content": "Hello!"}}],
"usage": {
"total_tokens": 100,
"prompt_tokens": 50,
"completion_tokens": 50,
},
}
start_time = datetime.datetime.now(timezone.utc)
end_time = datetime.datetime.now(timezone.utc)
payload = get_logging_payload(
kwargs=kwargs,
response_obj=response_obj,
start_time=start_time,
end_time=end_time,
)
metadata = json.loads(payload["metadata"])
assert (
metadata.get("attempted_retries") == 2
), f"Expected attempted_retries=2, got {metadata.get('attempted_retries')}"
assert (
metadata.get("max_retries") == 3
), f"Expected max_retries=3, got {metadata.get('max_retries')}"
@patch("litellm.proxy.proxy_server.master_key", None)
@patch("litellm.proxy.proxy_server.general_settings", {})
def test_get_logging_payload_handles_missing_retry_info_gracefully():
"""
Test that retry fields are None when not present in metadata (backward compatibility).
"""
kwargs = {
"model": "gpt-3.5-turbo",
"litellm_params": {
"metadata": {
"user_api_key": "sk-test-key",
}
},
"standard_logging_object": StandardLoggingPayload(
id="test-no-retry-456",
call_type="completion",
stream=False,
response_cost=0.001,
status="success",
total_tokens=100,
prompt_tokens=50,
completion_tokens=50,
startTime=1234567890.0,
endTime=1234567891.0,
completionStartTime=None,
model_map_information=StandardLoggingModelInformation(
model_map_key="gpt-3.5-turbo", model_map_value=None
),
model="gpt-3.5-turbo",
model_id="model-123",
model_group="openai",
custom_llm_provider="openai",
api_base="https://api.openai.com",
metadata=StandardLoggingMetadata(
user_api_key_hash="test_hash",
user_api_key_alias=None,
user_api_key_team_id=None,
user_api_key_org_id=None,
user_api_key_user_id=None,
user_api_key_team_alias=None,
spend_logs_metadata=None,
requester_ip_address=None,
requester_metadata=None,
user_api_key_end_user_id=None,
),
cache_hit=False,
cache_key=None,
saved_cache_cost=0.0,
request_tags=[],
end_user=None,
requester_ip_address=None,
messages=[],
response={},
error_str=None,
model_parameters={},
hidden_params=StandardLoggingHiddenParams(
model_id="model-123",
cache_key=None,
api_base="https://api.openai.com",
response_cost="0.001",
litellm_overhead_time_ms=None,
additional_headers=None,
batch_models=None,
litellm_model_name=None,
usage_object=None,
),
),
}
response_obj = {
"id": "test-response-no-retry",
"choices": [{"message": {"content": "Hello!"}}],
"usage": {
"total_tokens": 100,
"prompt_tokens": 50,
"completion_tokens": 50,
},
}
start_time = datetime.datetime.now(timezone.utc)
end_time = datetime.datetime.now(timezone.utc)
payload = get_logging_payload(
kwargs=kwargs,
response_obj=response_obj,
start_time=start_time,
end_time=end_time,
)
metadata = json.loads(payload["metadata"])
assert (
metadata.get("attempted_retries") is None
), "attempted_retries should be None when not provided"
assert (
metadata.get("max_retries") is None
), "max_retries should be None when not provided"

8
ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx
View file

@ -296,6 +296,14 @@ function MetricsSection({ logEntry, metadata }: { logEntry: LogEntry; metadata:
</Descriptions.Item>
)}
{metadata?.attempted_retries !== undefined && metadata?.attempted_retries !== null && (
<Descriptions.Item label="Retries">
{metadata.attempted_retries > 0
? <>{metadata.attempted_retries}{metadata.max_retries !== undefined && metadata.max_retries !== null ? ` / ${metadata.max_retries}` : ''}</>
: "Not Retried"}
</Descriptions.Item>
)}
<Descriptions.Item label="Start Time">
{moment(logEntry.startTime).format("YYYY-MM-DDTHH:mm:ss.SSS[Z]")}
</Descriptions.Item>

48
ui/litellm-dashboard/src/components/view_logs/index.test.tsx
View file

@ -123,6 +123,54 @@ describe("Request Viewer", () => {
expect(screen.queryByText("LiteLLM Overhead:")).not.toBeInTheDocument();
});
it("should display retry count when attempted_retries > 0 in metadata", () => {
render(
<RequestViewer
row={createRow({
metadata: {
status: "success",
attempted_retries: 2,
max_retries: 3,
additional_usage_values: {
cache_read_input_tokens: 0,
cache_creation_input_tokens: 0,
},
},
})}
/>,
);
expect(screen.getByText("Retries:")).toBeInTheDocument();
expect(screen.getByText("2 / 3")).toBeInTheDocument();
});
it("should display 'Not Retried' when attempted_retries is 0", () => {
render(
<RequestViewer
row={createRow({
metadata: {
status: "success",
attempted_retries: 0,
max_retries: 3,
additional_usage_values: {
cache_read_input_tokens: 0,
cache_creation_input_tokens: 0,
},
},
})}
/>,
);
expect(screen.getByText("Retries:")).toBeInTheDocument();
expect(screen.getByText("Not Retried")).toBeInTheDocument();
});
it("should not display Retries when attempted_retries is not present in metadata", () => {
render(<RequestViewer row={createRow()} />);
expect(screen.queryByText("Retries:")).not.toBeInTheDocument();
});
});
describe("SpendLogsTable", () => {

10
ui/litellm-dashboard/src/components/view_logs/index.tsx
View file

@ -960,6 +960,16 @@ export function RequestViewer({ row, onOpenSettings }: { row: Row<LogEntry>; onO
<span>{row.original.metadata.litellm_overhead_time_ms} ms</span>
</div>
)}
{row.original.metadata?.attempted_retries !== undefined && row.original.metadata?.attempted_retries !== null && (
<div className="flex">
<span className="font-medium w-1/3">Retries:</span>
<span>
{row.original.metadata.attempted_retries > 0
? `${row.original.metadata.attempted_retries}${row.original.metadata.max_retries !== undefined && row.original.metadata.max_retries !== null ? ` / ${row.original.metadata.max_retries}` : ''}`
: 'Not Retried'}
</span>
</div>
)}
</div>
</div>
</div>