scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-07 02:59:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

Merge pull request #21704 from BerriAI/litellm_logs_retry_count

Browse source 
[Feature] UI - Logs: Show retry count for requests
This commit is contained in:
yuneng-jiang 2026-02-20 15:08:55 -08:00 • committed by GitHub
commit 8356a69663
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 282 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
litellm/proxy/_types.py
View file

@ -3044,6 +3044,8 @@ class SpendLogsMetadata(TypedDict):
str
] # S3/GCS object key for cold storage retrieval
litellm_overhead_time_ms: Optional[float] # LiteLLM overhead time in milliseconds
attempted_retries: Optional[int] # Number of retries attempted (0 = first attempt succeeded)
max_retries: Optional[int] # Max retries configured for this request
cost_breakdown: Optional[
CostBreakdown
] # Detailed cost breakdown (input_cost, output_cost, margin, discount, etc.)

2
litellm/proxy/spend_tracking/spend_tracking_utils.py
View file

@ -86,6 +86,8 @@ def _get_spend_logs_metadata(
guardrail_information=None,
cold_storage_object_key=cold_storage_object_key,
litellm_overhead_time_ms=None,
attempted_retries=None,
max_retries=None,
cost_breakdown=None,
)
verbose_proxy_logger.debug(

9
litellm/router.py
View file

@ -5128,6 +5128,9 @@ class Router:
verbose_router_logger.debug(
f"async function w/ retries: original_function - {original_function}, num_retries - {num_retries}"
)
## ADD RETRY TRACKING TO METADATA - used for spend logs retry tracking
_metadata["attempted_retries"] = 0
_metadata["max_retries"] = num_retries # Updated after overrides in exception handler
try:
self._handle_mock_testing_rate_limit_error(
model_group=model_group, kwargs=kwargs
@ -5193,6 +5196,9 @@ class Router:
regular_fallbacks=fallbacks,
content_policy_fallbacks=content_policy_fallbacks,
)
# Update max_retries after overrides (deployment_num_retries / retry_policy)
_metadata["max_retries"] = num_retries
## LOGGING
if num_retries > 0:
kwargs = self.log_retry(kwargs=kwargs, e=original_exception)
@ -5215,6 +5221,9 @@ class Router:
for current_attempt in range(num_retries):
try:
# Update retry tracking metadata before each retry attempt
_metadata["attempted_retries"] = current_attempt + 1
_metadata["max_retries"] = num_retries
# if the function call is successful, no exception will be raised and we'll break out of the loop
response = await self.make_call(original_function, *args, **kwargs)
if coroutine_checker.is_async_callable(

201
tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
View file

@ -1031,3 +1031,204 @@ def test_get_logging_payload_guardrail_info_when_no_standard_logging_payload():
metadata_result = json.loads(payload["metadata"])
assert metadata_result["guardrail_information"] == guardrail_info
@patch("litellm.proxy.proxy_server.master_key", None)
@patch("litellm.proxy.proxy_server.general_settings", {})
def test_get_logging_payload_includes_retry_info_in_spend_logs_metadata():
"""
Test that retry info (attempted_retries, max_retries) from metadata
is included in the spend logs metadata JSON.
"""
kwargs = {
"model": "gpt-3.5-turbo",
"litellm_params": {
"metadata": {
"user_api_key": "sk-test-key",
"attempted_retries": 2,
"max_retries": 3,
}
},
"standard_logging_object": StandardLoggingPayload(
id="test-retry-123",
call_type="completion",
stream=False,
response_cost=0.001,
status="success",
total_tokens=100,
prompt_tokens=50,
completion_tokens=50,
startTime=1234567890.0,
endTime=1234567891.0,
completionStartTime=None,
model_map_information=StandardLoggingModelInformation(
model_map_key="gpt-3.5-turbo", model_map_value=None
),
model="gpt-3.5-turbo",
model_id="model-123",
model_group="openai",
custom_llm_provider="openai",
api_base="https://api.openai.com",
metadata=StandardLoggingMetadata(
user_api_key_hash="test_hash",
user_api_key_alias=None,
user_api_key_team_id=None,
user_api_key_org_id=None,
user_api_key_user_id=None,
user_api_key_team_alias=None,
spend_logs_metadata=None,
requester_ip_address=None,
requester_metadata=None,
user_api_key_end_user_id=None,
),
cache_hit=False,
cache_key=None,
saved_cache_cost=0.0,
request_tags=[],
end_user=None,
requester_ip_address=None,
messages=[],
response={},
error_str=None,
model_parameters={},
hidden_params=StandardLoggingHiddenParams(
model_id="model-123",
cache_key=None,
api_base="https://api.openai.com",
response_cost="0.001",
litellm_overhead_time_ms=None,
additional_headers=None,
batch_models=None,
litellm_model_name=None,
usage_object=None,
),
),
}
response_obj = {
"id": "test-response-retry",
"choices": [{"message": {"content": "Hello!"}}],
"usage": {
"total_tokens": 100,
"prompt_tokens": 50,
"completion_tokens": 50,
},
}
start_time = datetime.datetime.now(timezone.utc)
end_time = datetime.datetime.now(timezone.utc)
payload = get_logging_payload(
kwargs=kwargs,
response_obj=response_obj,
start_time=start_time,
end_time=end_time,
)
metadata = json.loads(payload["metadata"])
assert (
metadata.get("attempted_retries") == 2
), f"Expected attempted_retries=2, got {metadata.get('attempted_retries')}"
assert (
metadata.get("max_retries") == 3
), f"Expected max_retries=3, got {metadata.get('max_retries')}"
@patch("litellm.proxy.proxy_server.master_key", None)
@patch("litellm.proxy.proxy_server.general_settings", {})
def test_get_logging_payload_handles_missing_retry_info_gracefully():
"""
Test that retry fields are None when not present in metadata (backward compatibility).
"""
kwargs = {
"model": "gpt-3.5-turbo",
"litellm_params": {
"metadata": {
"user_api_key": "sk-test-key",
}
},
"standard_logging_object": StandardLoggingPayload(
id="test-no-retry-456",
call_type="completion",
stream=False,
response_cost=0.001,
status="success",
total_tokens=100,
prompt_tokens=50,
completion_tokens=50,
startTime=1234567890.0,
endTime=1234567891.0,
completionStartTime=None,
model_map_information=StandardLoggingModelInformation(
model_map_key="gpt-3.5-turbo", model_map_value=None
),
model="gpt-3.5-turbo",
model_id="model-123",
model_group="openai",
custom_llm_provider="openai",
api_base="https://api.openai.com",
metadata=StandardLoggingMetadata(
user_api_key_hash="test_hash",
user_api_key_alias=None,
user_api_key_team_id=None,
user_api_key_org_id=None,
user_api_key_user_id=None,
user_api_key_team_alias=None,
spend_logs_metadata=None,
requester_ip_address=None,
requester_metadata=None,
user_api_key_end_user_id=None,
),
cache_hit=False,
cache_key=None,
saved_cache_cost=0.0,
request_tags=[],
end_user=None,
requester_ip_address=None,
messages=[],
response={},
error_str=None,
model_parameters={},
hidden_params=StandardLoggingHiddenParams(
model_id="model-123",
cache_key=None,
api_base="https://api.openai.com",
response_cost="0.001",
litellm_overhead_time_ms=None,
additional_headers=None,
batch_models=None,
litellm_model_name=None,
usage_object=None,
),
),
}
response_obj = {
"id": "test-response-no-retry",
"choices": [{"message": {"content": "Hello!"}}],
"usage": {
"total_tokens": 100,
"prompt_tokens": 50,
"completion_tokens": 50,
},
}
start_time = datetime.datetime.now(timezone.utc)
end_time = datetime.datetime.now(timezone.utc)
payload = get_logging_payload(
kwargs=kwargs,
response_obj=response_obj,
start_time=start_time,
end_time=end_time,
)
metadata = json.loads(payload["metadata"])
assert (
metadata.get("attempted_retries") is None
), "attempted_retries should be None when not provided"
assert (
metadata.get("max_retries") is None
), "max_retries should be None when not provided"

8
ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx
View file

@ -296,6 +296,14 @@ function MetricsSection({ logEntry, metadata }: { logEntry: LogEntry; metadata:
</Descriptions.Item>
)}
<Descriptions.Item label="Retries">
{metadata?.attempted_retries !== undefined && metadata?.attempted_retries !== null
? metadata.attempted_retries > 0
? <>{metadata.attempted_retries}{metadata.max_retries !== undefined && metadata.max_retries !== null ? ` / ${metadata.max_retries}` : ''}</>
: <Tag color="green">None</Tag>
: "-"}
</Descriptions.Item>
<Descriptions.Item label="Start Time">
{moment(logEntry.startTime).format("YYYY-MM-DDTHH:mm:ss.SSS[Z]")}
</Descriptions.Item>

49
ui/litellm-dashboard/src/components/view_logs/index.test.tsx
View file

@ -123,6 +123,55 @@ describe("Request Viewer", () => {
expect(screen.queryByText("LiteLLM Overhead:")).not.toBeInTheDocument();
});
it("should display retry count when attempted_retries > 0 in metadata", () => {
render(
<RequestViewer
row={createRow({
metadata: {
status: "success",
attempted_retries: 2,
max_retries: 3,
additional_usage_values: {
cache_read_input_tokens: 0,
cache_creation_input_tokens: 0,
},
},
})}
/>,
);
expect(screen.getByText("Retries:")).toBeInTheDocument();
expect(screen.getByText("2 / 3")).toBeInTheDocument();
});
it("should display green 'None' tag when attempted_retries is 0", () => {
render(
<RequestViewer
row={createRow({
metadata: {
status: "success",
attempted_retries: 0,
max_retries: 3,
additional_usage_values: {
cache_read_input_tokens: 0,
cache_creation_input_tokens: 0,
},
},
})}
/>,
);
expect(screen.getByText("Retries:")).toBeInTheDocument();
expect(screen.getByText("None")).toBeInTheDocument();
});
it("should display '-' for Retries when attempted_retries is not present in metadata", () => {
render(<RequestViewer row={createRow()} />);
expect(screen.getByText("Retries:")).toBeInTheDocument();
expect(screen.getByText("-")).toBeInTheDocument();
});
});
describe("SpendLogsTable", () => {

12
ui/litellm-dashboard/src/components/view_logs/index.tsx
View file

@ -7,7 +7,7 @@ import { truncateString } from "@/utils/textUtils";
import { SettingOutlined, SyncOutlined } from "@ant-design/icons";
import { Row } from "@tanstack/react-table";
import { Switch, Tab, TabGroup, TabList, TabPanel, TabPanels } from "@tremor/react";
import { Button, Tooltip } from "antd";
import { Button, Tag, Tooltip } from "antd";
import { internalUserRoles } from "../../utils/roles";
import DeletedKeysPage from "../DeletedKeysPage/DeletedKeysPage";
import DeletedTeamsPage from "../DeletedTeamsPage/DeletedTeamsPage";
@ -960,6 +960,16 @@ export function RequestViewer({ row, onOpenSettings }: { row: Row<LogEntry>; onO
<span>{row.original.metadata.litellm_overhead_time_ms} ms</span>
</div>
)}
<div className="flex">
<span className="font-medium w-1/3">Retries:</span>
<span>
{row.original.metadata?.attempted_retries !== undefined && row.original.metadata?.attempted_retries !== null
? row.original.metadata.attempted_retries > 0
? `${row.original.metadata.attempted_retries}${row.original.metadata.max_retries !== undefined && row.original.metadata.max_retries !== null ? ` / ${row.original.metadata.max_retries}` : ''}`
: <Tag color="green">None</Tag>
: '-'}
</span>
</div>
</div>
</div>
</div>