scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-09 22:31:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 87

[UI] - Add Guardrail Tracing to LiteLLM SpendLogs (#10893)

Browse source 
* feat: trace guardrail SLP in spendLogs

* test: trace guardrail SLP in spendLogs

* add guardrail viewer

* checkpoint - working guardrail view on logs

* ui add guardrail view to SpendLogs

* test: fixes guardrails

* trace: fixes guardrails
This commit is contained in:
Ishaan Jaff 2025-05-16 12:20:20 -07:00 • committed by GitHub
parent b9460dfe37
commit 298912bd38
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 319 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
litellm/proxy/_types.py
View file

@ -27,6 +27,7 @@ from litellm.types.utils import (
ModelResponse,
ProviderField,
StandardCallbackDynamicParams,
StandardLoggingGuardrailInformation,
StandardLoggingMCPToolCall,
StandardLoggingModelInformation,
StandardLoggingPayloadErrorInformation,
@ -2008,6 +2009,7 @@ class SpendLogsMetadata(TypedDict):
applied_guardrails: Optional[List[str]]
mcp_tool_call_metadata: Optional[StandardLoggingMCPToolCall]
vector_store_request_metadata: Optional[List[StandardLoggingVectorStoreRequest]]
guardrail_information: Optional[StandardLoggingGuardrailInformation]
status: StandardLoggingPayloadStatus
proxy_server_request: Optional[str]
batch_models: Optional[List[str]]

9
litellm/proxy/spend_tracking/spend_tracking_utils.py
View file

@ -15,6 +15,7 @@ from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kw
from litellm.proxy._types import SpendLogsMetadata, SpendLogsPayload
from litellm.proxy.utils import PrismaClient, hash_token
from litellm.types.utils import (
StandardLoggingGuardrailInformation,
StandardLoggingMCPToolCall,
StandardLoggingModelInformation,
StandardLoggingPayload,
@ -48,6 +49,7 @@ def _get_spend_logs_metadata(
vector_store_request_metadata: Optional[
List[StandardLoggingVectorStoreRequest]
] = None,
guardrail_information: Optional[StandardLoggingGuardrailInformation] = None,
usage_object: Optional[dict] = None,
model_map_information: Optional[StandardLoggingModelInformation] = None,
) -> SpendLogsMetadata:
@ -71,6 +73,7 @@ def _get_spend_logs_metadata(
vector_store_request_metadata=None,
model_map_information=None,
usage_object=None,
guardrail_information=None,
)
verbose_proxy_logger.debug(
"getting payload for SpendLogs, available keys in metadata: "
@ -91,6 +94,7 @@ def _get_spend_logs_metadata(
clean_metadata[
"vector_store_request_metadata"
] = _get_vector_store_request_for_spend_logs_payload(vector_store_request_metadata)
clean_metadata["guardrail_information"] = guardrail_information
clean_metadata["usage_object"] = usage_object
clean_metadata["model_map_information"] = model_map_information
return clean_metadata
@ -252,6 +256,11 @@ def get_logging_payload( # noqa: PLR0915
if standard_logging_payload is not None
else None
),
guardrail_information=(
standard_logging_payload.get("guardrail_information", None)
if standard_logging_payload is not None
else None
),
)
special_usage_fields = ["completion_tokens", "prompt_tokens", "total_tokens"]

24
tests/litellm/proxy/spend_tracking/test_spend_management_endpoints.py
View file

@ -438,7 +438,7 @@ async def test_ui_view_spend_logs_with_status(client, monkeypatch):
"spend": 0.05,
"startTime": datetime.datetime.now(timezone.utc).isoformat(),
"model": "gpt-3.5-turbo",
"status": "success"
"status": "success",
},
{
"id": "log2",
@ -449,7 +449,7 @@ async def test_ui_view_spend_logs_with_status(client, monkeypatch):
"spend": 0.10,
"startTime": datetime.datetime.now(timezone.utc).isoformat(),
"model": "gpt-4",
"status": "failure"
"status": "failure",
},
]
@ -462,7 +462,10 @@ async def test_ui_view_spend_logs_with_status(client, monkeypatch):
if "OR" in where_conditions:
# Handle success case (which includes None status)
return [mock_spend_logs[0]]
elif "status" in where_conditions and where_conditions["status"]["equals"] == "failure":
elif (
"status" in where_conditions
and where_conditions["status"]["equals"] == "failure"
):
return [mock_spend_logs[1]]
return mock_spend_logs
@ -472,7 +475,10 @@ async def test_ui_view_spend_logs_with_status(client, monkeypatch):
where_conditions = kwargs["where"]
if "OR" in where_conditions:
return 1
elif "status" in where_conditions and where_conditions["status"]["equals"] == "failure":
elif (
"status" in where_conditions
and where_conditions["status"]["equals"] == "failure"
):
return 1
return len(mock_spend_logs)
@ -539,7 +545,7 @@ async def test_ui_view_spend_logs_with_model(client, monkeypatch):
"spend": 0.05,
"startTime": datetime.datetime.now(timezone.utc).isoformat(),
"model": "gpt-3.5-turbo",
"status": "success"
"status": "success",
},
{
"id": "log2",
@ -550,7 +556,7 @@ async def test_ui_view_spend_logs_with_model(client, monkeypatch):
"spend": 0.10,
"startTime": datetime.datetime.now(timezone.utc).isoformat(),
"model": "gpt-4",
"status": "success"
"status": "success",
},
]
@ -656,7 +662,7 @@ class TestSpendLogsPayload:
"model": "gpt-4o",
"user": "",
"team_id": "",
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "guardrail_information": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
"cache_key": "Cache OFF",
"spend": 0.00022500000000000002,
"total_tokens": 30,
@ -748,7 +754,7 @@ class TestSpendLogsPayload:
"model": "claude-3-7-sonnet-20250219",
"user": "",
"team_id": "",
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-3-7-sonnet-20250219", "model_map_value": {"key": "claude-3-7-sonnet-20250219", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "guardrail_information": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-3-7-sonnet-20250219", "model_map_value": {"key": "claude-3-7-sonnet-20250219", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
"cache_key": "Cache OFF",
"spend": 0.01383,
"total_tokens": 2598,
@ -838,7 +844,7 @@ class TestSpendLogsPayload:
"model": "claude-3-7-sonnet-20250219",
"user": "",
"team_id": "",
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-3-7-sonnet-20250219", "model_map_value": {"key": "claude-3-7-sonnet-20250219", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "guardrail_information": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-3-7-sonnet-20250219", "model_map_value": {"key": "claude-3-7-sonnet-20250219", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
"cache_key": "Cache OFF",
"spend": 0.01383,
"total_tokens": 2598,

2
tests/logging_callback_tests/gcs_pub_sub_body/spend_logs_payload.json
View file

@ -10,7 +10,7 @@
"model": "gpt-4o",
"user": "",
"team_id": "",
"metadata": "{\"applied_guardrails\": [], \"batch_models\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}}",
"metadata": "{\"applied_guardrails\": [], \"batch_models\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"guardrail_information\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}}",
"cache_key": "Cache OFF",
"spend": 0.00022500000000000002,
"total_tokens": 30,

3
tests/logging_callback_tests/gcs_pub_sub_body/standard_logging_payload.json
View file

@ -27,7 +27,8 @@
"prompt_management_metadata": null,
"applied_guardrails": [],
"mcp_tool_call_metadata": null,
"vector_store_request_metadata": null
"vector_store_request_metadata": null,
"guardrail_information": null
},
"cache_key": null,
"response_cost": 0.00022500000000000002,

257
ui/litellm-dashboard/src/components/view_logs/GuardrailViewer.tsx Normal file
View file

@ -0,0 +1,257 @@
import React, { useState } from "react";
interface RecognitionMetadata {
recognizer_name: string;
recognizer_identifier: string;
}
interface GuardrailEntity {
end: number;
score: number;
start: number;
entity_type: string;
analysis_explanation: string | null;
recognition_metadata: RecognitionMetadata;
}
interface MaskedEntityCount {
[key: string]: number;
}
interface GuardrailInformation {
duration: number;
end_time: number;
start_time: number;
guardrail_mode: string;
guardrail_name: string;
guardrail_status: string;
guardrail_response: GuardrailEntity[];
masked_entity_count: MaskedEntityCount;
}
interface GuardrailViewerProps {
data: GuardrailInformation;
}
export function GuardrailViewer({ data }: GuardrailViewerProps) {
const [sectionExpanded, setSectionExpanded] = useState(true);
const [entityListExpanded, setEntityListExpanded] = useState(true);
const [expandedEntities, setExpandedEntities] = useState<Record<number, boolean>>({});
if (!data) {
return null;
}
// Calculate total masked entities
const totalMaskedEntities = data.masked_entity_count ?
Object.values(data.masked_entity_count).reduce((sum, count) => sum + count, 0) : 0;
const formatTime = (timestamp: number): string => {
const date = new Date(timestamp * 1000);
return date.toLocaleString();
};
const toggleEntity = (index: number) => {
setExpandedEntities(prev => ({
...prev,
[index]: !prev[index]
}));
};
const getScoreColor = (score: number): string => {
if (score >= 0.8) return "text-green-600";
return "text-yellow-600";
};
return (
<div className="bg-white rounded-lg shadow mb-6">
<div
className="flex justify-between items-center p-4 border-b cursor-pointer hover:bg-gray-50"
onClick={() => setSectionExpanded(!sectionExpanded)}
>
<div className="flex items-center">
<svg
className={`w-5 h-5 mr-2 text-gray-600 transition-transform ${sectionExpanded ? 'transform rotate-90' : ''}`}
fill="none"
stroke="currentColor"
viewBox="0 0 24 24"
>
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M9 5l7 7-7 7" />
</svg>
<h3 className="text-lg font-medium">Guardrail Information</h3>
<span className={`ml-3 px-2 py-1 rounded-md text-xs font-medium inline-block ${
data.guardrail_status === "success"
? 'bg-green-100 text-green-800'
: 'bg-red-100 text-red-800'
}`}>
{data.guardrail_status}
</span>
{totalMaskedEntities > 0 && (
<span className="ml-3 px-2 py-1 bg-blue-50 text-blue-700 rounded-md text-xs font-medium">
{totalMaskedEntities} masked {totalMaskedEntities === 1 ? 'entity' : 'entities'}
</span>
)}
</div>
<span className="text-sm text-gray-500">{sectionExpanded ? 'Click to collapse' : 'Click to expand'}</span>
</div>
{sectionExpanded && (
<div className="p-4">
<div className="bg-white rounded-lg border p-4 mb-4">
<div className="grid grid-cols-2 gap-4">
<div className="space-y-2">
<div className="flex">
<span className="font-medium w-1/3">Guardrail Name:</span>
<span className="font-mono">{data.guardrail_name}</span>
</div>
<div className="flex">
<span className="font-medium w-1/3">Mode:</span>
<span className="font-mono">{data.guardrail_mode}</span>
</div>
<div className="flex">
<span className="font-medium w-1/3">Status:</span>
<span className={`px-2 py-1 rounded-md text-xs font-medium inline-block ${
data.guardrail_status === "success"
? 'bg-green-100 text-green-800'
: 'bg-red-100 text-red-800'
}`}>
{data.guardrail_status}
</span>
</div>
</div>
<div className="space-y-2">
<div className="flex">
<span className="font-medium w-1/3">Start Time:</span>
<span>{formatTime(data.start_time)}</span>
</div>
<div className="flex">
<span className="font-medium w-1/3">End Time:</span>
<span>{formatTime(data.end_time)}</span>
</div>
<div className="flex">
<span className="font-medium w-1/3">Duration:</span>
<span>{data.duration.toFixed(4)}s</span>
</div>
</div>
</div>
{/* Masked Entity Summary */}
{data.masked_entity_count && Object.keys(data.masked_entity_count).length > 0 && (
<div className="mt-4 pt-4 border-t">
<h4 className="font-medium mb-2">Masked Entity Summary</h4>
<div className="flex flex-wrap gap-2">
{Object.entries(data.masked_entity_count).map(([entityType, count]) => (
<span key={entityType} className="px-3 py-1.5 bg-blue-50 text-blue-700 rounded-md text-xs font-medium">
{entityType}: {count}
</span>
))}
</div>
</div>
)}
</div>
{/* Detected Entities Section */}
{data.guardrail_response && data.guardrail_response.length > 0 && (
<div className="mt-4">
<div
className="flex items-center mb-2 cursor-pointer"
onClick={() => setEntityListExpanded(!entityListExpanded)}
>
<svg
className={`w-5 h-5 mr-2 transition-transform ${entityListExpanded ? 'transform rotate-90' : ''}`}
fill="none"
stroke="currentColor"
viewBox="0 0 24 24"
>
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M9 5l7 7-7 7" />
</svg>
<h4 className="font-medium">Detected Entities ({data.guardrail_response.length})</h4>
</div>
{entityListExpanded && (
<div className="space-y-2">
{data.guardrail_response.map((entity, index) => {
const isExpanded = expandedEntities[index] || false;
return (
<div key={index} className="border rounded-lg overflow-hidden">
<div
className="flex items-center justify-between p-3 bg-gray-50 cursor-pointer hover:bg-gray-100"
onClick={() => toggleEntity(index)}
>
<div className="flex items-center">
<svg
className={`w-5 h-5 mr-2 transition-transform ${isExpanded ? 'transform rotate-90' : ''}`}
fill="none"
stroke="currentColor"
viewBox="0 0 24 24"
>
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M9 5l7 7-7 7" />
</svg>
<span className="font-medium mr-2">{entity.entity_type}</span>
<span className={`font-mono ${getScoreColor(entity.score)}`}>
Score: {entity.score.toFixed(2)}
</span>
</div>
<span className="text-xs text-gray-500">
Position: {entity.start}-{entity.end}
</span>
</div>
{isExpanded && (
<div className="p-3 border-t bg-white">
<div className="grid grid-cols-2 gap-4 mb-2">
<div className="space-y-2">
<div className="flex">
<span className="font-medium w-1/3">Entity Type:</span>
<span>{entity.entity_type}</span>
</div>
<div className="flex">
<span className="font-medium w-1/3">Position:</span>
<span>Characters {entity.start}-{entity.end}</span>
</div>
<div className="flex">
<span className="font-medium w-1/3">Confidence:</span>
<span className={getScoreColor(entity.score)}>
{entity.score.toFixed(2)}
</span>
</div>
</div>
<div className="space-y-2">
{entity.recognition_metadata && (
<>
<div className="flex">
<span className="font-medium w-1/3">Recognizer:</span>
<span>{entity.recognition_metadata.recognizer_name}</span>
</div>
<div className="flex overflow-hidden">
<span className="font-medium w-1/3">Identifier:</span>
<span className="truncate text-xs font-mono">
{entity.recognition_metadata.recognizer_identifier}
</span>
</div>
</>
)}
{entity.analysis_explanation && (
<div className="flex">
<span className="font-medium w-1/3">Explanation:</span>
<span>{entity.analysis_explanation}</span>
</div>
)}
</div>
</div>
</div>
)}
</div>
);
})}
</div>
)}
</div>
)}
</div>
)}
</div>
);
}

34
ui/litellm-dashboard/src/components/view_logs/index.tsx
View file

@ -17,6 +17,7 @@ import { KeyResponse, Team } from "../key_team_helpers/key_list";
import KeyInfoView from "../key_info_view";
import { SessionView } from './SessionView';
import { VectorStoreViewer } from './VectorStoreViewer';
import { GuardrailViewer } from './GuardrailViewer';
import FilterComponent from "../common_components/filter";
import { FilterOption } from "../common_components/filter";
import { useLogFilterLogic } from "./log_filter_logic";
@ -704,6 +705,20 @@ export function RequestViewer({ row }: { row: Row<LogEntry> }) {
Array.isArray(metadata.vector_store_request_metadata) &&
metadata.vector_store_request_metadata.length > 0;
// Extract guardrail information from metadata if available
const hasGuardrailData = row.original.metadata && row.original.metadata.guardrail_information;
// Calculate total masked entities if guardrail data exists
const getTotalMaskedEntities = (): number => {
if (!hasGuardrailData || !row.original.metadata?.guardrail_information.masked_entity_count) {
return 0;
}
return Object.values(row.original.metadata.guardrail_information.masked_entity_count)
.reduce((sum: number, count: any) => sum + (typeof count === 'number' ? count : 0), 0);
};
const totalMaskedEntities = getTotalMaskedEntities();
return (
<div className="p-6 bg-gray-50 space-y-6">
{/* Combined Info Card */}
@ -745,7 +760,19 @@ export function RequestViewer({ row }: { row: Row<LogEntry> }) {
<span>{row?.original?.requester_ip_address}</span>
</div>
)}
{hasGuardrailData && (
<div className="flex">
<span className="font-medium w-1/3">Guardrail:</span>
<div>
<span className="font-mono">{row.original.metadata!.guardrail_information.guardrail_name}</span>
{totalMaskedEntities > 0 && (
<span className="ml-2 px-2 py-0.5 bg-blue-50 text-blue-700 rounded-md text-xs font-medium">
{totalMaskedEntities} masked
</span>
)}
</div>
</div>
)}
</div>
<div className="space-y-2">
<div className="flex">
@ -798,6 +825,11 @@ export function RequestViewer({ row }: { row: Row<LogEntry> }) {
formattedResponse={formattedResponse}
/>
{/* Guardrail Data - Show only if present */}
{hasGuardrailData && (
<GuardrailViewer data={row.original.metadata!.guardrail_information} />
)}
{/* Vector Store Request Data - Show only if present */}
{hasVectorStoreData && (
<VectorStoreViewer data={metadata.vector_store_request_metadata} />