scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-30 06:43:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 250

fix(logging): set is_silent_experiment only on mirrored calls so real calls keep their payloads

Browse source 
Real calls no longer gain an is_silent_experiment key in spend rows or the standard logging payload, so OTEL, S3, GCS and every other logger receive what they did before. The Langfuse regression test now runs the real prompt helper.
This commit is contained in:
mateo-berri 2026-10-21 00:22:49 -07:00
parent f71b2cdfc9
commit 5a51c4d179
27 changed files with 45 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
litellm/litellm_core_utils/litellm_logging.py
View file

@ -155,7 +155,7 @@ from litellm.types.llms.openai import (
from litellm.types.mcp import MCPPostCallResponseObject
from litellm.types.prompts.init_prompts import PromptSpec
from litellm.types.rerank import RerankResponse
from litellm.types.router import RouterRateLimitError, SilentExperimentMarker
from litellm.types.router import RouterRateLimitError, SilentExperimentFlag, SilentExperimentMarker
from litellm.types.utils import (
DEPLOYMENT_SCOPED_PRICING_FIELDS,
CachingDetails,
@ -6437,10 +6437,12 @@ class StandardLoggingPayloadSetup:
proxy_stamped_used_client_oauth_token(metadata, litellm_params),
custom_llm_provider,
),
is_silent_experiment=(
metadata is not None
**(
SilentExperimentFlag(is_silent_experiment=True)
if metadata is not None
and SILENT_EXPERIMENT_METADATA_KEY in metadata
and metadata[SILENT_EXPERIMENT_METADATA_KEY] is SilentExperimentMarker.MIRRORED
else SilentExperimentFlag()
),
)
if isinstance(metadata, dict):

11
litellm/proxy/spend_tracking/spend_tracking_utils.py
View file

@ -75,7 +75,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import (
SETTLES_BACKGROUND_RESPONSE_KWARG,
BackgroundResponseSettlement,
)
from litellm.types.router import DeploymentTypedDict, LiteLLM_Params, SilentExperimentMarker
from litellm.types.router import DeploymentTypedDict, LiteLLM_Params, SilentExperimentFlag, SilentExperimentMarker
from litellm.types.utils import (
BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,
PROMPT_CARRYING_GUARDRAIL_FIELDS,
@ -253,7 +253,6 @@ def _get_spend_logs_metadata(
guardrail_information=None,
internal_call_origin=None,
litellm_roi_estimator=False,
is_silent_experiment=False,
billed_retry_attempt=False,
eval_information=None,
cold_storage_object_key=cold_storage_object_key,
@ -291,8 +290,12 @@ def _get_spend_logs_metadata(
used_client_oauth_token=used_client_oauth_token,
kubernetes_pod_routing=kubernetes_pod_routing,
litellm_roi_estimator=metadata.get("litellm_roi_estimator") is True,
is_silent_experiment=SILENT_EXPERIMENT_METADATA_KEY in metadata
and metadata[SILENT_EXPERIMENT_METADATA_KEY] is SilentExperimentMarker.MIRRORED,
**(
SilentExperimentFlag(is_silent_experiment=True)
if SILENT_EXPERIMENT_METADATA_KEY in metadata
and metadata[SILENT_EXPERIMENT_METADATA_KEY] is SilentExperimentMarker.MIRRORED
else SilentExperimentFlag()
),
billed_retry_attempt=billed_retry_attempt,
user_api_key_user_email=_stored_user_email(metadata),
)

4
litellm/types/router.py
View file

@ -70,6 +70,10 @@ class SilentExperimentMarker(str, enum.Enum):
MIRRORED = "mirrored"
class SilentExperimentFlag(TypedDict, total=False):
is_silent_experiment: ReadOnly[bool]
class FallbackSkipReason(str, enum.Enum):
NOT_AUTHORIZED = "not_authorized"
OVER_BUDGET = "over_budget"

2
tests/unit/integrations/gcs_pubsub/spend_logs_payload.json
View file

@ -11,7 +11,7 @@
"user": "",
"team_id": "",
"organization_id": "",
"metadata": "{\"actor_agent_id\": null, \"target_agent_id\": null, \"billing_agent_id\": null, \"agent_execution_mode\": null, \"verified_human_user_id\": null, \"used_client_oauth_token\": null, \"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"litellm_roi_estimator\": false, \"is_silent_experiment\": false, \"billed_retry_attempt\": false, \"router_metadata\": null, \"autorouter_savings_estimate\": null, \"autorouter_baseline_observation\": null, \"azure_spillover\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_user_email\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"user_agent\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null, \"kubernetes_pod_routing\": null}",
"metadata": "{\"actor_agent_id\": null, \"target_agent_id\": null, \"billing_agent_id\": null, \"agent_execution_mode\": null, \"verified_human_user_id\": null, \"used_client_oauth_token\": null, \"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"litellm_roi_estimator\": false, \"billed_retry_attempt\": false, \"router_metadata\": null, \"autorouter_savings_estimate\": null, \"autorouter_baseline_observation\": null, \"azure_spillover\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_user_email\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"user_agent\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null, \"kubernetes_pod_routing\": null}",
"cache_key": "Cache OFF",
"spend": 0.00022500000000000002,
"total_tokens": 30,

1
tests/unit/integrations/langfuse/langfuse_expected_request_body/completion.json
View file

@ -14,7 +14,6 @@
]
},
"langfuse.observation.level": "DEFAULT",
"langfuse.observation.metadata.is_silent_experiment": false,
"langfuse.observation.model.name": "gpt-3.5-turbo",
"langfuse.observation.model.parameters": {
"extra_body": "{}"

1
tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_bedrock_call.json
View file

@ -14,7 +14,6 @@
]
},
"langfuse.observation.level": "DEFAULT",
"langfuse.observation.metadata.is_silent_experiment": false,
"langfuse.observation.model.name": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
"langfuse.observation.model.parameters": {
"aws_region": "us-east-1"

1
tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_complex_metadata.json
View file

@ -14,7 +14,6 @@
]
},
"langfuse.observation.level": "DEFAULT",
"langfuse.observation.metadata.is_silent_experiment": false,
"langfuse.observation.model.name": "gpt-3.5-turbo",
"langfuse.observation.model.parameters": {
"extra_body": "{}"

1
tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_langfuse_metadata.json
View file

@ -14,7 +14,6 @@
]
},
"langfuse.observation.level": "DEFAULT",
"langfuse.observation.metadata.is_silent_experiment": false,
"langfuse.observation.model.name": "gpt-3.5-turbo",
"langfuse.observation.model.parameters": {
"extra_body": "{}"

1
tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_no_choices.json
View file

@ -14,7 +14,6 @@
]
},
"langfuse.observation.level": "DEFAULT",
"langfuse.observation.metadata.is_silent_experiment": false,
"langfuse.observation.model.name": "gpt-3.5-turbo",
"langfuse.observation.model.parameters": {
"extra_body": "{}"

1
tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_router.json
View file

@ -14,7 +14,6 @@
]
},
"langfuse.observation.level": "DEFAULT",
"langfuse.observation.metadata.is_silent_experiment": false,
"langfuse.observation.model.name": "gpt-3.5-turbo",
"langfuse.observation.model.parameters": {
"stream": false,

1
tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_tags.json
View file

@ -14,7 +14,6 @@
]
},
"langfuse.observation.level": "DEFAULT",
"langfuse.observation.metadata.is_silent_experiment": false,
"langfuse.observation.model.name": "gpt-3.5-turbo",
"langfuse.observation.model.parameters": {
"extra_body": "{}"

1
tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_tags_stream.json
View file

@ -14,7 +14,6 @@
]
},
"langfuse.observation.level": "DEFAULT",
"langfuse.observation.metadata.is_silent_experiment": false,
"langfuse.observation.model.name": "gpt-3.5-turbo",
"langfuse.observation.model.parameters": {
"extra_body": "{}",

1
tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_vertex_call.json
View file

@ -14,7 +14,6 @@
]
},
"langfuse.observation.level": "DEFAULT",
"langfuse.observation.metadata.is_silent_experiment": false,
"langfuse.observation.model.name": "gemini-3-flash-preview",
"langfuse.observation.model.parameters": {},
"langfuse.observation.type": "generation",

1
tests/unit/integrations/langfuse/langfuse_expected_request_body/complex_metadata.json
View file

@ -14,7 +14,6 @@
]
},
"langfuse.observation.level": "DEFAULT",
"langfuse.observation.metadata.is_silent_experiment": false,
"langfuse.observation.model.name": "gpt-3.5-turbo",
"langfuse.observation.model.parameters": {
"extra_body": "{}"

1
tests/unit/integrations/langfuse/langfuse_expected_request_body/complex_metadata_2.json
View file

@ -14,7 +14,6 @@
]
},
"langfuse.observation.level": "DEFAULT",
"langfuse.observation.metadata.is_silent_experiment": false,
"langfuse.observation.model.name": "gpt-3.5-turbo",
"langfuse.observation.model.parameters": {
"extra_body": "{}"

1
tests/unit/integrations/langfuse/langfuse_expected_request_body/empty_metadata.json
View file

@ -14,7 +14,6 @@
]
},
"langfuse.observation.level": "DEFAULT",
"langfuse.observation.metadata.is_silent_experiment": false,
"langfuse.observation.model.name": "gpt-3.5-turbo",
"langfuse.observation.model.parameters": {
"extra_body": "{}"

1
tests/unit/integrations/langfuse/langfuse_expected_request_body/metadata_with_function.json
View file

@ -14,7 +14,6 @@
]
},
"langfuse.observation.level": "DEFAULT",
"langfuse.observation.metadata.is_silent_experiment": false,
"langfuse.observation.model.name": "gpt-3.5-turbo",
"langfuse.observation.model.parameters": {
"extra_body": "{}"

1
tests/unit/integrations/langfuse/langfuse_expected_request_body/metadata_with_lock.json
View file

@ -14,7 +14,6 @@
]
},
"langfuse.observation.level": "DEFAULT",
"langfuse.observation.metadata.is_silent_experiment": false,
"langfuse.observation.model.name": "gpt-3.5-turbo",
"langfuse.observation.model.parameters": {
"extra_body": "{}"

1
tests/unit/integrations/langfuse/langfuse_expected_request_body/nested_metadata.json
View file

@ -14,7 +14,6 @@
]
},
"langfuse.observation.level": "DEFAULT",
"langfuse.observation.metadata.is_silent_experiment": false,
"langfuse.observation.model.name": "gpt-3.5-turbo",
"langfuse.observation.model.parameters": {
"extra_body": "{}"

1
tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata.json
View file

@ -14,7 +14,6 @@
]
},
"langfuse.observation.level": "DEFAULT",
"langfuse.observation.metadata.is_silent_experiment": false,
"langfuse.observation.model.name": "gpt-3.5-turbo",
"langfuse.observation.model.parameters": {
"extra_body": "{}"

1
tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata2.json
View file

@ -14,7 +14,6 @@
]
},
"langfuse.observation.level": "DEFAULT",
"langfuse.observation.metadata.is_silent_experiment": false,
"langfuse.observation.model.name": "gpt-3.5-turbo",
"langfuse.observation.model.parameters": {
"extra_body": "{}"

1
tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata3.json
View file

@ -14,7 +14,6 @@
]
},
"langfuse.observation.level": "DEFAULT",
"langfuse.observation.metadata.is_silent_experiment": false,
"langfuse.observation.model.name": "gpt-3.5-turbo",
"langfuse.observation.model.parameters": {
"extra_body": "{}"

33
tests/unit/integrations/test_langfuse.py
View file

@ -838,25 +838,20 @@ class TestLangfuseUsageDetails(unittest.TestCase):
metadata = self._canary_request_metadata()
self.use_real_langfuse_client()
with patch(
"litellm.integrations.langfuse.langfuse._add_prompt_to_generation_params",
side_effect=lambda generation_params, **kw: generation_params,
create=True,
):
self.logger._log_langfuse_v2(
user_id="user-1",
metadata=metadata,
litellm_params={"metadata": metadata},
output=None,
start_time=datetime.datetime(2024, 1, 1, 12, 0, 0),
end_time=datetime.datetime(2024, 1, 1, 12, 0, 1),
kwargs=kwargs,
optional_params={},
input=None,
response_obj=None,
level="INFO",
litellm_call_id="canary-call-id",
)
self.logger._log_langfuse_v2(
user_id="user-1",
metadata=metadata,
litellm_params={"metadata": metadata},
output=None,
start_time=datetime.datetime(2024, 1, 1, 12, 0, 0),
end_time=datetime.datetime(2024, 1, 1, 12, 0, 1),
kwargs=kwargs,
optional_params={},
input=None,
response_obj=None,
level="INFO",
litellm_call_id="canary-call-id",
)
assert self.exported_generation_metadata()["is_silent_experiment"] is True

2
tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py
View file

@ -10373,7 +10373,7 @@ def test_init_kwargs_client_metadata_cannot_mark_a_pass_through_call_as_a_silent
start_time=started,
end_time=started,
)
assert json.loads(row["metadata"])["is_silent_experiment"] is False
assert "is_silent_experiment" not in json.loads(row["metadata"])
assert result["litellm_params"]["metadata"]["client_key"] == "kept"
@pytest.mark.asyncio

2
tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py
View file

@ -3725,7 +3725,7 @@ class TestSpendLogsPayload:
"model": "gpt-4o",
"user": "",
"team_id": "",
"metadata": '{"user_api_key_user_email": null, "actor_agent_id": null, "target_agent_id": null, "billing_agent_id": null, "agent_execution_mode": null, "verified_human_user_id": null, "applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "azure_spillover": null, "used_client_oauth_token": null, "litellm_roi_estimator": false, "is_silent_experiment": false, "billed_retry_attempt": false, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
"metadata": '{"user_api_key_user_email": null, "actor_agent_id": null, "target_agent_id": null, "billing_agent_id": null, "agent_execution_mode": null, "verified_human_user_id": null, "applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "azure_spillover": null, "used_client_oauth_token": null, "litellm_roi_estimator": false, "billed_retry_attempt": false, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
"cache_key": "Cache OFF",
"spend": 0.00022500000000000002,
"total_tokens": 30,

14
tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py
View file

@ -169,14 +169,14 @@ def test_get_logging_payload_includes_kubernetes_pod_routing(
("silent_experiment_metadata", "expected"),
[
({SILENT_EXPERIMENT_METADATA_KEY: SilentExperimentMarker.MIRRORED}, True),
({SILENT_EXPERIMENT_METADATA_KEY: SilentExperimentMarker.MIRRORED.value}, False),
({SILENT_EXPERIMENT_METADATA_KEY: True}, False),
({"is_silent_experiment": True}, False),
({}, False),
({SILENT_EXPERIMENT_METADATA_KEY: SilentExperimentMarker.MIRRORED.value}, None),
({SILENT_EXPERIMENT_METADATA_KEY: True}, None),
({"is_silent_experiment": True}, None),
({}, None),
],
)
def test_get_logging_payload_records_whether_the_call_was_a_silent_experiment(
silent_experiment_metadata: dict[str, object], expected: bool
silent_experiment_metadata: dict[str, object], expected: bool | None
) -> None:
now: Final = datetime.datetime(2026, 1, 1, tzinfo=timezone.utc)
payload: Final = get_logging_payload(
@ -189,7 +189,7 @@ def test_get_logging_payload_records_whether_the_call_was_a_silent_experiment(
end_time=now,
)
assert json.loads(payload["metadata"])["is_silent_experiment"] is expected
assert json.loads(payload["metadata"]).get("is_silent_experiment") is expected
@pytest.mark.parametrize("store_prompts,redact", [(True, False), (False, False), (True, True)])
@ -6642,7 +6642,7 @@ def test_spend_logs_payload(model_id: str | None):
)
assert len(payload["request_id"]) > 0
expected_metadata_keys: Final = SpendLogsMetadata.__annotations__.keys()
expected_metadata_keys: Final = SpendLogsMetadata.__annotations__.keys() - {"is_silent_experiment"}
assert "metadata" in payload
assert isinstance(payload["metadata"], str)

10
tests/unit/test_router_silent_experiment.py
View file

@ -284,7 +284,7 @@ async def test_standard_logging_payload_marks_only_the_mirrored_calls_as_silent_
metadata_by_group: Final = {payload["model_group"]: payload["metadata"] for payload in payloads}
assert metadata_by_group["shadow-a"]["is_silent_experiment"] is True
assert metadata_by_group["shadow-b"]["is_silent_experiment"] is True
assert metadata_by_group["primary-model"]["is_silent_experiment"] is False
assert "is_silent_experiment" not in metadata_by_group["primary-model"]
def _spend_row_is_silent_experiment(call: dict[str, object]) -> object:
@ -297,7 +297,7 @@ def _spend_row_is_silent_experiment(call: dict[str, object]) -> object:
start_time=now,
end_time=now,
)
return json.loads(row["metadata"])["is_silent_experiment"]
return json.loads(row["metadata"]).get("is_silent_experiment")
@pytest.mark.asyncio
@ -317,7 +317,7 @@ async def test_spend_rows_mark_only_the_mirrored_calls_when_the_caller_sends_lit
call["standard_logging_object"]["model_group"]: _spend_row_is_silent_experiment(call)
for call in recording_logger.success_kwargs
}
assert silent_by_group == {"primary-model": False, "shadow-a": True, "shadow-b": True}
assert silent_by_group == {"primary-model": None, "shadow-a": True, "shadow-b": True}
@pytest.mark.asyncio
@ -343,8 +343,8 @@ async def test_a_caller_sent_silent_experiment_flag_does_not_mark_a_real_call_as
await _wait_for_all_successes(recording_logger, expected=1)
call: Final = recording_logger.success_kwargs[0]
assert call["standard_logging_object"]["metadata"]["is_silent_experiment"] is False
assert _spend_row_is_silent_experiment(call) is False
assert "is_silent_experiment" not in call["standard_logging_object"]["metadata"]
assert _spend_row_is_silent_experiment(call) is None
def test_multiple_shadow_targets_fan_out_sync(recording_logger):