From a7e8895aa83c6d6f4a860972008a01ba7636ceaf Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 17:48:30 -0700 Subject: [PATCH 1/7] fix(logging): mark mirrored silent_model calls with is_silent_experiment in spend logs and logging payloads --- litellm/proxy/_types.py | 1 + .../spend_tracking/spend_tracking_utils.py | 3 +++ litellm/types/utils.py | 1 + .../gcs_pubsub/spend_logs_payload.json | 2 +- .../test_litellm_logging.py | 2 +- .../test_spend_management_endpoints.py | 2 +- .../test_spend_tracking_utils.py | 21 +++++++++++++++++++ tests/unit/test_router_silent_experiment.py | 19 +++++++++++++++++ 8 files changed, 48 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 54acf59ef80..1790cba7506 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3733,6 +3733,7 @@ class SpendLogsMetadata(TypedDict): internal_call_origin: InternalCallOrigin | None billed_retry_attempt: ReadOnly[NotRequired[bool | None]] litellm_roi_estimator: ReadOnly[NotRequired[bool | None]] + is_silent_experiment: ReadOnly[NotRequired[bool | None]] guardrail_information: list[StandardLoggingGuardrailInformation] | None eval_information: Any | None status: StandardLoggingPayloadStatus diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 62aee788ed0..6183ee5cb3e 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -188,6 +188,7 @@ _STAMPED_METADATA_KEYS: Final = frozenset( "used_client_oauth_token", KUBERNETES_POD_ROUTING_KEY, "litellm_roi_estimator", + "is_silent_experiment", BILLED_RETRY_ATTEMPT_METADATA_KEY, "user_api_key_user_email", ) @@ -251,6 +252,7 @@ def _get_spend_logs_metadata( guardrail_information=None, internal_call_origin=None, litellm_roi_estimator=False, + is_silent_experiment=False, billed_retry_attempt=False, eval_information=None, cold_storage_object_key=cold_storage_object_key, @@ -288,6 +290,7 @@ def _get_spend_logs_metadata( used_client_oauth_token=used_client_oauth_token, kubernetes_pod_routing=kubernetes_pod_routing, litellm_roi_estimator=metadata.get("litellm_roi_estimator") is True, + is_silent_experiment="is_silent_experiment" in metadata and metadata["is_silent_experiment"] is True, billed_retry_attempt=billed_retry_attempt, user_api_key_user_email=_stored_user_email(metadata), ) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 5939a350970..40b4d86e274 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3355,6 +3355,7 @@ class StandardLoggingMetadata(StandardLoggingUserAPIKeyMetadata): team_alias: str | None team_id: str | None used_client_oauth_token: ReadOnly[bool | None] + is_silent_experiment: ReadOnly[NotRequired[bool]] class AzureSpillover(TypedDict): diff --git a/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json b/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json index 6e5461c3b60..6b172c687a3 100644 --- a/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json +++ b/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json @@ -11,7 +11,7 @@ "user": "", "team_id": "", "organization_id": "", - "metadata": "{\"actor_agent_id\": null, \"target_agent_id\": null, \"billing_agent_id\": null, \"agent_execution_mode\": null, \"verified_human_user_id\": null, \"used_client_oauth_token\": null, \"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"litellm_roi_estimator\": false, \"billed_retry_attempt\": false, \"router_metadata\": null, \"autorouter_savings_estimate\": null, \"autorouter_baseline_observation\": null, \"azure_spillover\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_user_email\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"user_agent\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null, \"kubernetes_pod_routing\": null}", + "metadata": "{\"actor_agent_id\": null, \"target_agent_id\": null, \"billing_agent_id\": null, \"agent_execution_mode\": null, \"verified_human_user_id\": null, \"used_client_oauth_token\": null, \"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"litellm_roi_estimator\": false, \"is_silent_experiment\": false, \"billed_retry_attempt\": false, \"router_metadata\": null, \"autorouter_savings_estimate\": null, \"autorouter_baseline_observation\": null, \"azure_spillover\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_user_email\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"user_agent\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null, \"kubernetes_pod_routing\": null}", "cache_key": "Cache OFF", "spend": 0.00022500000000000002, "total_tokens": 30, diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 79645ffe8f6..fec5e5c820c 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -11766,7 +11766,7 @@ def test_get_additional_headers(): assert additional_logging_headers.get("llm_provider-anthropic-ratelimit-requests-reset") == "2024-10-29T23:57:40Z" def all_fields_present(standard_logging_metadata: StandardLoggingMetadata): - for field in StandardLoggingMetadata.__annotations__.keys(): + for field in StandardLoggingMetadata.__required_keys__: assert field in standard_logging_metadata @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") diff --git a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py index 23bbe521670..695452fce40 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py @@ -3725,7 +3725,7 @@ class TestSpendLogsPayload: "model": "gpt-4o", "user": "", "team_id": "", - "metadata": '{"user_api_key_user_email": null, "actor_agent_id": null, "target_agent_id": null, "billing_agent_id": null, "agent_execution_mode": null, "verified_human_user_id": null, "applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "azure_spillover": null, "used_client_oauth_token": null, "litellm_roi_estimator": false, "billed_retry_attempt": false, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}', + "metadata": '{"user_api_key_user_email": null, "actor_agent_id": null, "target_agent_id": null, "billing_agent_id": null, "agent_execution_mode": null, "verified_human_user_id": null, "applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "azure_spillover": null, "used_client_oauth_token": null, "litellm_roi_estimator": false, "is_silent_experiment": false, "billed_retry_attempt": false, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}', "cache_key": "Cache OFF", "spend": 0.00022500000000000002, "total_tokens": 30, diff --git a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py index 55358a3b0de..c9ec799a9c0 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py @@ -163,6 +163,27 @@ def test_get_logging_payload_includes_kubernetes_pod_routing( assert _get_spend_logs_metadata(None)["kubernetes_pod_routing"] is None +@pytest.mark.parametrize( + ("silent_experiment_metadata", "expected"), + [({"is_silent_experiment": True}, True), ({"is_silent_experiment": "true"}, False), ({}, False)], +) +def test_get_logging_payload_records_whether_the_call_was_a_silent_experiment( + silent_experiment_metadata: dict[str, object], expected: bool +) -> None: + now: Final = datetime.datetime(2026, 1, 1, tzinfo=timezone.utc) + payload: Final = get_logging_payload( + kwargs={ + "model": "gpt-4o-mini", + "litellm_params": {"metadata": {"user_api_key": "test-key", **silent_experiment_metadata}}, + }, + response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]), + start_time=now, + end_time=now, + ) + + assert json.loads(payload["metadata"])["is_silent_experiment"] is expected + + @pytest.mark.parametrize("store_prompts,redact", [(True, False), (False, False), (True, True)]) def test_classifier_audit_spend_storage_obeys_privacy_and_truncation(monkeypatch, store_prompts, redact): from litellm.proxy import proxy_server diff --git a/tests/unit/test_router_silent_experiment.py b/tests/unit/test_router_silent_experiment.py index fd648ceb5d2..e659c8e0b7a 100644 --- a/tests/unit/test_router_silent_experiment.py +++ b/tests/unit/test_router_silent_experiment.py @@ -263,6 +263,25 @@ async def test_multiple_shadow_targets_fan_out_async(recording_logger): assert primary_successes[0]["litellm_params"]["metadata"]["model_group"] == "primary-model" +@pytest.mark.asyncio +async def test_standard_logging_payload_marks_only_the_mirrored_calls_as_silent_experiments(recording_logger): + router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"])) + response: Final = await router.acompletion( + model="primary-model", + messages=[{"role": "user", "content": "hi"}], + mock_response="pong", + ) + assert response.choices[0].message.content == "pong" + await _wait_for_all_successes(recording_logger, expected=3) + + payloads: Final = [call["standard_logging_object"] for call in recording_logger.success_kwargs] + assert sorted(payload["model_group"] for payload in payloads) == ["primary-model", "shadow-a", "shadow-b"] + metadata_by_group: Final = {payload["model_group"]: payload["metadata"] for payload in payloads} + assert metadata_by_group["shadow-a"]["is_silent_experiment"] is True + assert metadata_by_group["shadow-b"]["is_silent_experiment"] is True + assert "is_silent_experiment" not in metadata_by_group["primary-model"] + + def test_multiple_shadow_targets_fan_out_sync(recording_logger): router = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"])) response = router.completion( From f2d06470fdcebc73dd7946c89c8dee9d1d43aa36 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:18:06 -0700 Subject: [PATCH 2/7] fix(router): keep the mirror flag when litellm_metadata is set and strip a client-sent is_silent_experiment Spend rows read router metadata from litellm_metadata when a call carries it, so the router now sets the flag there too. The proxy drops a client-sent is_silent_experiment with the other router-owned keys, so real traffic cannot be recorded as mirrored. --- litellm/proxy/litellm_pre_call_utils.py | 1 + litellm/router.py | 2 ++ .../unit/proxy/test_litellm_pre_call_utils.py | 2 ++ tests/unit/test_router_silent_experiment.py | 29 +++++++++++++++++++ 4 files changed, 34 insertions(+) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 09f53cecb72..a6e971f002a 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -469,6 +469,7 @@ _CLIENT_PRICING_METADATA_FIELDS: Final = frozenset({"model_info", "standard_logg _ROUTER_RESERVED_METADATA_FIELDS: Final = frozenset( { "attempted_fallbacks", + "is_silent_experiment", "original_model_group", "previous_models", "request_retry_count", diff --git a/litellm/router.py b/litellm/router.py index a18ad7c3a97..e1271970c22 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -3124,6 +3124,8 @@ class Router: silent_kwargs["metadata"].pop("litellm_parent_otel_span", None) silent_kwargs["metadata"]["is_silent_experiment"] = True + if "litellm_metadata" in silent_kwargs and isinstance(silent_kwargs["litellm_metadata"], dict): + silent_kwargs["litellm_metadata"] = {**silent_kwargs["litellm_metadata"], "is_silent_experiment": True} # Pop logging objects and call IDs to ensure a fresh logging context # This prevents collisions in the Proxy's database (spend_logs) diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index c2a5b01fe4e..7064b52ad8c 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -8278,6 +8278,7 @@ def _reserved_stamp_key(key_metadata: dict | None = None) -> UserAPIKeyAuth: _PLANTED_STAMPS = { "attempted_fallbacks": 99, + "is_silent_experiment": True, "original_model_group": "spoofed-group", "request_retry_count": -100, "previous_models": [{"exception_type": "Timeout", "deployment_id": "spoofed", "attempted_retries": 101}], @@ -8312,6 +8313,7 @@ async def test_add_litellm_data_to_request_strips_router_reserved_stamps_from_bo assert "litellm_metadata" not in updated assert "attempted_fallbacks" not in updated["metadata"] + assert "is_silent_experiment" not in updated["metadata"] assert "original_model_group" not in updated["metadata"] assert "_client_output_ceiling" not in updated["metadata"] assert "request_retry_count" not in updated["metadata"] diff --git a/tests/unit/test_router_silent_experiment.py b/tests/unit/test_router_silent_experiment.py index e659c8e0b7a..97a0ba146cf 100644 --- a/tests/unit/test_router_silent_experiment.py +++ b/tests/unit/test_router_silent_experiment.py @@ -1,4 +1,5 @@ import asyncio +import datetime import json import time from collections.abc import Callable, Mapping @@ -282,6 +283,34 @@ async def test_standard_logging_payload_marks_only_the_mirrored_calls_as_silent_ assert "is_silent_experiment" not in metadata_by_group["primary-model"] +@pytest.mark.asyncio +async def test_spend_rows_mark_only_the_mirrored_calls_when_the_caller_sends_litellm_metadata(recording_logger): + from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload + + router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"])) + await router.acompletion( + model="primary-model", + messages=[{"role": "user", "content": "hi"}], + mock_response="pong", + litellm_metadata={"user_api_key": "test-key"}, + ) + await _wait_for_all_successes(recording_logger, expected=3) + + now: Final = datetime.datetime.now(datetime.timezone.utc) + silent_by_group: Final = { + call["standard_logging_object"]["model_group"]: json.loads( + get_logging_payload( + kwargs=call, + response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]), + start_time=now, + end_time=now, + )["metadata"] + )["is_silent_experiment"] + for call in recording_logger.success_kwargs + } + assert silent_by_group == {"primary-model": False, "shadow-a": True, "shadow-b": True} + + def test_multiple_shadow_targets_fan_out_sync(recording_logger): router = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"])) response = router.completion( From d23c9fe3200cd6fde9ebd4345bb6ff7ca3adf5d1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:53:31 -0700 Subject: [PATCH 3/7] fix(logging): mark mirrored calls with a router-only key so clients cannot set is_silent_experiment Spend rows and the standard logging payload now derive is_silent_experiment from _litellm_silent_experiment, which only the router writes and the proxy and pass-through routes strip from client metadata. A caller's own is_silent_experiment key stays untouched, so it keeps its current effect on mirroring and on provider metadata, and unmirrored calls log false in both places. --- litellm/constants.py | 1 + litellm/litellm_core_utils/litellm_logging.py | 8 +++- litellm/proxy/litellm_pre_call_utils.py | 3 +- .../pass_through_endpoints.py | 7 ++- .../spend_tracking/spend_tracking_utils.py | 4 +- litellm/router.py | 7 ++- .../test_pass_through_endpoints.py | 38 +++++++++++++++- .../test_spend_tracking_utils.py | 8 +++- .../unit/proxy/test_litellm_pre_call_utils.py | 5 ++- tests/unit/test_router_silent_experiment.py | 43 +++++++++++++------ 10 files changed, 103 insertions(+), 21 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 1c1dfb9aa1c..33c1bee0b25 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1653,6 +1653,7 @@ ROUTING_REQUEST_TAGS_METADATA_KEY: Final = "_routing_request_tags" BAGGAGE_MAPPED_TAGS_METADATA_KEY: Final = "_baggage_mapped_tags" VECTOR_STORE_CALLER_METADATA_KWARG: Final = "litellm_vector_store_caller_metadata" ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted_tokens" +SILENT_EXPERIMENT_METADATA_KEY: Final = "_litellm_silent_experiment" INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin" BILLED_RETRY_ATTEMPT_METADATA_KEY: Final = "billed_retry_attempt" MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response" diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 473666e5ba1..ce0dc373f66 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -47,6 +47,7 @@ from litellm.constants import ( LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL, PROVIDER_REQUEST_ID_HEADERS, REDACTED_BY_LITELLM, + SILENT_EXPERIMENT_METADATA_KEY, ) from litellm.cost_calculator import ( RealtimeAPITokenUsageProcessor, @@ -305,7 +306,7 @@ else: _in_memory_loggers: Final[list[CustomLogger]] = [] _STANDARD_LOGGING_METADATA_RESOLVED_KEYS: Final[frozenset[str]] = frozenset( - ("used_client_oauth_token", "usage_object", KUBERNETES_POD_ROUTING_KEY) + ("used_client_oauth_token", "usage_object", KUBERNETES_POD_ROUTING_KEY, "is_silent_experiment") ) _STANDARD_LOGGING_METADATA_KEYS: Final[frozenset[str]] = ( frozenset(StandardLoggingMetadata.__annotations__.keys()) - _STANDARD_LOGGING_METADATA_RESOLVED_KEYS @@ -6436,6 +6437,11 @@ class StandardLoggingPayloadSetup: proxy_stamped_used_client_oauth_token(metadata, litellm_params), custom_llm_provider, ), + is_silent_experiment=( + metadata is not None + and SILENT_EXPERIMENT_METADATA_KEY in metadata + and metadata[SILENT_EXPERIMENT_METADATA_KEY] is True + ), ) if isinstance(metadata, dict): for key in metadata.keys() & _STANDARD_LOGGING_METADATA_KEYS: diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index a6e971f002a..9124e4f50e9 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -43,6 +43,7 @@ from litellm.constants import ( SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY, SESSION_ID_OMITTED_METADATA_KEY, + SILENT_EXPERIMENT_METADATA_KEY, TYPESAFE_DANGER_ZONE_HEADER_PREFIX, VECTOR_STORE_CALLER_METADATA_KWARG, X_LITELLM_DISABLE_CALLBACKS, @@ -469,12 +470,12 @@ _CLIENT_PRICING_METADATA_FIELDS: Final = frozenset({"model_info", "standard_logg _ROUTER_RESERVED_METADATA_FIELDS: Final = frozenset( { "attempted_fallbacks", - "is_silent_experiment", "original_model_group", "previous_models", "request_retry_count", CLIENT_OUTPUT_CEILING_METADATA_KEY, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY, + SILENT_EXPERIMENT_METADATA_KEY, } ) _ALLOW_CLIENT_PRICING_OVERRIDE_METADATA_KEY: Final = "allow_client_pricing_override" diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index fe7ac30e434..21da02cd572 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -113,6 +113,7 @@ from litellm.proxy.common_utils.sse_keepalive import ( wrap_passthrough_sse_bytes_with_keepalive_pings, ) from litellm.proxy.litellm_pre_call_utils import ( # noqa: F401 # legacy module exports + _ROUTER_RESERVED_METADATA_FIELDS, # pyright: ignore[reportPrivateUsage] # pass-through applies the proxy's router-owned key strip LiteLLMProxyRequestSetup, _get_dynamic_logging_metadata, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _key_or_team_allows_client_pricing_override, # pyright: ignore[reportPrivateUsage] # reuse the proxy's pricing trust policy @@ -165,7 +166,11 @@ _PRICING_MODEL_OVERRIDE_PARAMS: Final = frozenset(("base_model",)) def _is_mergeable_client_metadata(key: str, value: object) -> bool: - return not key.startswith("user_api_key_") and (key != "model_info" or isinstance(value, Mapping)) + return ( + not key.startswith("user_api_key_") + and key not in _ROUTER_RESERVED_METADATA_FIELDS + and (key != "model_info" or isinstance(value, Mapping)) + ) # Global registry to track registered pass-through routes and prevent memory leaks diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 6183ee5cb3e..875762a7284 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -31,6 +31,7 @@ from litellm.constants import ( NON_INFERENCE_CALL_TYPES, REDACTED_BY_LITELM_STRING, SESSION_ID_OMITTED_METADATA_KEY, + SILENT_EXPERIMENT_METADATA_KEY, UNKNOWN_MODEL_SPEND_LOG_MODEL, ) from litellm.constants import ( @@ -290,7 +291,8 @@ def _get_spend_logs_metadata( used_client_oauth_token=used_client_oauth_token, kubernetes_pod_routing=kubernetes_pod_routing, litellm_roi_estimator=metadata.get("litellm_roi_estimator") is True, - is_silent_experiment="is_silent_experiment" in metadata and metadata["is_silent_experiment"] is True, + is_silent_experiment=SILENT_EXPERIMENT_METADATA_KEY in metadata + and metadata[SILENT_EXPERIMENT_METADATA_KEY] is True, billed_retry_attempt=billed_retry_attempt, user_api_key_user_email=_stored_user_email(metadata), ) diff --git a/litellm/router.py b/litellm/router.py index e1271970c22..89a1124ff7b 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -78,6 +78,7 @@ from litellm.constants import ( ROUTING_REQUEST_TAGS_METADATA_KEY, RUNTIME_UPDATABLE_ROUTER_SETTINGS, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, + SILENT_EXPERIMENT_METADATA_KEY, ) from litellm.integrations.custom_guardrail import CustomGuardrail, is_guardrail_intervention from litellm.integrations.custom_logger import CustomLogger @@ -3124,8 +3125,12 @@ class Router: silent_kwargs["metadata"].pop("litellm_parent_otel_span", None) silent_kwargs["metadata"]["is_silent_experiment"] = True + silent_kwargs["metadata"][SILENT_EXPERIMENT_METADATA_KEY] = True if "litellm_metadata" in silent_kwargs and isinstance(silent_kwargs["litellm_metadata"], dict): - silent_kwargs["litellm_metadata"] = {**silent_kwargs["litellm_metadata"], "is_silent_experiment": True} + silent_kwargs["litellm_metadata"] = { + **silent_kwargs["litellm_metadata"], + SILENT_EXPERIMENT_METADATA_KEY: True, + } # Pop logging objects and call IDs to ensure a fresh logging context # This prevents collisions in the Proxy's database (spend_logs) diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index f0baceecdaa..2ee5e17d6df 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -30,7 +30,7 @@ from starlette.datastructures import UploadFile as StarletteUploadFile import litellm from litellm._logging import verbose_proxy_logger -from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS +from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS, SILENT_EXPERIMENT_METADATA_KEY from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -10338,6 +10338,42 @@ def test_init_kwargs_no_authenticated_identity_field_is_client_settable(mock_req survived: Final = {key: metadata.get(key) for key in expected if metadata.get(key) != expected[key]} assert survived == {}, f"client-supplied values survived for: {sorted(survived)}" + +def test_init_kwargs_client_metadata_cannot_mark_a_pass_through_call_as_a_silent_experiment( + mock_request, mock_user_api_key_dict +): + from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload + + started: Final = datetime(2026, 1, 15, 12, 0, 0) + result: Final = HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint( + request=mock_request(), + user_api_key_dict=mock_user_api_key_dict, + passthrough_logging_payload=PassthroughStandardLoggingPayload(url="https://test.com", request_body={}), + litellm_call_id="test-call-id", + logging_obj=LiteLLMLoggingObj( + model="test-model", + messages=[], + stream=False, + call_type="test-call-type", + start_time=started, + litellm_call_id="test-call-id", + function_id="test-function-id", + ), + _parsed_body={ + "litellm_metadata": {SILENT_EXPERIMENT_METADATA_KEY: True, "client_key": "kept"}, + "metadata": {SILENT_EXPERIMENT_METADATA_KEY: True}, + }, + ) + + row: Final = get_logging_payload( + kwargs={"model": "test-model", "litellm_params": result["litellm_params"]}, + response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]), + start_time=started, + end_time=started, + ) + assert json.loads(row["metadata"])["is_silent_experiment"] is False + assert result["litellm_params"]["metadata"]["client_key"] == "kept" + @pytest.mark.asyncio async def test_direct_passthrough_keeps_mapped_baggage_tags_outside_upstream_body(): from litellm.proxy.common_utils.http_parsing_utils import read_request_body diff --git a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py index c9ec799a9c0..f5e95f87a52 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py @@ -16,6 +16,7 @@ import litellm.constants as litellm_constants import litellm.proxy.spend_tracking.spend_tracking_utils as spend_tracking_utils from litellm.constants import KUBERNETES_POD_ROUTING_KEY, LITELLM_TRUNCATED_PAYLOAD_FIELD, LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, LITTELM_CLI_SERVICE_ACCOUNT_NAME, LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, MAX_SPEND_LOG_MODEL_NAME_LENGTH, REDACTED_BY_LITELM_STRING, SESSION_ID_GENERATED_METADATA_KEY, SESSION_ID_OMITTED_METADATA_KEY, UNKNOWN_MODEL_SPEND_LOG_MODEL from litellm.constants import CALLER_SESSION_ID_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY, MANAGED_BACKGROUND_RESPONSE_METADATA_KEY +from litellm.constants import SILENT_EXPERIMENT_METADATA_KEY from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup from litellm.llms.base_llm.ocr.transformation import OCRResponse, OCRUsageInfo from litellm.litellm_core_utils.safe_json_dumps import safe_dumps @@ -165,7 +166,12 @@ def test_get_logging_payload_includes_kubernetes_pod_routing( @pytest.mark.parametrize( ("silent_experiment_metadata", "expected"), - [({"is_silent_experiment": True}, True), ({"is_silent_experiment": "true"}, False), ({}, False)], + [ + ({SILENT_EXPERIMENT_METADATA_KEY: True}, True), + ({SILENT_EXPERIMENT_METADATA_KEY: "true"}, False), + ({"is_silent_experiment": True}, False), + ({}, False), + ], ) def test_get_logging_payload_records_whether_the_call_was_a_silent_experiment( silent_experiment_metadata: dict[str, object], expected: bool diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 7064b52ad8c..f1940fa2d2f 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -23,6 +23,7 @@ from litellm.constants import ( SERVER_STREAMING_CLASSIFICATION_MARKER, SESSION_ID_GENERATED_METADATA_KEY, SESSION_ID_OMITTED_METADATA_KEY, + SILENT_EXPERIMENT_METADATA_KEY, ) from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs from litellm.litellm_core_utils.get_provider_specific_headers import ( @@ -8278,12 +8279,12 @@ def _reserved_stamp_key(key_metadata: dict | None = None) -> UserAPIKeyAuth: _PLANTED_STAMPS = { "attempted_fallbacks": 99, - "is_silent_experiment": True, "original_model_group": "spoofed-group", "request_retry_count": -100, "previous_models": [{"exception_type": "Timeout", "deployment_id": "spoofed", "attempted_retries": 101}], "_client_output_ceiling": {"api_base": "https://attacker.example"}, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: 10**9, + SILENT_EXPERIMENT_METADATA_KEY: True, "client_key": "client_value", } @@ -8313,7 +8314,7 @@ async def test_add_litellm_data_to_request_strips_router_reserved_stamps_from_bo assert "litellm_metadata" not in updated assert "attempted_fallbacks" not in updated["metadata"] - assert "is_silent_experiment" not in updated["metadata"] + assert SILENT_EXPERIMENT_METADATA_KEY not in updated["metadata"] assert "original_model_group" not in updated["metadata"] assert "_client_output_ceiling" not in updated["metadata"] assert "request_retry_count" not in updated["metadata"] diff --git a/tests/unit/test_router_silent_experiment.py b/tests/unit/test_router_silent_experiment.py index 97a0ba146cf..cea56219215 100644 --- a/tests/unit/test_router_silent_experiment.py +++ b/tests/unit/test_router_silent_experiment.py @@ -280,13 +280,24 @@ async def test_standard_logging_payload_marks_only_the_mirrored_calls_as_silent_ metadata_by_group: Final = {payload["model_group"]: payload["metadata"] for payload in payloads} assert metadata_by_group["shadow-a"]["is_silent_experiment"] is True assert metadata_by_group["shadow-b"]["is_silent_experiment"] is True - assert "is_silent_experiment" not in metadata_by_group["primary-model"] + assert metadata_by_group["primary-model"]["is_silent_experiment"] is False + + +def _spend_row_is_silent_experiment(call: dict[str, object]) -> object: + from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload + + now: Final = datetime.datetime.now(datetime.timezone.utc) + row: Final = get_logging_payload( + kwargs=call, + response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]), + start_time=now, + end_time=now, + ) + return json.loads(row["metadata"])["is_silent_experiment"] @pytest.mark.asyncio async def test_spend_rows_mark_only_the_mirrored_calls_when_the_caller_sends_litellm_metadata(recording_logger): - from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload - router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"])) await router.acompletion( model="primary-model", @@ -296,21 +307,29 @@ async def test_spend_rows_mark_only_the_mirrored_calls_when_the_caller_sends_lit ) await _wait_for_all_successes(recording_logger, expected=3) - now: Final = datetime.datetime.now(datetime.timezone.utc) silent_by_group: Final = { - call["standard_logging_object"]["model_group"]: json.loads( - get_logging_payload( - kwargs=call, - response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]), - start_time=now, - end_time=now, - )["metadata"] - )["is_silent_experiment"] + call["standard_logging_object"]["model_group"]: _spend_row_is_silent_experiment(call) for call in recording_logger.success_kwargs } assert silent_by_group == {"primary-model": False, "shadow-a": True, "shadow-b": True} +@pytest.mark.asyncio +async def test_a_caller_sent_is_silent_experiment_does_not_mark_a_real_call_as_mirrored(recording_logger): + router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"])) + await router.acompletion( + model="shadow-b", + messages=[{"role": "user", "content": "hi"}], + mock_response="pong", + metadata={"is_silent_experiment": True}, + ) + await _wait_for_all_successes(recording_logger, expected=1) + + call: Final = recording_logger.success_kwargs[0] + assert call["standard_logging_object"]["metadata"]["is_silent_experiment"] is False + assert _spend_row_is_silent_experiment(call) is False + + def test_multiple_shadow_targets_fan_out_sync(recording_logger): router = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"])) response = router.completion( From 1fb6b63a932fe7a154936d864f5ee897ccb8d1fa Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:08:01 -0700 Subject: [PATCH 4/7] test(langfuse): expect is_silent_experiment false on every logged generation --- .../langfuse/langfuse_expected_request_body/completion.json | 1 + .../completion_with_bedrock_call.json | 1 + .../completion_with_complex_metadata.json | 1 + .../completion_with_langfuse_metadata.json | 1 + .../completion_with_no_choices.json | 1 + .../langfuse_expected_request_body/completion_with_router.json | 1 + .../langfuse_expected_request_body/completion_with_tags.json | 1 + .../completion_with_tags_stream.json | 1 + .../completion_with_vertex_call.json | 1 + .../langfuse_expected_request_body/complex_metadata.json | 1 + .../langfuse_expected_request_body/complex_metadata_2.json | 1 + .../langfuse/langfuse_expected_request_body/empty_metadata.json | 1 + .../langfuse_expected_request_body/metadata_with_function.json | 1 + .../langfuse_expected_request_body/metadata_with_lock.json | 1 + .../langfuse/langfuse_expected_request_body/nested_metadata.json | 1 + .../langfuse/langfuse_expected_request_body/simple_metadata.json | 1 + .../langfuse_expected_request_body/simple_metadata2.json | 1 + .../langfuse_expected_request_body/simple_metadata3.json | 1 + 18 files changed, 18 insertions(+) diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion.json index 906b1a42a8f..c2ae9409734 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion.json @@ -14,6 +14,7 @@ ] }, "langfuse.observation.level": "DEFAULT", + "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_bedrock_call.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_bedrock_call.json index 6f359380245..067c4683607 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_bedrock_call.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_bedrock_call.json @@ -14,6 +14,7 @@ ] }, "langfuse.observation.level": "DEFAULT", + "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "langfuse.observation.model.parameters": { "aws_region": "us-east-1" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_complex_metadata.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_complex_metadata.json index 906b1a42a8f..c2ae9409734 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_complex_metadata.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_complex_metadata.json @@ -14,6 +14,7 @@ ] }, "langfuse.observation.level": "DEFAULT", + "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_langfuse_metadata.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_langfuse_metadata.json index 5ed49cde972..440aa0827d5 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_langfuse_metadata.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_langfuse_metadata.json @@ -14,6 +14,7 @@ ] }, "langfuse.observation.level": "DEFAULT", + "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_no_choices.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_no_choices.json index b5a0737cf39..6a2096e3e16 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_no_choices.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_no_choices.json @@ -14,6 +14,7 @@ ] }, "langfuse.observation.level": "DEFAULT", + "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_router.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_router.json index 749796e0d04..af495a567f8 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_router.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_router.json @@ -14,6 +14,7 @@ ] }, "langfuse.observation.level": "DEFAULT", + "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "stream": false, diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_tags.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_tags.json index 39e26f5957d..013bcdf64b2 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_tags.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_tags.json @@ -14,6 +14,7 @@ ] }, "langfuse.observation.level": "DEFAULT", + "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_tags_stream.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_tags_stream.json index 83f4e648b8d..bfd768c81a8 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_tags_stream.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_tags_stream.json @@ -14,6 +14,7 @@ ] }, "langfuse.observation.level": "DEFAULT", + "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}", diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_vertex_call.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_vertex_call.json index d7d292390a0..a41e6b404c5 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_vertex_call.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_vertex_call.json @@ -14,6 +14,7 @@ ] }, "langfuse.observation.level": "DEFAULT", + "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gemini-3-flash-preview", "langfuse.observation.model.parameters": {}, "langfuse.observation.type": "generation", diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/complex_metadata.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/complex_metadata.json index 906b1a42a8f..c2ae9409734 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/complex_metadata.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/complex_metadata.json @@ -14,6 +14,7 @@ ] }, "langfuse.observation.level": "DEFAULT", + "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/complex_metadata_2.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/complex_metadata_2.json index 906b1a42a8f..c2ae9409734 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/complex_metadata_2.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/complex_metadata_2.json @@ -14,6 +14,7 @@ ] }, "langfuse.observation.level": "DEFAULT", + "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/empty_metadata.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/empty_metadata.json index 906b1a42a8f..c2ae9409734 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/empty_metadata.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/empty_metadata.json @@ -14,6 +14,7 @@ ] }, "langfuse.observation.level": "DEFAULT", + "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/metadata_with_function.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/metadata_with_function.json index 906b1a42a8f..c2ae9409734 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/metadata_with_function.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/metadata_with_function.json @@ -14,6 +14,7 @@ ] }, "langfuse.observation.level": "DEFAULT", + "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/metadata_with_lock.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/metadata_with_lock.json index 906b1a42a8f..c2ae9409734 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/metadata_with_lock.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/metadata_with_lock.json @@ -14,6 +14,7 @@ ] }, "langfuse.observation.level": "DEFAULT", + "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/nested_metadata.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/nested_metadata.json index 906b1a42a8f..c2ae9409734 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/nested_metadata.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/nested_metadata.json @@ -14,6 +14,7 @@ ] }, "langfuse.observation.level": "DEFAULT", + "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata.json index 906b1a42a8f..c2ae9409734 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata.json @@ -14,6 +14,7 @@ ] }, "langfuse.observation.level": "DEFAULT", + "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata2.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata2.json index 906b1a42a8f..c2ae9409734 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata2.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata2.json @@ -14,6 +14,7 @@ ] }, "langfuse.observation.level": "DEFAULT", + "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata3.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata3.json index 906b1a42a8f..c2ae9409734 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata3.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata3.json @@ -14,6 +14,7 @@ ] }, "langfuse.observation.level": "DEFAULT", + "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" From f71b2cdfc96529bf4214d09412f8230c5fc7d2ef Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:07:45 -0700 Subject: [PATCH 5/7] fix(logging): mark mirrored calls with a router-only enum value so no client route can set is_silent_experiment The router stamps SilentExperimentMarker.MIRRORED, which client JSON can never produce, so batches, Responses WebSocket frames and pass-through need no strip. Langfuse layers the flag after its requester-metadata dedupe. --- litellm/integrations/langfuse/langfuse.py | 6 ++++ litellm/litellm_core_utils/litellm_logging.py | 4 +-- litellm/proxy/litellm_pre_call_utils.py | 2 -- .../pass_through_endpoints.py | 7 +---- .../spend_tracking/spend_tracking_utils.py | 4 +-- litellm/router.py | 5 ++-- litellm/types/router.py | 4 +++ tests/unit/integrations/test_langfuse.py | 30 +++++++++++++++++++ .../test_pass_through_endpoints.py | 12 ++++---- .../test_spend_tracking_utils.py | 6 ++-- .../unit/proxy/test_litellm_pre_call_utils.py | 3 -- tests/unit/test_router_silent_experiment.py | 27 +++++++++++++---- 12 files changed, 81 insertions(+), 29 deletions(-) diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index 86f80047e52..28c0d3ac8e2 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -832,6 +832,12 @@ class LangFuseLogger: ("vertex_location", vertex_location, bool(vertex_location)), ("aws_region_name", aws_region_name, bool(aws_region_name)), ("cache_hit", kwargs.get("cache_hit") or False, "cache_hit" in kwargs), + ( + "is_silent_experiment", + "is_silent_experiment" in allowlisted_metadata + and allowlisted_metadata["is_silent_experiment"] is True, + "is_silent_experiment" in allowlisted_metadata, + ), ) enrichments: Final[Mapping[str, object]] = { key: value for key, value, include in candidate_enrichments if include diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index ce0dc373f66..cf5c163fc22 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -155,7 +155,7 @@ from litellm.types.llms.openai import ( from litellm.types.mcp import MCPPostCallResponseObject from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.rerank import RerankResponse -from litellm.types.router import RouterRateLimitError +from litellm.types.router import RouterRateLimitError, SilentExperimentMarker from litellm.types.utils import ( DEPLOYMENT_SCOPED_PRICING_FIELDS, CachingDetails, @@ -6440,7 +6440,7 @@ class StandardLoggingPayloadSetup: is_silent_experiment=( metadata is not None and SILENT_EXPERIMENT_METADATA_KEY in metadata - and metadata[SILENT_EXPERIMENT_METADATA_KEY] is True + and metadata[SILENT_EXPERIMENT_METADATA_KEY] is SilentExperimentMarker.MIRRORED ), ) if isinstance(metadata, dict): diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 9124e4f50e9..09f53cecb72 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -43,7 +43,6 @@ from litellm.constants import ( SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY, SESSION_ID_OMITTED_METADATA_KEY, - SILENT_EXPERIMENT_METADATA_KEY, TYPESAFE_DANGER_ZONE_HEADER_PREFIX, VECTOR_STORE_CALLER_METADATA_KWARG, X_LITELLM_DISABLE_CALLBACKS, @@ -475,7 +474,6 @@ _ROUTER_RESERVED_METADATA_FIELDS: Final = frozenset( "request_retry_count", CLIENT_OUTPUT_CEILING_METADATA_KEY, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY, - SILENT_EXPERIMENT_METADATA_KEY, } ) _ALLOW_CLIENT_PRICING_OVERRIDE_METADATA_KEY: Final = "allow_client_pricing_override" diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 21da02cd572..fe7ac30e434 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -113,7 +113,6 @@ from litellm.proxy.common_utils.sse_keepalive import ( wrap_passthrough_sse_bytes_with_keepalive_pings, ) from litellm.proxy.litellm_pre_call_utils import ( # noqa: F401 # legacy module exports - _ROUTER_RESERVED_METADATA_FIELDS, # pyright: ignore[reportPrivateUsage] # pass-through applies the proxy's router-owned key strip LiteLLMProxyRequestSetup, _get_dynamic_logging_metadata, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _key_or_team_allows_client_pricing_override, # pyright: ignore[reportPrivateUsage] # reuse the proxy's pricing trust policy @@ -166,11 +165,7 @@ _PRICING_MODEL_OVERRIDE_PARAMS: Final = frozenset(("base_model",)) def _is_mergeable_client_metadata(key: str, value: object) -> bool: - return ( - not key.startswith("user_api_key_") - and key not in _ROUTER_RESERVED_METADATA_FIELDS - and (key != "model_info" or isinstance(value, Mapping)) - ) + return not key.startswith("user_api_key_") and (key != "model_info" or isinstance(value, Mapping)) # Global registry to track registered pass-through routes and prevent memory leaks diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 875762a7284..3bddb89b04a 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -75,7 +75,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( SETTLES_BACKGROUND_RESPONSE_KWARG, BackgroundResponseSettlement, ) -from litellm.types.router import DeploymentTypedDict, LiteLLM_Params +from litellm.types.router import DeploymentTypedDict, LiteLLM_Params, SilentExperimentMarker from litellm.types.utils import ( BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, PROMPT_CARRYING_GUARDRAIL_FIELDS, @@ -292,7 +292,7 @@ def _get_spend_logs_metadata( kubernetes_pod_routing=kubernetes_pod_routing, litellm_roi_estimator=metadata.get("litellm_roi_estimator") is True, is_silent_experiment=SILENT_EXPERIMENT_METADATA_KEY in metadata - and metadata[SILENT_EXPERIMENT_METADATA_KEY] is True, + and metadata[SILENT_EXPERIMENT_METADATA_KEY] is SilentExperimentMarker.MIRRORED, billed_retry_attempt=billed_retry_attempt, user_api_key_user_email=_stored_user_email(metadata), ) diff --git a/litellm/router.py b/litellm/router.py index 89a1124ff7b..2f61f274ab3 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -409,6 +409,7 @@ from litellm.types.router import ( RoutingPlugin, RoutingStrategy, SearchToolTypedDict, + SilentExperimentMarker, TaggedPreRoutingStrategy, holds_secret_pointer, ) @@ -3125,11 +3126,11 @@ class Router: silent_kwargs["metadata"].pop("litellm_parent_otel_span", None) silent_kwargs["metadata"]["is_silent_experiment"] = True - silent_kwargs["metadata"][SILENT_EXPERIMENT_METADATA_KEY] = True + silent_kwargs["metadata"][SILENT_EXPERIMENT_METADATA_KEY] = SilentExperimentMarker.MIRRORED if "litellm_metadata" in silent_kwargs and isinstance(silent_kwargs["litellm_metadata"], dict): silent_kwargs["litellm_metadata"] = { **silent_kwargs["litellm_metadata"], - SILENT_EXPERIMENT_METADATA_KEY: True, + SILENT_EXPERIMENT_METADATA_KEY: SilentExperimentMarker.MIRRORED, } # Pop logging objects and call IDs to ensure a fresh logging context diff --git a/litellm/types/router.py b/litellm/types/router.py index cd1389aea9d..2a4bd095255 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -66,6 +66,10 @@ from .utils import ( AllowedModelRegion: TypeAlias = Literal["eu", "us"] +class SilentExperimentMarker(str, enum.Enum): + MIRRORED = "mirrored" + + class FallbackSkipReason(str, enum.Enum): NOT_AUTHORIZED = "not_authorized" OVER_BUDGET = "over_budget" diff --git a/tests/unit/integrations/test_langfuse.py b/tests/unit/integrations/test_langfuse.py index 00d85b64566..462f82eeb52 100644 --- a/tests/unit/integrations/test_langfuse.py +++ b/tests/unit/integrations/test_langfuse.py @@ -830,6 +830,36 @@ class TestLangfuseUsageDetails(unittest.TestCase): assert generation_metadata["litellm_response_cost"] == 0.25 assert generation_metadata["api_base"] == "https://real-api-base" + def test_caller_sent_is_silent_experiment_cannot_hide_the_mirrored_call_flag(self): + payload = self._build_standard_logging_payload(trace_id="canary-trace-id") + payload["metadata"]["is_silent_experiment"] = True + payload["metadata"]["requester_metadata"] = {"is_silent_experiment": False} + kwargs = self._build_langfuse_kwargs(payload) + metadata = self._canary_request_metadata() + self.use_real_langfuse_client() + + with patch( + "litellm.integrations.langfuse.langfuse._add_prompt_to_generation_params", + side_effect=lambda generation_params, **kw: generation_params, + create=True, + ): + self.logger._log_langfuse_v2( + user_id="user-1", + metadata=metadata, + litellm_params={"metadata": metadata}, + output=None, + start_time=datetime.datetime(2024, 1, 1, 12, 0, 0), + end_time=datetime.datetime(2024, 1, 1, 12, 0, 1), + kwargs=kwargs, + optional_params={}, + input=None, + response_obj=None, + level="INFO", + litellm_call_id="canary-call-id", + ) + + assert self.exported_generation_metadata()["is_silent_experiment"] is True + def test_generation_metadata_carries_the_call_id_and_response_id(self): """ v2's generation id was ``time-_``, so a generation could diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 2ee5e17d6df..b7124236fbf 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -63,6 +63,7 @@ from litellm.proxy.pass_through_endpoints.success_handler import ( from litellm.proxy.route_llm_request import ProxyModelNotFoundError from litellm.types import utils as types_utils from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.router import SilentExperimentMarker from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, @@ -10339,14 +10340,15 @@ def test_init_kwargs_no_authenticated_identity_field_is_client_settable(mock_req assert survived == {}, f"client-supplied values survived for: {sorted(survived)}" +@pytest.mark.parametrize("planted", [True, SilentExperimentMarker.MIRRORED.value]) def test_init_kwargs_client_metadata_cannot_mark_a_pass_through_call_as_a_silent_experiment( - mock_request, mock_user_api_key_dict -): + planted: object, mock_user_api_key_dict: UserAPIKeyAuth +) -> None: from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload started: Final = datetime(2026, 1, 15, 12, 0, 0) result: Final = HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint( - request=mock_request(), + request=Request({"type": "http", "method": "POST", "path": "/test", "headers": [], "query_string": b""}), user_api_key_dict=mock_user_api_key_dict, passthrough_logging_payload=PassthroughStandardLoggingPayload(url="https://test.com", request_body={}), litellm_call_id="test-call-id", @@ -10360,8 +10362,8 @@ def test_init_kwargs_client_metadata_cannot_mark_a_pass_through_call_as_a_silent function_id="test-function-id", ), _parsed_body={ - "litellm_metadata": {SILENT_EXPERIMENT_METADATA_KEY: True, "client_key": "kept"}, - "metadata": {SILENT_EXPERIMENT_METADATA_KEY: True}, + "litellm_metadata": {SILENT_EXPERIMENT_METADATA_KEY: planted, "client_key": "kept"}, + "metadata": {SILENT_EXPERIMENT_METADATA_KEY: planted}, }, ) diff --git a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py index f5e95f87a52..ec3ba361b3f 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py @@ -17,6 +17,7 @@ import litellm.proxy.spend_tracking.spend_tracking_utils as spend_tracking_utils from litellm.constants import KUBERNETES_POD_ROUTING_KEY, LITELLM_TRUNCATED_PAYLOAD_FIELD, LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, LITTELM_CLI_SERVICE_ACCOUNT_NAME, LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, MAX_SPEND_LOG_MODEL_NAME_LENGTH, REDACTED_BY_LITELM_STRING, SESSION_ID_GENERATED_METADATA_KEY, SESSION_ID_OMITTED_METADATA_KEY, UNKNOWN_MODEL_SPEND_LOG_MODEL from litellm.constants import CALLER_SESSION_ID_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY, MANAGED_BACKGROUND_RESPONSE_METADATA_KEY from litellm.constants import SILENT_EXPERIMENT_METADATA_KEY +from litellm.types.router import SilentExperimentMarker from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup from litellm.llms.base_llm.ocr.transformation import OCRResponse, OCRUsageInfo from litellm.litellm_core_utils.safe_json_dumps import safe_dumps @@ -167,8 +168,9 @@ def test_get_logging_payload_includes_kubernetes_pod_routing( @pytest.mark.parametrize( ("silent_experiment_metadata", "expected"), [ - ({SILENT_EXPERIMENT_METADATA_KEY: True}, True), - ({SILENT_EXPERIMENT_METADATA_KEY: "true"}, False), + ({SILENT_EXPERIMENT_METADATA_KEY: SilentExperimentMarker.MIRRORED}, True), + ({SILENT_EXPERIMENT_METADATA_KEY: SilentExperimentMarker.MIRRORED.value}, False), + ({SILENT_EXPERIMENT_METADATA_KEY: True}, False), ({"is_silent_experiment": True}, False), ({}, False), ], diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index f1940fa2d2f..c2a5b01fe4e 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -23,7 +23,6 @@ from litellm.constants import ( SERVER_STREAMING_CLASSIFICATION_MARKER, SESSION_ID_GENERATED_METADATA_KEY, SESSION_ID_OMITTED_METADATA_KEY, - SILENT_EXPERIMENT_METADATA_KEY, ) from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs from litellm.litellm_core_utils.get_provider_specific_headers import ( @@ -8284,7 +8283,6 @@ _PLANTED_STAMPS = { "previous_models": [{"exception_type": "Timeout", "deployment_id": "spoofed", "attempted_retries": 101}], "_client_output_ceiling": {"api_base": "https://attacker.example"}, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: 10**9, - SILENT_EXPERIMENT_METADATA_KEY: True, "client_key": "client_value", } @@ -8314,7 +8312,6 @@ async def test_add_litellm_data_to_request_strips_router_reserved_stamps_from_bo assert "litellm_metadata" not in updated assert "attempted_fallbacks" not in updated["metadata"] - assert SILENT_EXPERIMENT_METADATA_KEY not in updated["metadata"] assert "original_model_group" not in updated["metadata"] assert "_client_output_ceiling" not in updated["metadata"] assert "request_retry_count" not in updated["metadata"] diff --git a/tests/unit/test_router_silent_experiment.py b/tests/unit/test_router_silent_experiment.py index cea56219215..c455153b3cf 100644 --- a/tests/unit/test_router_silent_experiment.py +++ b/tests/unit/test_router_silent_experiment.py @@ -12,11 +12,13 @@ import pytest import respx import litellm +from litellm.constants import SILENT_EXPERIMENT_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.router import Router from litellm.router import _silent_experiment_kwargs_snapshot from litellm.router import _silent_experiment_targets +from litellm.types.router import SilentExperimentMarker class _RecordingLogger(CustomLogger): @@ -265,7 +267,9 @@ async def test_multiple_shadow_targets_fan_out_async(recording_logger): @pytest.mark.asyncio -async def test_standard_logging_payload_marks_only_the_mirrored_calls_as_silent_experiments(recording_logger): +async def test_standard_logging_payload_marks_only_the_mirrored_calls_as_silent_experiments( + recording_logger: _RecordingLogger, +) -> None: router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"])) response: Final = await router.acompletion( model="primary-model", @@ -286,7 +290,7 @@ async def test_standard_logging_payload_marks_only_the_mirrored_calls_as_silent_ def _spend_row_is_silent_experiment(call: dict[str, object]) -> object: from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload - now: Final = datetime.datetime.now(datetime.timezone.utc) + now: Final = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc) row: Final = get_logging_payload( kwargs=call, response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]), @@ -297,7 +301,9 @@ def _spend_row_is_silent_experiment(call: dict[str, object]) -> object: @pytest.mark.asyncio -async def test_spend_rows_mark_only_the_mirrored_calls_when_the_caller_sends_litellm_metadata(recording_logger): +async def test_spend_rows_mark_only_the_mirrored_calls_when_the_caller_sends_litellm_metadata( + recording_logger: _RecordingLogger, +) -> None: router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"])) await router.acompletion( model="primary-model", @@ -315,13 +321,24 @@ async def test_spend_rows_mark_only_the_mirrored_calls_when_the_caller_sends_lit @pytest.mark.asyncio -async def test_a_caller_sent_is_silent_experiment_does_not_mark_a_real_call_as_mirrored(recording_logger): +@pytest.mark.parametrize( + ("bucket", "planted"), + [ + ("metadata", {"is_silent_experiment": True}), + ("metadata", {SILENT_EXPERIMENT_METADATA_KEY: SilentExperimentMarker.MIRRORED.value}), + ("litellm_metadata", {SILENT_EXPERIMENT_METADATA_KEY: SilentExperimentMarker.MIRRORED.value}), + ("litellm_metadata", {SILENT_EXPERIMENT_METADATA_KEY: True}), + ], +) +async def test_a_caller_sent_silent_experiment_flag_does_not_mark_a_real_call_as_mirrored( + recording_logger: _RecordingLogger, bucket: str, planted: dict[str, object] +) -> None: router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"])) await router.acompletion( model="shadow-b", messages=[{"role": "user", "content": "hi"}], mock_response="pong", - metadata={"is_silent_experiment": True}, + **{bucket: planted}, ) await _wait_for_all_successes(recording_logger, expected=1) From 5a51c4d1798c1dae727deb479bed244c21deaf4f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 21 Oct 2026 00:22:49 -0700 Subject: [PATCH 6/7] fix(logging): set is_silent_experiment only on mirrored calls so real calls keep their payloads Real calls no longer gain an is_silent_experiment key in spend rows or the standard logging payload, so OTEL, S3, GCS and every other logger receive what they did before. The Langfuse regression test now runs the real prompt helper. --- litellm/litellm_core_utils/litellm_logging.py | 8 +++-- .../spend_tracking/spend_tracking_utils.py | 11 ++++--- litellm/types/router.py | 4 +++ .../gcs_pubsub/spend_logs_payload.json | 2 +- .../completion.json | 1 - .../completion_with_bedrock_call.json | 1 - .../completion_with_complex_metadata.json | 1 - .../completion_with_langfuse_metadata.json | 1 - .../completion_with_no_choices.json | 1 - .../completion_with_router.json | 1 - .../completion_with_tags.json | 1 - .../completion_with_tags_stream.json | 1 - .../completion_with_vertex_call.json | 1 - .../complex_metadata.json | 1 - .../complex_metadata_2.json | 1 - .../empty_metadata.json | 1 - .../metadata_with_function.json | 1 - .../metadata_with_lock.json | 1 - .../nested_metadata.json | 1 - .../simple_metadata.json | 1 - .../simple_metadata2.json | 1 - .../simple_metadata3.json | 1 - tests/unit/integrations/test_langfuse.py | 33 ++++++++----------- .../test_pass_through_endpoints.py | 2 +- .../test_spend_management_endpoints.py | 2 +- .../test_spend_tracking_utils.py | 14 ++++---- tests/unit/test_router_silent_experiment.py | 10 +++--- 27 files changed, 45 insertions(+), 59 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index cf5c163fc22..39dcc874c58 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -155,7 +155,7 @@ from litellm.types.llms.openai import ( from litellm.types.mcp import MCPPostCallResponseObject from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.rerank import RerankResponse -from litellm.types.router import RouterRateLimitError, SilentExperimentMarker +from litellm.types.router import RouterRateLimitError, SilentExperimentFlag, SilentExperimentMarker from litellm.types.utils import ( DEPLOYMENT_SCOPED_PRICING_FIELDS, CachingDetails, @@ -6437,10 +6437,12 @@ class StandardLoggingPayloadSetup: proxy_stamped_used_client_oauth_token(metadata, litellm_params), custom_llm_provider, ), - is_silent_experiment=( - metadata is not None + **( + SilentExperimentFlag(is_silent_experiment=True) + if metadata is not None and SILENT_EXPERIMENT_METADATA_KEY in metadata and metadata[SILENT_EXPERIMENT_METADATA_KEY] is SilentExperimentMarker.MIRRORED + else SilentExperimentFlag() ), ) if isinstance(metadata, dict): diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 3bddb89b04a..5107a662717 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -75,7 +75,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( SETTLES_BACKGROUND_RESPONSE_KWARG, BackgroundResponseSettlement, ) -from litellm.types.router import DeploymentTypedDict, LiteLLM_Params, SilentExperimentMarker +from litellm.types.router import DeploymentTypedDict, LiteLLM_Params, SilentExperimentFlag, SilentExperimentMarker from litellm.types.utils import ( BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, PROMPT_CARRYING_GUARDRAIL_FIELDS, @@ -253,7 +253,6 @@ def _get_spend_logs_metadata( guardrail_information=None, internal_call_origin=None, litellm_roi_estimator=False, - is_silent_experiment=False, billed_retry_attempt=False, eval_information=None, cold_storage_object_key=cold_storage_object_key, @@ -291,8 +290,12 @@ def _get_spend_logs_metadata( used_client_oauth_token=used_client_oauth_token, kubernetes_pod_routing=kubernetes_pod_routing, litellm_roi_estimator=metadata.get("litellm_roi_estimator") is True, - is_silent_experiment=SILENT_EXPERIMENT_METADATA_KEY in metadata - and metadata[SILENT_EXPERIMENT_METADATA_KEY] is SilentExperimentMarker.MIRRORED, + **( + SilentExperimentFlag(is_silent_experiment=True) + if SILENT_EXPERIMENT_METADATA_KEY in metadata + and metadata[SILENT_EXPERIMENT_METADATA_KEY] is SilentExperimentMarker.MIRRORED + else SilentExperimentFlag() + ), billed_retry_attempt=billed_retry_attempt, user_api_key_user_email=_stored_user_email(metadata), ) diff --git a/litellm/types/router.py b/litellm/types/router.py index 2a4bd095255..091ca6ac200 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -70,6 +70,10 @@ class SilentExperimentMarker(str, enum.Enum): MIRRORED = "mirrored" +class SilentExperimentFlag(TypedDict, total=False): + is_silent_experiment: ReadOnly[bool] + + class FallbackSkipReason(str, enum.Enum): NOT_AUTHORIZED = "not_authorized" OVER_BUDGET = "over_budget" diff --git a/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json b/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json index 6b172c687a3..6e5461c3b60 100644 --- a/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json +++ b/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json @@ -11,7 +11,7 @@ "user": "", "team_id": "", "organization_id": "", - "metadata": "{\"actor_agent_id\": null, \"target_agent_id\": null, \"billing_agent_id\": null, \"agent_execution_mode\": null, \"verified_human_user_id\": null, \"used_client_oauth_token\": null, \"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"litellm_roi_estimator\": false, \"is_silent_experiment\": false, \"billed_retry_attempt\": false, \"router_metadata\": null, \"autorouter_savings_estimate\": null, \"autorouter_baseline_observation\": null, \"azure_spillover\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_user_email\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"user_agent\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null, \"kubernetes_pod_routing\": null}", + "metadata": "{\"actor_agent_id\": null, \"target_agent_id\": null, \"billing_agent_id\": null, \"agent_execution_mode\": null, \"verified_human_user_id\": null, \"used_client_oauth_token\": null, \"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"litellm_roi_estimator\": false, \"billed_retry_attempt\": false, \"router_metadata\": null, \"autorouter_savings_estimate\": null, \"autorouter_baseline_observation\": null, \"azure_spillover\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_user_email\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"user_agent\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null, \"kubernetes_pod_routing\": null}", "cache_key": "Cache OFF", "spend": 0.00022500000000000002, "total_tokens": 30, diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion.json index c2ae9409734..906b1a42a8f 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_bedrock_call.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_bedrock_call.json index 067c4683607..6f359380245 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_bedrock_call.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_bedrock_call.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "langfuse.observation.model.parameters": { "aws_region": "us-east-1" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_complex_metadata.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_complex_metadata.json index c2ae9409734..906b1a42a8f 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_complex_metadata.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_complex_metadata.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_langfuse_metadata.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_langfuse_metadata.json index 440aa0827d5..5ed49cde972 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_langfuse_metadata.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_langfuse_metadata.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_no_choices.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_no_choices.json index 6a2096e3e16..b5a0737cf39 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_no_choices.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_no_choices.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_router.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_router.json index af495a567f8..749796e0d04 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_router.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_router.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "stream": false, diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_tags.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_tags.json index 013bcdf64b2..39e26f5957d 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_tags.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_tags.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_tags_stream.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_tags_stream.json index bfd768c81a8..83f4e648b8d 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_tags_stream.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_tags_stream.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}", diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_vertex_call.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_vertex_call.json index a41e6b404c5..d7d292390a0 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_vertex_call.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_vertex_call.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gemini-3-flash-preview", "langfuse.observation.model.parameters": {}, "langfuse.observation.type": "generation", diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/complex_metadata.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/complex_metadata.json index c2ae9409734..906b1a42a8f 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/complex_metadata.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/complex_metadata.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/complex_metadata_2.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/complex_metadata_2.json index c2ae9409734..906b1a42a8f 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/complex_metadata_2.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/complex_metadata_2.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/empty_metadata.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/empty_metadata.json index c2ae9409734..906b1a42a8f 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/empty_metadata.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/empty_metadata.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/metadata_with_function.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/metadata_with_function.json index c2ae9409734..906b1a42a8f 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/metadata_with_function.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/metadata_with_function.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/metadata_with_lock.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/metadata_with_lock.json index c2ae9409734..906b1a42a8f 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/metadata_with_lock.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/metadata_with_lock.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/nested_metadata.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/nested_metadata.json index c2ae9409734..906b1a42a8f 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/nested_metadata.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/nested_metadata.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata.json index c2ae9409734..906b1a42a8f 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata2.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata2.json index c2ae9409734..906b1a42a8f 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata2.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata2.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata3.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata3.json index c2ae9409734..906b1a42a8f 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata3.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata3.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/test_langfuse.py b/tests/unit/integrations/test_langfuse.py index 462f82eeb52..2d1e49e32df 100644 --- a/tests/unit/integrations/test_langfuse.py +++ b/tests/unit/integrations/test_langfuse.py @@ -838,25 +838,20 @@ class TestLangfuseUsageDetails(unittest.TestCase): metadata = self._canary_request_metadata() self.use_real_langfuse_client() - with patch( - "litellm.integrations.langfuse.langfuse._add_prompt_to_generation_params", - side_effect=lambda generation_params, **kw: generation_params, - create=True, - ): - self.logger._log_langfuse_v2( - user_id="user-1", - metadata=metadata, - litellm_params={"metadata": metadata}, - output=None, - start_time=datetime.datetime(2024, 1, 1, 12, 0, 0), - end_time=datetime.datetime(2024, 1, 1, 12, 0, 1), - kwargs=kwargs, - optional_params={}, - input=None, - response_obj=None, - level="INFO", - litellm_call_id="canary-call-id", - ) + self.logger._log_langfuse_v2( + user_id="user-1", + metadata=metadata, + litellm_params={"metadata": metadata}, + output=None, + start_time=datetime.datetime(2024, 1, 1, 12, 0, 0), + end_time=datetime.datetime(2024, 1, 1, 12, 0, 1), + kwargs=kwargs, + optional_params={}, + input=None, + response_obj=None, + level="INFO", + litellm_call_id="canary-call-id", + ) assert self.exported_generation_metadata()["is_silent_experiment"] is True diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index b7124236fbf..58d62763a84 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -10373,7 +10373,7 @@ def test_init_kwargs_client_metadata_cannot_mark_a_pass_through_call_as_a_silent start_time=started, end_time=started, ) - assert json.loads(row["metadata"])["is_silent_experiment"] is False + assert "is_silent_experiment" not in json.loads(row["metadata"]) assert result["litellm_params"]["metadata"]["client_key"] == "kept" @pytest.mark.asyncio diff --git a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py index 695452fce40..23bbe521670 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py @@ -3725,7 +3725,7 @@ class TestSpendLogsPayload: "model": "gpt-4o", "user": "", "team_id": "", - "metadata": '{"user_api_key_user_email": null, "actor_agent_id": null, "target_agent_id": null, "billing_agent_id": null, "agent_execution_mode": null, "verified_human_user_id": null, "applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "azure_spillover": null, "used_client_oauth_token": null, "litellm_roi_estimator": false, "is_silent_experiment": false, "billed_retry_attempt": false, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}', + "metadata": '{"user_api_key_user_email": null, "actor_agent_id": null, "target_agent_id": null, "billing_agent_id": null, "agent_execution_mode": null, "verified_human_user_id": null, "applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "azure_spillover": null, "used_client_oauth_token": null, "litellm_roi_estimator": false, "billed_retry_attempt": false, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}', "cache_key": "Cache OFF", "spend": 0.00022500000000000002, "total_tokens": 30, diff --git a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py index ec3ba361b3f..caca6e03790 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py @@ -169,14 +169,14 @@ def test_get_logging_payload_includes_kubernetes_pod_routing( ("silent_experiment_metadata", "expected"), [ ({SILENT_EXPERIMENT_METADATA_KEY: SilentExperimentMarker.MIRRORED}, True), - ({SILENT_EXPERIMENT_METADATA_KEY: SilentExperimentMarker.MIRRORED.value}, False), - ({SILENT_EXPERIMENT_METADATA_KEY: True}, False), - ({"is_silent_experiment": True}, False), - ({}, False), + ({SILENT_EXPERIMENT_METADATA_KEY: SilentExperimentMarker.MIRRORED.value}, None), + ({SILENT_EXPERIMENT_METADATA_KEY: True}, None), + ({"is_silent_experiment": True}, None), + ({}, None), ], ) def test_get_logging_payload_records_whether_the_call_was_a_silent_experiment( - silent_experiment_metadata: dict[str, object], expected: bool + silent_experiment_metadata: dict[str, object], expected: bool | None ) -> None: now: Final = datetime.datetime(2026, 1, 1, tzinfo=timezone.utc) payload: Final = get_logging_payload( @@ -189,7 +189,7 @@ def test_get_logging_payload_records_whether_the_call_was_a_silent_experiment( end_time=now, ) - assert json.loads(payload["metadata"])["is_silent_experiment"] is expected + assert json.loads(payload["metadata"]).get("is_silent_experiment") is expected @pytest.mark.parametrize("store_prompts,redact", [(True, False), (False, False), (True, True)]) @@ -6642,7 +6642,7 @@ def test_spend_logs_payload(model_id: str | None): ) assert len(payload["request_id"]) > 0 - expected_metadata_keys: Final = SpendLogsMetadata.__annotations__.keys() + expected_metadata_keys: Final = SpendLogsMetadata.__annotations__.keys() - {"is_silent_experiment"} assert "metadata" in payload assert isinstance(payload["metadata"], str) diff --git a/tests/unit/test_router_silent_experiment.py b/tests/unit/test_router_silent_experiment.py index c455153b3cf..7b3435a73c2 100644 --- a/tests/unit/test_router_silent_experiment.py +++ b/tests/unit/test_router_silent_experiment.py @@ -284,7 +284,7 @@ async def test_standard_logging_payload_marks_only_the_mirrored_calls_as_silent_ metadata_by_group: Final = {payload["model_group"]: payload["metadata"] for payload in payloads} assert metadata_by_group["shadow-a"]["is_silent_experiment"] is True assert metadata_by_group["shadow-b"]["is_silent_experiment"] is True - assert metadata_by_group["primary-model"]["is_silent_experiment"] is False + assert "is_silent_experiment" not in metadata_by_group["primary-model"] def _spend_row_is_silent_experiment(call: dict[str, object]) -> object: @@ -297,7 +297,7 @@ def _spend_row_is_silent_experiment(call: dict[str, object]) -> object: start_time=now, end_time=now, ) - return json.loads(row["metadata"])["is_silent_experiment"] + return json.loads(row["metadata"]).get("is_silent_experiment") @pytest.mark.asyncio @@ -317,7 +317,7 @@ async def test_spend_rows_mark_only_the_mirrored_calls_when_the_caller_sends_lit call["standard_logging_object"]["model_group"]: _spend_row_is_silent_experiment(call) for call in recording_logger.success_kwargs } - assert silent_by_group == {"primary-model": False, "shadow-a": True, "shadow-b": True} + assert silent_by_group == {"primary-model": None, "shadow-a": True, "shadow-b": True} @pytest.mark.asyncio @@ -343,8 +343,8 @@ async def test_a_caller_sent_silent_experiment_flag_does_not_mark_a_real_call_as await _wait_for_all_successes(recording_logger, expected=1) call: Final = recording_logger.success_kwargs[0] - assert call["standard_logging_object"]["metadata"]["is_silent_experiment"] is False - assert _spend_row_is_silent_experiment(call) is False + assert "is_silent_experiment" not in call["standard_logging_object"]["metadata"] + assert _spend_row_is_silent_experiment(call) is None def test_multiple_shadow_targets_fan_out_sync(recording_logger): From bc1491409544fb31e578313b6589cf7859834c1b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 21 Oct 2026 06:08:31 -0700 Subject: [PATCH 7/7] fix(logging): keep the mirror marker out of callback metadata and read the spend-row flag from the logging payload Logging now strips the router's mirror marker from the request metadata buckets before any callback or vendor sees it, without touching the router's own dicts, so retries keep it. Spend rows read is_silent_experiment from the standard logging payload, which survives the collector's JSON round trip, so collector-mode rows keep the flag too. --- litellm/litellm_core_utils/litellm_logging.py | 36 ++++++++--- .../spend_tracking/spend_tracking_utils.py | 16 ++--- .../test_spend_tracking_utils.py | 29 --------- tests/unit/test_router_silent_experiment.py | 63 +++++++++++++++++-- 4 files changed, 92 insertions(+), 52 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 32425e12e62..17886c45c89 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -309,6 +309,7 @@ _in_memory_loggers: Final[list[CustomLogger]] = [] _STANDARD_LOGGING_METADATA_RESOLVED_KEYS: Final[frozenset[str]] = frozenset( ("used_client_oauth_token", "usage_object", KUBERNETES_POD_ROUTING_KEY, "is_silent_experiment") ) +_SILENT_EXPERIMENT_BUCKETS: Final = ("metadata", "litellm_metadata") _STANDARD_LOGGING_METADATA_KEYS: Final[frozenset[str]] = ( frozenset(StandardLoggingMetadata.__annotations__.keys()) - _STANDARD_LOGGING_METADATA_RESOLVED_KEYS ) @@ -798,6 +799,7 @@ class Logging(LiteLLMLoggingBaseClass): litellm_params = scrub_sensitive_keys_in_metadata(litellm_params) self.litellm_params = litellm_params + self.is_silent_experiment = False # Initialize cost breakdown field self.cost_breakdown: CostBreakdown | None = None @@ -1113,6 +1115,7 @@ class Logging(LiteLLMLoggingBaseClass): if model is not None: self.model = model self.user = user + self.is_silent_experiment = self.is_silent_experiment or is_silent_experiment_call(litellm_params) self.litellm_params = { **self.litellm_params, **scrub_sensitive_keys_in_metadata(litellm_params), @@ -6491,6 +6494,7 @@ class StandardLoggingPayloadSetup: start_time: dt_object | None = None, response_id: str | None = None, custom_llm_provider: str | None = None, + is_silent_experiment: bool = False, ) -> StandardLoggingMetadata: """ Clean and filter the metadata dictionary to include only the specified keys in StandardLoggingMetadata. @@ -6564,13 +6568,7 @@ class StandardLoggingPayloadSetup: proxy_stamped_used_client_oauth_token(metadata, litellm_params), custom_llm_provider, ), - **( - SilentExperimentFlag(is_silent_experiment=True) - if metadata is not None - and SILENT_EXPERIMENT_METADATA_KEY in metadata - and metadata[SILENT_EXPERIMENT_METADATA_KEY] is SilentExperimentMarker.MIRRORED - else SilentExperimentFlag() - ), + **(SilentExperimentFlag(is_silent_experiment=True) if is_silent_experiment else SilentExperimentFlag()), ) if isinstance(metadata, dict): for key in metadata.keys() & _STANDARD_LOGGING_METADATA_KEYS: @@ -7373,6 +7371,7 @@ def get_standard_logging_object_payload( start_time=start_time, response_id=id, custom_llm_provider=selected_provider if isinstance(selected_provider, str) else None, + is_silent_experiment=logging_obj.is_silent_experiment is True, ) _request_body: Final = proxy_server_request.get("body", {}) end_user_id: Final = clean_metadata["user_api_key_end_user_id"] or _request_body.get( @@ -7667,7 +7666,9 @@ def get_standard_logging_metadata( return clean_metadata -def scrub_sensitive_keys_in_metadata(litellm_params: dict | None): +def scrub_sensitive_keys_in_metadata( + litellm_params: dict[str, Any] | None, # mutable-ok: the langfuse masking function is moved onto these params +) -> dict[str, Any]: # mutable-ok: loggers write into the returned params and their metadata if litellm_params is None: litellm_params = {} @@ -7682,7 +7683,24 @@ def scrub_sensitive_keys_in_metadata(litellm_params: dict | None): litellm_params["_langfuse_masking_function"] = masking_fn litellm_params["metadata"] = metadata - return litellm_params + if not is_silent_experiment_call(litellm_params): + return litellm_params + return { + name: ( + {key: value for key, value in bucket.items() if key != SILENT_EXPERIMENT_METADATA_KEY} + if name in _SILENT_EXPERIMENT_BUCKETS and _carries_silent_experiment_marker(bucket) + else bucket + ) + for name, bucket in litellm_params.items() + } + + +def _carries_silent_experiment_marker(bucket: object) -> bool: + return isinstance(bucket, dict) and bucket.get(SILENT_EXPERIMENT_METADATA_KEY) is SilentExperimentMarker.MIRRORED + + +def is_silent_experiment_call(litellm_params: Mapping[str, object]) -> bool: + return any(_carries_silent_experiment_marker(litellm_params.get(name)) for name in _SILENT_EXPERIMENT_BUCKETS) # integration helper function diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 189c67bd9cf..833ee71ae9c 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -32,7 +32,6 @@ from litellm.constants import ( NON_INFERENCE_CALL_TYPES, REDACTED_BY_LITELM_STRING, SESSION_ID_OMITTED_METADATA_KEY, - SILENT_EXPERIMENT_METADATA_KEY, UNKNOWN_MODEL_SPEND_LOG_MODEL, ) from litellm.constants import ( @@ -81,7 +80,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( SETTLES_BACKGROUND_RESPONSE_KWARG, BackgroundResponseSettlement, ) -from litellm.types.router import DeploymentTypedDict, LiteLLM_Params, SilentExperimentFlag, SilentExperimentMarker +from litellm.types.router import DeploymentTypedDict, LiteLLM_Params, SilentExperimentFlag from litellm.types.utils import ( BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, PROMPT_CARRYING_GUARDRAIL_FIELDS, @@ -229,6 +228,7 @@ def _get_spend_logs_metadata( azure_spillover: AzureSpillover | None = None, used_client_oauth_token: bool | None = None, kubernetes_pod_routing: StandardLoggingKubernetesPodRouting | None = None, + is_silent_experiment: bool = False, ) -> SpendLogsMetadata: if metadata is None: return SpendLogsMetadata( @@ -296,12 +296,7 @@ def _get_spend_logs_metadata( used_client_oauth_token=used_client_oauth_token, kubernetes_pod_routing=kubernetes_pod_routing, litellm_roi_estimator=metadata.get("litellm_roi_estimator") is True, - **( - SilentExperimentFlag(is_silent_experiment=True) - if SILENT_EXPERIMENT_METADATA_KEY in metadata - and metadata[SILENT_EXPERIMENT_METADATA_KEY] is SilentExperimentMarker.MIRRORED - else SilentExperimentFlag() - ), + **(SilentExperimentFlag(is_silent_experiment=True) if is_silent_experiment else SilentExperimentFlag()), billed_retry_attempt=billed_retry_attempt, user_api_key_user_email=_stored_user_email(metadata), ) @@ -746,6 +741,11 @@ def get_logging_payload( # clean up litellm metadata clean_metadata = _get_spend_logs_metadata( persisted_metadata, + is_silent_experiment=( + standard_logging_payload is not None + and "is_silent_experiment" in standard_logging_payload["metadata"] + and standard_logging_payload["metadata"]["is_silent_experiment"] + ), applied_guardrails=( standard_logging_payload["metadata"].get("applied_guardrails", None) if standard_logging_payload is not None diff --git a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py index faa307bd9c6..190ffb18ca9 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py @@ -31,8 +31,6 @@ from litellm.constants import ( INTERNAL_CALL_ORIGIN_METADATA_KEY, MANAGED_BACKGROUND_RESPONSE_METADATA_KEY, ) -from litellm.constants import SILENT_EXPERIMENT_METADATA_KEY -from litellm.types.router import SilentExperimentMarker from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup from litellm.llms.base_llm.ocr.transformation import OCRResponse, OCRUsageInfo from litellm.litellm_core_utils.safe_json_dumps import safe_dumps @@ -180,33 +178,6 @@ def test_get_logging_payload_includes_kubernetes_pod_routing( assert _get_spend_logs_metadata(None)["kubernetes_pod_routing"] is None -@pytest.mark.parametrize( - ("silent_experiment_metadata", "expected"), - [ - ({SILENT_EXPERIMENT_METADATA_KEY: SilentExperimentMarker.MIRRORED}, True), - ({SILENT_EXPERIMENT_METADATA_KEY: SilentExperimentMarker.MIRRORED.value}, None), - ({SILENT_EXPERIMENT_METADATA_KEY: True}, None), - ({"is_silent_experiment": True}, None), - ({}, None), - ], -) -def test_get_logging_payload_records_whether_the_call_was_a_silent_experiment( - silent_experiment_metadata: dict[str, object], expected: bool | None -) -> None: - now: Final = datetime.datetime(2026, 1, 1, tzinfo=timezone.utc) - payload: Final = get_logging_payload( - kwargs={ - "model": "gpt-4o-mini", - "litellm_params": {"metadata": {"user_api_key": "test-key", **silent_experiment_metadata}}, - }, - response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]), - start_time=now, - end_time=now, - ) - - assert json.loads(payload["metadata"]).get("is_silent_experiment") is expected - - @pytest.mark.parametrize("store_prompts,redact", [(True, False), (False, False), (True, True)]) def test_classifier_audit_spend_storage_obeys_privacy_and_truncation(monkeypatch, store_prompts, redact): from litellm.proxy import proxy_server diff --git a/tests/unit/test_router_silent_experiment.py b/tests/unit/test_router_silent_experiment.py index 7b3435a73c2..8939624bda5 100644 --- a/tests/unit/test_router_silent_experiment.py +++ b/tests/unit/test_router_silent_experiment.py @@ -287,22 +287,50 @@ async def test_standard_logging_payload_marks_only_the_mirrored_calls_as_silent_ assert "is_silent_experiment" not in metadata_by_group["primary-model"] -def _spend_row_is_silent_experiment(call: dict[str, object]) -> object: +_SPEND_ROW_TIME: Final = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc) + + +def _spend_row_is_silent_experiment(call: Mapping[str, object]) -> object: from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload - now: Final = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc) row: Final = get_logging_payload( - kwargs=call, + kwargs=dict(call), response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]), - start_time=now, - end_time=now, + start_time=_SPEND_ROW_TIME, + end_time=_SPEND_ROW_TIME, ) return json.loads(row["metadata"]).get("is_silent_experiment") +def _through_the_spend_collector(call: Mapping[str, object]) -> Mapping[str, object]: + from litellm.proxy.spend_tracking.spend_event import ( + SpendEvent, + build_spend_event, + decode_spend_event, + spend_event_callback_args, + ) + + line: Final = build_spend_event( + call, + litellm.ModelResponse(id="chatcmpl-test", choices=[]), + _SPEND_ROW_TIME, + _SPEND_ROW_TIME, + store_bodies=False, + ) + assert isinstance(line, bytes) + event: Final = decode_spend_event(line) + assert isinstance(event, SpendEvent) + return spend_event_callback_args(event).kwargs + + @pytest.mark.asyncio +@pytest.mark.parametrize( + "spend_kwargs", + [pytest.param(lambda call: call, id="in_process"), pytest.param(_through_the_spend_collector, id="collector")], +) async def test_spend_rows_mark_only_the_mirrored_calls_when_the_caller_sends_litellm_metadata( recording_logger: _RecordingLogger, + spend_kwargs: Callable[[Mapping[str, object]], Mapping[str, object]], ) -> None: router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"])) await router.acompletion( @@ -314,12 +342,35 @@ async def test_spend_rows_mark_only_the_mirrored_calls_when_the_caller_sends_lit await _wait_for_all_successes(recording_logger, expected=3) silent_by_group: Final = { - call["standard_logging_object"]["model_group"]: _spend_row_is_silent_experiment(call) + call["standard_logging_object"]["model_group"]: _spend_row_is_silent_experiment(spend_kwargs(call)) for call in recording_logger.success_kwargs } assert silent_by_group == {"primary-model": None, "shadow-a": True, "shadow-b": True} +@pytest.mark.asyncio +async def test_logging_callbacks_never_receive_the_router_mirror_marker(recording_logger: _RecordingLogger) -> None: + router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"])) + await router.acompletion( + model="primary-model", + messages=[{"role": "user", "content": "hi"}], + mock_response="pong", + litellm_metadata={"user_api_key": "test-key"}, + ) + await _wait_for_all_successes(recording_logger, expected=3) + + request_metadata_keys: Final = { + call["standard_logging_object"]["model_group"]: { + *call["litellm_params"]["metadata"], + *call["litellm_params"]["litellm_metadata"], + } + for call in recording_logger.success_kwargs + } + assert sorted(request_metadata_keys) == ["primary-model", "shadow-a", "shadow-b"] + assert all(SILENT_EXPERIMENT_METADATA_KEY not in keys for keys in request_metadata_keys.values()) + assert all("is_silent_experiment" in request_metadata_keys[group] for group in ("shadow-a", "shadow-b")) + + @pytest.mark.asyncio @pytest.mark.parametrize( ("bucket", "planted"),