diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index cf5c163fc22..39dcc874c58 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -155,7 +155,7 @@ from litellm.types.llms.openai import ( from litellm.types.mcp import MCPPostCallResponseObject from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.rerank import RerankResponse -from litellm.types.router import RouterRateLimitError, SilentExperimentMarker +from litellm.types.router import RouterRateLimitError, SilentExperimentFlag, SilentExperimentMarker from litellm.types.utils import ( DEPLOYMENT_SCOPED_PRICING_FIELDS, CachingDetails, @@ -6437,10 +6437,12 @@ class StandardLoggingPayloadSetup: proxy_stamped_used_client_oauth_token(metadata, litellm_params), custom_llm_provider, ), - is_silent_experiment=( - metadata is not None + **( + SilentExperimentFlag(is_silent_experiment=True) + if metadata is not None and SILENT_EXPERIMENT_METADATA_KEY in metadata and metadata[SILENT_EXPERIMENT_METADATA_KEY] is SilentExperimentMarker.MIRRORED + else SilentExperimentFlag() ), ) if isinstance(metadata, dict): diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 3bddb89b04a..5107a662717 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -75,7 +75,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( SETTLES_BACKGROUND_RESPONSE_KWARG, BackgroundResponseSettlement, ) -from litellm.types.router import DeploymentTypedDict, LiteLLM_Params, SilentExperimentMarker +from litellm.types.router import DeploymentTypedDict, LiteLLM_Params, SilentExperimentFlag, SilentExperimentMarker from litellm.types.utils import ( BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, PROMPT_CARRYING_GUARDRAIL_FIELDS, @@ -253,7 +253,6 @@ def _get_spend_logs_metadata( guardrail_information=None, internal_call_origin=None, litellm_roi_estimator=False, - is_silent_experiment=False, billed_retry_attempt=False, eval_information=None, cold_storage_object_key=cold_storage_object_key, @@ -291,8 +290,12 @@ def _get_spend_logs_metadata( used_client_oauth_token=used_client_oauth_token, kubernetes_pod_routing=kubernetes_pod_routing, litellm_roi_estimator=metadata.get("litellm_roi_estimator") is True, - is_silent_experiment=SILENT_EXPERIMENT_METADATA_KEY in metadata - and metadata[SILENT_EXPERIMENT_METADATA_KEY] is SilentExperimentMarker.MIRRORED, + **( + SilentExperimentFlag(is_silent_experiment=True) + if SILENT_EXPERIMENT_METADATA_KEY in metadata + and metadata[SILENT_EXPERIMENT_METADATA_KEY] is SilentExperimentMarker.MIRRORED + else SilentExperimentFlag() + ), billed_retry_attempt=billed_retry_attempt, user_api_key_user_email=_stored_user_email(metadata), ) diff --git a/litellm/types/router.py b/litellm/types/router.py index 2a4bd095255..091ca6ac200 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -70,6 +70,10 @@ class SilentExperimentMarker(str, enum.Enum): MIRRORED = "mirrored" +class SilentExperimentFlag(TypedDict, total=False): + is_silent_experiment: ReadOnly[bool] + + class FallbackSkipReason(str, enum.Enum): NOT_AUTHORIZED = "not_authorized" OVER_BUDGET = "over_budget" diff --git a/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json b/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json index 6b172c687a3..6e5461c3b60 100644 --- a/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json +++ b/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json @@ -11,7 +11,7 @@ "user": "", "team_id": "", "organization_id": "", - "metadata": "{\"actor_agent_id\": null, \"target_agent_id\": null, \"billing_agent_id\": null, \"agent_execution_mode\": null, \"verified_human_user_id\": null, \"used_client_oauth_token\": null, \"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"litellm_roi_estimator\": false, \"is_silent_experiment\": false, \"billed_retry_attempt\": false, \"router_metadata\": null, \"autorouter_savings_estimate\": null, \"autorouter_baseline_observation\": null, \"azure_spillover\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_user_email\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"user_agent\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null, \"kubernetes_pod_routing\": null}", + "metadata": "{\"actor_agent_id\": null, \"target_agent_id\": null, \"billing_agent_id\": null, \"agent_execution_mode\": null, \"verified_human_user_id\": null, \"used_client_oauth_token\": null, \"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"litellm_roi_estimator\": false, \"billed_retry_attempt\": false, \"router_metadata\": null, \"autorouter_savings_estimate\": null, \"autorouter_baseline_observation\": null, \"azure_spillover\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_user_email\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"user_agent\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null, \"kubernetes_pod_routing\": null}", "cache_key": "Cache OFF", "spend": 0.00022500000000000002, "total_tokens": 30, diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion.json index c2ae9409734..906b1a42a8f 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_bedrock_call.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_bedrock_call.json index 067c4683607..6f359380245 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_bedrock_call.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_bedrock_call.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "langfuse.observation.model.parameters": { "aws_region": "us-east-1" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_complex_metadata.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_complex_metadata.json index c2ae9409734..906b1a42a8f 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_complex_metadata.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_complex_metadata.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_langfuse_metadata.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_langfuse_metadata.json index 440aa0827d5..5ed49cde972 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_langfuse_metadata.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_langfuse_metadata.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_no_choices.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_no_choices.json index 6a2096e3e16..b5a0737cf39 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_no_choices.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_no_choices.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_router.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_router.json index af495a567f8..749796e0d04 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_router.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_router.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "stream": false, diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_tags.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_tags.json index 013bcdf64b2..39e26f5957d 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_tags.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_tags.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_tags_stream.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_tags_stream.json index bfd768c81a8..83f4e648b8d 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_tags_stream.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_tags_stream.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}", diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_vertex_call.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_vertex_call.json index a41e6b404c5..d7d292390a0 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_vertex_call.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_vertex_call.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gemini-3-flash-preview", "langfuse.observation.model.parameters": {}, "langfuse.observation.type": "generation", diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/complex_metadata.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/complex_metadata.json index c2ae9409734..906b1a42a8f 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/complex_metadata.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/complex_metadata.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/complex_metadata_2.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/complex_metadata_2.json index c2ae9409734..906b1a42a8f 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/complex_metadata_2.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/complex_metadata_2.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/empty_metadata.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/empty_metadata.json index c2ae9409734..906b1a42a8f 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/empty_metadata.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/empty_metadata.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/metadata_with_function.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/metadata_with_function.json index c2ae9409734..906b1a42a8f 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/metadata_with_function.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/metadata_with_function.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/metadata_with_lock.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/metadata_with_lock.json index c2ae9409734..906b1a42a8f 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/metadata_with_lock.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/metadata_with_lock.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/nested_metadata.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/nested_metadata.json index c2ae9409734..906b1a42a8f 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/nested_metadata.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/nested_metadata.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata.json index c2ae9409734..906b1a42a8f 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata2.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata2.json index c2ae9409734..906b1a42a8f 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata2.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata2.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata3.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata3.json index c2ae9409734..906b1a42a8f 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata3.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/simple_metadata3.json @@ -14,7 +14,6 @@ ] }, "langfuse.observation.level": "DEFAULT", - "langfuse.observation.metadata.is_silent_experiment": false, "langfuse.observation.model.name": "gpt-3.5-turbo", "langfuse.observation.model.parameters": { "extra_body": "{}" diff --git a/tests/unit/integrations/test_langfuse.py b/tests/unit/integrations/test_langfuse.py index 462f82eeb52..2d1e49e32df 100644 --- a/tests/unit/integrations/test_langfuse.py +++ b/tests/unit/integrations/test_langfuse.py @@ -838,25 +838,20 @@ class TestLangfuseUsageDetails(unittest.TestCase): metadata = self._canary_request_metadata() self.use_real_langfuse_client() - with patch( - "litellm.integrations.langfuse.langfuse._add_prompt_to_generation_params", - side_effect=lambda generation_params, **kw: generation_params, - create=True, - ): - self.logger._log_langfuse_v2( - user_id="user-1", - metadata=metadata, - litellm_params={"metadata": metadata}, - output=None, - start_time=datetime.datetime(2024, 1, 1, 12, 0, 0), - end_time=datetime.datetime(2024, 1, 1, 12, 0, 1), - kwargs=kwargs, - optional_params={}, - input=None, - response_obj=None, - level="INFO", - litellm_call_id="canary-call-id", - ) + self.logger._log_langfuse_v2( + user_id="user-1", + metadata=metadata, + litellm_params={"metadata": metadata}, + output=None, + start_time=datetime.datetime(2024, 1, 1, 12, 0, 0), + end_time=datetime.datetime(2024, 1, 1, 12, 0, 1), + kwargs=kwargs, + optional_params={}, + input=None, + response_obj=None, + level="INFO", + litellm_call_id="canary-call-id", + ) assert self.exported_generation_metadata()["is_silent_experiment"] is True diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index b7124236fbf..58d62763a84 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -10373,7 +10373,7 @@ def test_init_kwargs_client_metadata_cannot_mark_a_pass_through_call_as_a_silent start_time=started, end_time=started, ) - assert json.loads(row["metadata"])["is_silent_experiment"] is False + assert "is_silent_experiment" not in json.loads(row["metadata"]) assert result["litellm_params"]["metadata"]["client_key"] == "kept" @pytest.mark.asyncio diff --git a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py index 695452fce40..23bbe521670 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py @@ -3725,7 +3725,7 @@ class TestSpendLogsPayload: "model": "gpt-4o", "user": "", "team_id": "", - "metadata": '{"user_api_key_user_email": null, "actor_agent_id": null, "target_agent_id": null, "billing_agent_id": null, "agent_execution_mode": null, "verified_human_user_id": null, "applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "azure_spillover": null, "used_client_oauth_token": null, "litellm_roi_estimator": false, "is_silent_experiment": false, "billed_retry_attempt": false, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}', + "metadata": '{"user_api_key_user_email": null, "actor_agent_id": null, "target_agent_id": null, "billing_agent_id": null, "agent_execution_mode": null, "verified_human_user_id": null, "applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "azure_spillover": null, "used_client_oauth_token": null, "litellm_roi_estimator": false, "billed_retry_attempt": false, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}', "cache_key": "Cache OFF", "spend": 0.00022500000000000002, "total_tokens": 30, diff --git a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py index ec3ba361b3f..caca6e03790 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py @@ -169,14 +169,14 @@ def test_get_logging_payload_includes_kubernetes_pod_routing( ("silent_experiment_metadata", "expected"), [ ({SILENT_EXPERIMENT_METADATA_KEY: SilentExperimentMarker.MIRRORED}, True), - ({SILENT_EXPERIMENT_METADATA_KEY: SilentExperimentMarker.MIRRORED.value}, False), - ({SILENT_EXPERIMENT_METADATA_KEY: True}, False), - ({"is_silent_experiment": True}, False), - ({}, False), + ({SILENT_EXPERIMENT_METADATA_KEY: SilentExperimentMarker.MIRRORED.value}, None), + ({SILENT_EXPERIMENT_METADATA_KEY: True}, None), + ({"is_silent_experiment": True}, None), + ({}, None), ], ) def test_get_logging_payload_records_whether_the_call_was_a_silent_experiment( - silent_experiment_metadata: dict[str, object], expected: bool + silent_experiment_metadata: dict[str, object], expected: bool | None ) -> None: now: Final = datetime.datetime(2026, 1, 1, tzinfo=timezone.utc) payload: Final = get_logging_payload( @@ -189,7 +189,7 @@ def test_get_logging_payload_records_whether_the_call_was_a_silent_experiment( end_time=now, ) - assert json.loads(payload["metadata"])["is_silent_experiment"] is expected + assert json.loads(payload["metadata"]).get("is_silent_experiment") is expected @pytest.mark.parametrize("store_prompts,redact", [(True, False), (False, False), (True, True)]) @@ -6642,7 +6642,7 @@ def test_spend_logs_payload(model_id: str | None): ) assert len(payload["request_id"]) > 0 - expected_metadata_keys: Final = SpendLogsMetadata.__annotations__.keys() + expected_metadata_keys: Final = SpendLogsMetadata.__annotations__.keys() - {"is_silent_experiment"} assert "metadata" in payload assert isinstance(payload["metadata"], str) diff --git a/tests/unit/test_router_silent_experiment.py b/tests/unit/test_router_silent_experiment.py index c455153b3cf..7b3435a73c2 100644 --- a/tests/unit/test_router_silent_experiment.py +++ b/tests/unit/test_router_silent_experiment.py @@ -284,7 +284,7 @@ async def test_standard_logging_payload_marks_only_the_mirrored_calls_as_silent_ metadata_by_group: Final = {payload["model_group"]: payload["metadata"] for payload in payloads} assert metadata_by_group["shadow-a"]["is_silent_experiment"] is True assert metadata_by_group["shadow-b"]["is_silent_experiment"] is True - assert metadata_by_group["primary-model"]["is_silent_experiment"] is False + assert "is_silent_experiment" not in metadata_by_group["primary-model"] def _spend_row_is_silent_experiment(call: dict[str, object]) -> object: @@ -297,7 +297,7 @@ def _spend_row_is_silent_experiment(call: dict[str, object]) -> object: start_time=now, end_time=now, ) - return json.loads(row["metadata"])["is_silent_experiment"] + return json.loads(row["metadata"]).get("is_silent_experiment") @pytest.mark.asyncio @@ -317,7 +317,7 @@ async def test_spend_rows_mark_only_the_mirrored_calls_when_the_caller_sends_lit call["standard_logging_object"]["model_group"]: _spend_row_is_silent_experiment(call) for call in recording_logger.success_kwargs } - assert silent_by_group == {"primary-model": False, "shadow-a": True, "shadow-b": True} + assert silent_by_group == {"primary-model": None, "shadow-a": True, "shadow-b": True} @pytest.mark.asyncio @@ -343,8 +343,8 @@ async def test_a_caller_sent_silent_experiment_flag_does_not_mark_a_real_call_as await _wait_for_all_successes(recording_logger, expected=1) call: Final = recording_logger.success_kwargs[0] - assert call["standard_logging_object"]["metadata"]["is_silent_experiment"] is False - assert _spend_row_is_silent_experiment(call) is False + assert "is_silent_experiment" not in call["standard_logging_object"]["metadata"] + assert _spend_row_is_silent_experiment(call) is None def test_multiple_shadow_targets_fan_out_sync(recording_logger):