diff --git a/litellm/constants.py b/litellm/constants.py index 2901b7d8948..54f5feb5c4c 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1660,6 +1660,7 @@ ROUTING_REQUEST_TAGS_METADATA_KEY: Final = "_routing_request_tags" BAGGAGE_MAPPED_TAGS_METADATA_KEY: Final = "_baggage_mapped_tags" VECTOR_STORE_CALLER_METADATA_KWARG: Final = "litellm_vector_store_caller_metadata" ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted_tokens" +SILENT_EXPERIMENT_METADATA_KEY: Final = "_litellm_silent_experiment" INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin" BILLED_RETRY_ATTEMPT_METADATA_KEY: Final = "billed_retry_attempt" MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response" diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index b8c6e91d0d7..bd68ffe8cd6 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -833,6 +833,12 @@ class LangFuseLogger: ("vertex_location", vertex_location, bool(vertex_location)), ("aws_region_name", aws_region_name, bool(aws_region_name)), ("cache_hit", kwargs.get("cache_hit") or False, "cache_hit" in kwargs), + ( + "is_silent_experiment", + "is_silent_experiment" in allowlisted_metadata + and allowlisted_metadata["is_silent_experiment"] is True, + "is_silent_experiment" in allowlisted_metadata, + ), ) enrichments: Final[Mapping[str, object]] = { key: value for key, value, include in candidate_enrichments if include diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 9e0461e951d..dfa9591af88 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -47,6 +47,7 @@ from litellm.constants import ( LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL, PROVIDER_REQUEST_ID_HEADERS, REDACTED_BY_LITELLM, + SILENT_EXPERIMENT_METADATA_KEY, ) from litellm.cost_calculator import ( RealtimeAPITokenUsageProcessor, @@ -157,7 +158,7 @@ from litellm.types.llms.openai import ( from litellm.types.mcp import MCPPostCallResponseObject from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.rerank import RerankResponse -from litellm.types.router import RouterRateLimitError +from litellm.types.router import RouterRateLimitError, SilentExperimentFlag, SilentExperimentMarker from litellm.types.utils import ( DEPLOYMENT_SCOPED_PRICING_FIELDS, CachingDetails, @@ -309,8 +310,9 @@ else: _in_memory_loggers: Final[list[CustomLogger]] = [] _STANDARD_LOGGING_METADATA_RESOLVED_KEYS: Final[frozenset[str]] = frozenset( - ("used_client_oauth_token", "usage_object", KUBERNETES_POD_ROUTING_KEY) + ("used_client_oauth_token", "usage_object", KUBERNETES_POD_ROUTING_KEY, "is_silent_experiment") ) +_SILENT_EXPERIMENT_BUCKETS: Final = ("metadata", "litellm_metadata") _STANDARD_LOGGING_METADATA_KEYS: Final[frozenset[str]] = ( frozenset(StandardLoggingMetadata.__annotations__.keys()) - _STANDARD_LOGGING_METADATA_RESOLVED_KEYS ) @@ -803,6 +805,7 @@ class Logging(LiteLLMLoggingBaseClass): litellm_params = scrub_sensitive_keys_in_metadata(litellm_params) self.litellm_params = litellm_params + self.is_silent_experiment = False # Initialize cost breakdown field self.cost_breakdown: CostBreakdown | None = None @@ -1118,6 +1121,7 @@ class Logging(LiteLLMLoggingBaseClass): if model is not None: self.model = model self.user = user + self.is_silent_experiment = self.is_silent_experiment or is_silent_experiment_call(litellm_params) self.litellm_params = { **self.litellm_params, **scrub_sensitive_keys_in_metadata(litellm_params), @@ -6523,6 +6527,7 @@ class StandardLoggingPayloadSetup: start_time: dt_object | None = None, response_id: str | None = None, custom_llm_provider: str | None = None, + is_silent_experiment: bool = False, ) -> StandardLoggingMetadata: """ Clean and filter the metadata dictionary to include only the specified keys in StandardLoggingMetadata. @@ -6596,6 +6601,7 @@ class StandardLoggingPayloadSetup: proxy_stamped_used_client_oauth_token(metadata, litellm_params), custom_llm_provider, ), + **(SilentExperimentFlag(is_silent_experiment=True) if is_silent_experiment else SilentExperimentFlag()), ) if isinstance(metadata, dict): for key in metadata.keys() & _STANDARD_LOGGING_METADATA_KEYS: @@ -7398,6 +7404,7 @@ def get_standard_logging_object_payload( start_time=start_time, response_id=id, custom_llm_provider=selected_provider if isinstance(selected_provider, str) else None, + is_silent_experiment=logging_obj.is_silent_experiment is True, ) _request_body: Final = proxy_server_request.get("body", {}) end_user_id: Final = clean_metadata["user_api_key_end_user_id"] or _request_body.get( @@ -7692,7 +7699,9 @@ def get_standard_logging_metadata( return clean_metadata -def scrub_sensitive_keys_in_metadata(litellm_params: dict | None): +def scrub_sensitive_keys_in_metadata( + litellm_params: dict[str, Any] | None, # mutable-ok: the langfuse masking function is moved onto these params +) -> dict[str, Any]: # mutable-ok: loggers write into the returned params and their metadata if litellm_params is None: litellm_params = {} @@ -7707,7 +7716,24 @@ def scrub_sensitive_keys_in_metadata(litellm_params: dict | None): litellm_params["_langfuse_masking_function"] = masking_fn litellm_params["metadata"] = metadata - return litellm_params + if not is_silent_experiment_call(litellm_params): + return litellm_params + return { + name: ( + {key: value for key, value in bucket.items() if key != SILENT_EXPERIMENT_METADATA_KEY} + if name in _SILENT_EXPERIMENT_BUCKETS and _carries_silent_experiment_marker(bucket) + else bucket + ) + for name, bucket in litellm_params.items() + } + + +def _carries_silent_experiment_marker(bucket: object) -> bool: + return isinstance(bucket, dict) and bucket.get(SILENT_EXPERIMENT_METADATA_KEY) is SilentExperimentMarker.MIRRORED + + +def is_silent_experiment_call(litellm_params: Mapping[str, object]) -> bool: + return any(_carries_silent_experiment_marker(litellm_params.get(name)) for name in _SILENT_EXPERIMENT_BUCKETS) # integration helper function diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 49d23d82aa9..1f295aa2f47 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3744,6 +3744,7 @@ class SpendLogsMetadata(TypedDict): internal_call_origin: InternalCallOrigin | None billed_retry_attempt: ReadOnly[NotRequired[bool | None]] litellm_roi_estimator: ReadOnly[NotRequired[bool | None]] + is_silent_experiment: ReadOnly[NotRequired[bool | None]] guardrail_information: list[StandardLoggingGuardrailInformation] | None eval_information: Any | None status: StandardLoggingPayloadStatus diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 94b079163e9..833ee71ae9c 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -80,7 +80,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( SETTLES_BACKGROUND_RESPONSE_KWARG, BackgroundResponseSettlement, ) -from litellm.types.router import DeploymentTypedDict, LiteLLM_Params +from litellm.types.router import DeploymentTypedDict, LiteLLM_Params, SilentExperimentFlag from litellm.types.utils import ( BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, PROMPT_CARRYING_GUARDRAIL_FIELDS, @@ -194,6 +194,7 @@ _STAMPED_METADATA_KEYS: Final = frozenset( "used_client_oauth_token", KUBERNETES_POD_ROUTING_KEY, "litellm_roi_estimator", + "is_silent_experiment", BILLED_RETRY_ATTEMPT_METADATA_KEY, "user_api_key_user_email", ) @@ -227,6 +228,7 @@ def _get_spend_logs_metadata( azure_spillover: AzureSpillover | None = None, used_client_oauth_token: bool | None = None, kubernetes_pod_routing: StandardLoggingKubernetesPodRouting | None = None, + is_silent_experiment: bool = False, ) -> SpendLogsMetadata: if metadata is None: return SpendLogsMetadata( @@ -294,6 +296,7 @@ def _get_spend_logs_metadata( used_client_oauth_token=used_client_oauth_token, kubernetes_pod_routing=kubernetes_pod_routing, litellm_roi_estimator=metadata.get("litellm_roi_estimator") is True, + **(SilentExperimentFlag(is_silent_experiment=True) if is_silent_experiment else SilentExperimentFlag()), billed_retry_attempt=billed_retry_attempt, user_api_key_user_email=_stored_user_email(metadata), ) @@ -738,6 +741,11 @@ def get_logging_payload( # clean up litellm metadata clean_metadata = _get_spend_logs_metadata( persisted_metadata, + is_silent_experiment=( + standard_logging_payload is not None + and "is_silent_experiment" in standard_logging_payload["metadata"] + and standard_logging_payload["metadata"]["is_silent_experiment"] + ), applied_guardrails=( standard_logging_payload["metadata"].get("applied_guardrails", None) if standard_logging_payload is not None diff --git a/litellm/router.py b/litellm/router.py index 8d95e3e91d4..69cfaf66dce 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -79,6 +79,7 @@ from litellm.constants import ( ROUTING_REQUEST_TAGS_METADATA_KEY, RUNTIME_UPDATABLE_ROUTER_SETTINGS, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, + SILENT_EXPERIMENT_METADATA_KEY, ) from litellm.integrations.custom_guardrail import CustomGuardrail, is_guardrail_intervention from litellm.integrations.custom_logger import CustomLogger @@ -413,6 +414,7 @@ from litellm.types.router import ( RoutingPlugin, RoutingStrategy, SearchToolTypedDict, + SilentExperimentMarker, TaggedPreRoutingStrategy, holds_secret_pointer, ) @@ -3249,6 +3251,12 @@ class Router: silent_kwargs["metadata"].pop("litellm_parent_otel_span", None) silent_kwargs["metadata"]["is_silent_experiment"] = True + silent_kwargs["metadata"][SILENT_EXPERIMENT_METADATA_KEY] = SilentExperimentMarker.MIRRORED + if "litellm_metadata" in silent_kwargs and isinstance(silent_kwargs["litellm_metadata"], dict): + silent_kwargs["litellm_metadata"] = { + **silent_kwargs["litellm_metadata"], + SILENT_EXPERIMENT_METADATA_KEY: SilentExperimentMarker.MIRRORED, + } # Pop logging objects and call IDs to ensure a fresh logging context # This prevents collisions in the Proxy's database (spend_logs) diff --git a/litellm/types/router.py b/litellm/types/router.py index ba9b82e2a57..b162e4529ce 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -66,6 +66,14 @@ from .utils import ( AllowedModelRegion: TypeAlias = Literal["eu", "us"] +class SilentExperimentMarker(str, enum.Enum): + MIRRORED = "mirrored" + + +class SilentExperimentFlag(TypedDict, total=False): + is_silent_experiment: ReadOnly[bool] + + class FallbackSkipReason(str, enum.Enum): NOT_AUTHORIZED = "not_authorized" OVER_BUDGET = "over_budget" diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 7e8645fd0dd..672014bdcd9 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3372,6 +3372,7 @@ class StandardLoggingMetadata(StandardLoggingUserAPIKeyMetadata): team_alias: str | None team_id: str | None used_client_oauth_token: ReadOnly[bool | None] + is_silent_experiment: ReadOnly[NotRequired[bool]] class AzureSpillover(TypedDict): diff --git a/tests/unit/integrations/test_langfuse.py b/tests/unit/integrations/test_langfuse.py index 00d85b64566..2d1e49e32df 100644 --- a/tests/unit/integrations/test_langfuse.py +++ b/tests/unit/integrations/test_langfuse.py @@ -830,6 +830,31 @@ class TestLangfuseUsageDetails(unittest.TestCase): assert generation_metadata["litellm_response_cost"] == 0.25 assert generation_metadata["api_base"] == "https://real-api-base" + def test_caller_sent_is_silent_experiment_cannot_hide_the_mirrored_call_flag(self): + payload = self._build_standard_logging_payload(trace_id="canary-trace-id") + payload["metadata"]["is_silent_experiment"] = True + payload["metadata"]["requester_metadata"] = {"is_silent_experiment": False} + kwargs = self._build_langfuse_kwargs(payload) + metadata = self._canary_request_metadata() + self.use_real_langfuse_client() + + self.logger._log_langfuse_v2( + user_id="user-1", + metadata=metadata, + litellm_params={"metadata": metadata}, + output=None, + start_time=datetime.datetime(2024, 1, 1, 12, 0, 0), + end_time=datetime.datetime(2024, 1, 1, 12, 0, 1), + kwargs=kwargs, + optional_params={}, + input=None, + response_obj=None, + level="INFO", + litellm_call_id="canary-call-id", + ) + + assert self.exported_generation_metadata()["is_silent_experiment"] is True + def test_generation_metadata_carries_the_call_id_and_response_id(self): """ v2's generation id was ``time-_``, so a generation could diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 561ac9b2b33..1e669da9e97 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -11857,7 +11857,7 @@ def test_get_additional_headers(): def all_fields_present(standard_logging_metadata: StandardLoggingMetadata): - for field in StandardLoggingMetadata.__annotations__.keys(): + for field in StandardLoggingMetadata.__required_keys__: assert field in standard_logging_metadata diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index db0a74d360a..504e422000d 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -30,7 +30,7 @@ from starlette.datastructures import UploadFile as StarletteUploadFile import litellm from litellm._logging import verbose_proxy_logger -from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS +from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS, SILENT_EXPERIMENT_METADATA_KEY from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -63,6 +63,7 @@ from litellm.proxy.pass_through_endpoints.success_handler import ( from litellm.proxy.route_llm_request import ProxyModelNotFoundError from litellm.types import utils as types_utils from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.router import SilentExperimentMarker from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, @@ -10339,6 +10340,43 @@ def test_init_kwargs_no_authenticated_identity_field_is_client_settable(mock_req survived: Final = {key: metadata.get(key) for key in expected if metadata.get(key) != expected[key]} assert survived == {}, f"client-supplied values survived for: {sorted(survived)}" + +@pytest.mark.parametrize("planted", [True, SilentExperimentMarker.MIRRORED.value]) +def test_init_kwargs_client_metadata_cannot_mark_a_pass_through_call_as_a_silent_experiment( + planted: object, mock_user_api_key_dict: UserAPIKeyAuth +) -> None: + from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload + + started: Final = datetime(2026, 1, 15, 12, 0, 0) + result: Final = HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint( + request=Request({"type": "http", "method": "POST", "path": "/test", "headers": [], "query_string": b""}), + user_api_key_dict=mock_user_api_key_dict, + passthrough_logging_payload=PassthroughStandardLoggingPayload(url="https://test.com", request_body={}), + litellm_call_id="test-call-id", + logging_obj=LiteLLMLoggingObj( + model="test-model", + messages=[], + stream=False, + call_type="test-call-type", + start_time=started, + litellm_call_id="test-call-id", + function_id="test-function-id", + ), + _parsed_body={ + "litellm_metadata": {SILENT_EXPERIMENT_METADATA_KEY: planted, "client_key": "kept"}, + "metadata": {SILENT_EXPERIMENT_METADATA_KEY: planted}, + }, + ) + + row: Final = get_logging_payload( + kwargs={"model": "test-model", "litellm_params": result["litellm_params"]}, + response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]), + start_time=started, + end_time=started, + ) + assert "is_silent_experiment" not in json.loads(row["metadata"]) + assert result["litellm_params"]["metadata"]["client_key"] == "kept" + @pytest.mark.asyncio async def test_direct_passthrough_keeps_mapped_baggage_tags_outside_upstream_body(): from litellm.proxy.common_utils.http_parsing_utils import read_request_body diff --git a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py index 78f5e1ace0c..190ffb18ca9 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py @@ -6830,7 +6830,7 @@ def test_spend_logs_payload(model_id: str | None): ) assert len(payload["request_id"]) > 0 - expected_metadata_keys: Final = SpendLogsMetadata.__annotations__.keys() + expected_metadata_keys: Final = SpendLogsMetadata.__annotations__.keys() - {"is_silent_experiment"} assert "metadata" in payload assert isinstance(payload["metadata"], str) diff --git a/tests/unit/test_router_silent_experiment.py b/tests/unit/test_router_silent_experiment.py index fd648ceb5d2..8939624bda5 100644 --- a/tests/unit/test_router_silent_experiment.py +++ b/tests/unit/test_router_silent_experiment.py @@ -1,4 +1,5 @@ import asyncio +import datetime import json import time from collections.abc import Callable, Mapping @@ -11,11 +12,13 @@ import pytest import respx import litellm +from litellm.constants import SILENT_EXPERIMENT_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.router import Router from litellm.router import _silent_experiment_kwargs_snapshot from litellm.router import _silent_experiment_targets +from litellm.types.router import SilentExperimentMarker class _RecordingLogger(CustomLogger): @@ -263,6 +266,138 @@ async def test_multiple_shadow_targets_fan_out_async(recording_logger): assert primary_successes[0]["litellm_params"]["metadata"]["model_group"] == "primary-model" +@pytest.mark.asyncio +async def test_standard_logging_payload_marks_only_the_mirrored_calls_as_silent_experiments( + recording_logger: _RecordingLogger, +) -> None: + router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"])) + response: Final = await router.acompletion( + model="primary-model", + messages=[{"role": "user", "content": "hi"}], + mock_response="pong", + ) + assert response.choices[0].message.content == "pong" + await _wait_for_all_successes(recording_logger, expected=3) + + payloads: Final = [call["standard_logging_object"] for call in recording_logger.success_kwargs] + assert sorted(payload["model_group"] for payload in payloads) == ["primary-model", "shadow-a", "shadow-b"] + metadata_by_group: Final = {payload["model_group"]: payload["metadata"] for payload in payloads} + assert metadata_by_group["shadow-a"]["is_silent_experiment"] is True + assert metadata_by_group["shadow-b"]["is_silent_experiment"] is True + assert "is_silent_experiment" not in metadata_by_group["primary-model"] + + +_SPEND_ROW_TIME: Final = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc) + + +def _spend_row_is_silent_experiment(call: Mapping[str, object]) -> object: + from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload + + row: Final = get_logging_payload( + kwargs=dict(call), + response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]), + start_time=_SPEND_ROW_TIME, + end_time=_SPEND_ROW_TIME, + ) + return json.loads(row["metadata"]).get("is_silent_experiment") + + +def _through_the_spend_collector(call: Mapping[str, object]) -> Mapping[str, object]: + from litellm.proxy.spend_tracking.spend_event import ( + SpendEvent, + build_spend_event, + decode_spend_event, + spend_event_callback_args, + ) + + line: Final = build_spend_event( + call, + litellm.ModelResponse(id="chatcmpl-test", choices=[]), + _SPEND_ROW_TIME, + _SPEND_ROW_TIME, + store_bodies=False, + ) + assert isinstance(line, bytes) + event: Final = decode_spend_event(line) + assert isinstance(event, SpendEvent) + return spend_event_callback_args(event).kwargs + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "spend_kwargs", + [pytest.param(lambda call: call, id="in_process"), pytest.param(_through_the_spend_collector, id="collector")], +) +async def test_spend_rows_mark_only_the_mirrored_calls_when_the_caller_sends_litellm_metadata( + recording_logger: _RecordingLogger, + spend_kwargs: Callable[[Mapping[str, object]], Mapping[str, object]], +) -> None: + router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"])) + await router.acompletion( + model="primary-model", + messages=[{"role": "user", "content": "hi"}], + mock_response="pong", + litellm_metadata={"user_api_key": "test-key"}, + ) + await _wait_for_all_successes(recording_logger, expected=3) + + silent_by_group: Final = { + call["standard_logging_object"]["model_group"]: _spend_row_is_silent_experiment(spend_kwargs(call)) + for call in recording_logger.success_kwargs + } + assert silent_by_group == {"primary-model": None, "shadow-a": True, "shadow-b": True} + + +@pytest.mark.asyncio +async def test_logging_callbacks_never_receive_the_router_mirror_marker(recording_logger: _RecordingLogger) -> None: + router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"])) + await router.acompletion( + model="primary-model", + messages=[{"role": "user", "content": "hi"}], + mock_response="pong", + litellm_metadata={"user_api_key": "test-key"}, + ) + await _wait_for_all_successes(recording_logger, expected=3) + + request_metadata_keys: Final = { + call["standard_logging_object"]["model_group"]: { + *call["litellm_params"]["metadata"], + *call["litellm_params"]["litellm_metadata"], + } + for call in recording_logger.success_kwargs + } + assert sorted(request_metadata_keys) == ["primary-model", "shadow-a", "shadow-b"] + assert all(SILENT_EXPERIMENT_METADATA_KEY not in keys for keys in request_metadata_keys.values()) + assert all("is_silent_experiment" in request_metadata_keys[group] for group in ("shadow-a", "shadow-b")) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("bucket", "planted"), + [ + ("metadata", {"is_silent_experiment": True}), + ("metadata", {SILENT_EXPERIMENT_METADATA_KEY: SilentExperimentMarker.MIRRORED.value}), + ("litellm_metadata", {SILENT_EXPERIMENT_METADATA_KEY: SilentExperimentMarker.MIRRORED.value}), + ("litellm_metadata", {SILENT_EXPERIMENT_METADATA_KEY: True}), + ], +) +async def test_a_caller_sent_silent_experiment_flag_does_not_mark_a_real_call_as_mirrored( + recording_logger: _RecordingLogger, bucket: str, planted: dict[str, object] +) -> None: + router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"])) + await router.acompletion( + model="shadow-b", + messages=[{"role": "user", "content": "hi"}], + mock_response="pong", + **{bucket: planted}, + ) + await _wait_for_all_successes(recording_logger, expected=1) + + call: Final = recording_logger.success_kwargs[0] + assert "is_silent_experiment" not in call["standard_logging_object"]["metadata"] + assert _spend_row_is_silent_experiment(call) is None + + def test_multiple_shadow_targets_fan_out_sync(recording_logger): router = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"])) response = router.completion(