diff --git a/litellm/constants.py b/litellm/constants.py index 1c1dfb9aa1c..33c1bee0b25 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1653,6 +1653,7 @@ ROUTING_REQUEST_TAGS_METADATA_KEY: Final = "_routing_request_tags" BAGGAGE_MAPPED_TAGS_METADATA_KEY: Final = "_baggage_mapped_tags" VECTOR_STORE_CALLER_METADATA_KWARG: Final = "litellm_vector_store_caller_metadata" ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted_tokens" +SILENT_EXPERIMENT_METADATA_KEY: Final = "_litellm_silent_experiment" INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin" BILLED_RETRY_ATTEMPT_METADATA_KEY: Final = "billed_retry_attempt" MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response" diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 473666e5ba1..ce0dc373f66 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -47,6 +47,7 @@ from litellm.constants import ( LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL, PROVIDER_REQUEST_ID_HEADERS, REDACTED_BY_LITELLM, + SILENT_EXPERIMENT_METADATA_KEY, ) from litellm.cost_calculator import ( RealtimeAPITokenUsageProcessor, @@ -305,7 +306,7 @@ else: _in_memory_loggers: Final[list[CustomLogger]] = [] _STANDARD_LOGGING_METADATA_RESOLVED_KEYS: Final[frozenset[str]] = frozenset( - ("used_client_oauth_token", "usage_object", KUBERNETES_POD_ROUTING_KEY) + ("used_client_oauth_token", "usage_object", KUBERNETES_POD_ROUTING_KEY, "is_silent_experiment") ) _STANDARD_LOGGING_METADATA_KEYS: Final[frozenset[str]] = ( frozenset(StandardLoggingMetadata.__annotations__.keys()) - _STANDARD_LOGGING_METADATA_RESOLVED_KEYS @@ -6436,6 +6437,11 @@ class StandardLoggingPayloadSetup: proxy_stamped_used_client_oauth_token(metadata, litellm_params), custom_llm_provider, ), + is_silent_experiment=( + metadata is not None + and SILENT_EXPERIMENT_METADATA_KEY in metadata + and metadata[SILENT_EXPERIMENT_METADATA_KEY] is True + ), ) if isinstance(metadata, dict): for key in metadata.keys() & _STANDARD_LOGGING_METADATA_KEYS: diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index a6e971f002a..9124e4f50e9 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -43,6 +43,7 @@ from litellm.constants import ( SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY, SESSION_ID_OMITTED_METADATA_KEY, + SILENT_EXPERIMENT_METADATA_KEY, TYPESAFE_DANGER_ZONE_HEADER_PREFIX, VECTOR_STORE_CALLER_METADATA_KWARG, X_LITELLM_DISABLE_CALLBACKS, @@ -469,12 +470,12 @@ _CLIENT_PRICING_METADATA_FIELDS: Final = frozenset({"model_info", "standard_logg _ROUTER_RESERVED_METADATA_FIELDS: Final = frozenset( { "attempted_fallbacks", - "is_silent_experiment", "original_model_group", "previous_models", "request_retry_count", CLIENT_OUTPUT_CEILING_METADATA_KEY, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY, + SILENT_EXPERIMENT_METADATA_KEY, } ) _ALLOW_CLIENT_PRICING_OVERRIDE_METADATA_KEY: Final = "allow_client_pricing_override" diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index fe7ac30e434..21da02cd572 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -113,6 +113,7 @@ from litellm.proxy.common_utils.sse_keepalive import ( wrap_passthrough_sse_bytes_with_keepalive_pings, ) from litellm.proxy.litellm_pre_call_utils import ( # noqa: F401 # legacy module exports + _ROUTER_RESERVED_METADATA_FIELDS, # pyright: ignore[reportPrivateUsage] # pass-through applies the proxy's router-owned key strip LiteLLMProxyRequestSetup, _get_dynamic_logging_metadata, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _key_or_team_allows_client_pricing_override, # pyright: ignore[reportPrivateUsage] # reuse the proxy's pricing trust policy @@ -165,7 +166,11 @@ _PRICING_MODEL_OVERRIDE_PARAMS: Final = frozenset(("base_model",)) def _is_mergeable_client_metadata(key: str, value: object) -> bool: - return not key.startswith("user_api_key_") and (key != "model_info" or isinstance(value, Mapping)) + return ( + not key.startswith("user_api_key_") + and key not in _ROUTER_RESERVED_METADATA_FIELDS + and (key != "model_info" or isinstance(value, Mapping)) + ) # Global registry to track registered pass-through routes and prevent memory leaks diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 6183ee5cb3e..875762a7284 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -31,6 +31,7 @@ from litellm.constants import ( NON_INFERENCE_CALL_TYPES, REDACTED_BY_LITELM_STRING, SESSION_ID_OMITTED_METADATA_KEY, + SILENT_EXPERIMENT_METADATA_KEY, UNKNOWN_MODEL_SPEND_LOG_MODEL, ) from litellm.constants import ( @@ -290,7 +291,8 @@ def _get_spend_logs_metadata( used_client_oauth_token=used_client_oauth_token, kubernetes_pod_routing=kubernetes_pod_routing, litellm_roi_estimator=metadata.get("litellm_roi_estimator") is True, - is_silent_experiment="is_silent_experiment" in metadata and metadata["is_silent_experiment"] is True, + is_silent_experiment=SILENT_EXPERIMENT_METADATA_KEY in metadata + and metadata[SILENT_EXPERIMENT_METADATA_KEY] is True, billed_retry_attempt=billed_retry_attempt, user_api_key_user_email=_stored_user_email(metadata), ) diff --git a/litellm/router.py b/litellm/router.py index e1271970c22..89a1124ff7b 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -78,6 +78,7 @@ from litellm.constants import ( ROUTING_REQUEST_TAGS_METADATA_KEY, RUNTIME_UPDATABLE_ROUTER_SETTINGS, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, + SILENT_EXPERIMENT_METADATA_KEY, ) from litellm.integrations.custom_guardrail import CustomGuardrail, is_guardrail_intervention from litellm.integrations.custom_logger import CustomLogger @@ -3124,8 +3125,12 @@ class Router: silent_kwargs["metadata"].pop("litellm_parent_otel_span", None) silent_kwargs["metadata"]["is_silent_experiment"] = True + silent_kwargs["metadata"][SILENT_EXPERIMENT_METADATA_KEY] = True if "litellm_metadata" in silent_kwargs and isinstance(silent_kwargs["litellm_metadata"], dict): - silent_kwargs["litellm_metadata"] = {**silent_kwargs["litellm_metadata"], "is_silent_experiment": True} + silent_kwargs["litellm_metadata"] = { + **silent_kwargs["litellm_metadata"], + SILENT_EXPERIMENT_METADATA_KEY: True, + } # Pop logging objects and call IDs to ensure a fresh logging context # This prevents collisions in the Proxy's database (spend_logs) diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index f0baceecdaa..2ee5e17d6df 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -30,7 +30,7 @@ from starlette.datastructures import UploadFile as StarletteUploadFile import litellm from litellm._logging import verbose_proxy_logger -from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS +from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS, SILENT_EXPERIMENT_METADATA_KEY from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -10338,6 +10338,42 @@ def test_init_kwargs_no_authenticated_identity_field_is_client_settable(mock_req survived: Final = {key: metadata.get(key) for key in expected if metadata.get(key) != expected[key]} assert survived == {}, f"client-supplied values survived for: {sorted(survived)}" + +def test_init_kwargs_client_metadata_cannot_mark_a_pass_through_call_as_a_silent_experiment( + mock_request, mock_user_api_key_dict +): + from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload + + started: Final = datetime(2026, 1, 15, 12, 0, 0) + result: Final = HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint( + request=mock_request(), + user_api_key_dict=mock_user_api_key_dict, + passthrough_logging_payload=PassthroughStandardLoggingPayload(url="https://test.com", request_body={}), + litellm_call_id="test-call-id", + logging_obj=LiteLLMLoggingObj( + model="test-model", + messages=[], + stream=False, + call_type="test-call-type", + start_time=started, + litellm_call_id="test-call-id", + function_id="test-function-id", + ), + _parsed_body={ + "litellm_metadata": {SILENT_EXPERIMENT_METADATA_KEY: True, "client_key": "kept"}, + "metadata": {SILENT_EXPERIMENT_METADATA_KEY: True}, + }, + ) + + row: Final = get_logging_payload( + kwargs={"model": "test-model", "litellm_params": result["litellm_params"]}, + response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]), + start_time=started, + end_time=started, + ) + assert json.loads(row["metadata"])["is_silent_experiment"] is False + assert result["litellm_params"]["metadata"]["client_key"] == "kept" + @pytest.mark.asyncio async def test_direct_passthrough_keeps_mapped_baggage_tags_outside_upstream_body(): from litellm.proxy.common_utils.http_parsing_utils import read_request_body diff --git a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py index c9ec799a9c0..f5e95f87a52 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py @@ -16,6 +16,7 @@ import litellm.constants as litellm_constants import litellm.proxy.spend_tracking.spend_tracking_utils as spend_tracking_utils from litellm.constants import KUBERNETES_POD_ROUTING_KEY, LITELLM_TRUNCATED_PAYLOAD_FIELD, LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, LITTELM_CLI_SERVICE_ACCOUNT_NAME, LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, MAX_SPEND_LOG_MODEL_NAME_LENGTH, REDACTED_BY_LITELM_STRING, SESSION_ID_GENERATED_METADATA_KEY, SESSION_ID_OMITTED_METADATA_KEY, UNKNOWN_MODEL_SPEND_LOG_MODEL from litellm.constants import CALLER_SESSION_ID_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY, MANAGED_BACKGROUND_RESPONSE_METADATA_KEY +from litellm.constants import SILENT_EXPERIMENT_METADATA_KEY from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup from litellm.llms.base_llm.ocr.transformation import OCRResponse, OCRUsageInfo from litellm.litellm_core_utils.safe_json_dumps import safe_dumps @@ -165,7 +166,12 @@ def test_get_logging_payload_includes_kubernetes_pod_routing( @pytest.mark.parametrize( ("silent_experiment_metadata", "expected"), - [({"is_silent_experiment": True}, True), ({"is_silent_experiment": "true"}, False), ({}, False)], + [ + ({SILENT_EXPERIMENT_METADATA_KEY: True}, True), + ({SILENT_EXPERIMENT_METADATA_KEY: "true"}, False), + ({"is_silent_experiment": True}, False), + ({}, False), + ], ) def test_get_logging_payload_records_whether_the_call_was_a_silent_experiment( silent_experiment_metadata: dict[str, object], expected: bool diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 7064b52ad8c..f1940fa2d2f 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -23,6 +23,7 @@ from litellm.constants import ( SERVER_STREAMING_CLASSIFICATION_MARKER, SESSION_ID_GENERATED_METADATA_KEY, SESSION_ID_OMITTED_METADATA_KEY, + SILENT_EXPERIMENT_METADATA_KEY, ) from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs from litellm.litellm_core_utils.get_provider_specific_headers import ( @@ -8278,12 +8279,12 @@ def _reserved_stamp_key(key_metadata: dict | None = None) -> UserAPIKeyAuth: _PLANTED_STAMPS = { "attempted_fallbacks": 99, - "is_silent_experiment": True, "original_model_group": "spoofed-group", "request_retry_count": -100, "previous_models": [{"exception_type": "Timeout", "deployment_id": "spoofed", "attempted_retries": 101}], "_client_output_ceiling": {"api_base": "https://attacker.example"}, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: 10**9, + SILENT_EXPERIMENT_METADATA_KEY: True, "client_key": "client_value", } @@ -8313,7 +8314,7 @@ async def test_add_litellm_data_to_request_strips_router_reserved_stamps_from_bo assert "litellm_metadata" not in updated assert "attempted_fallbacks" not in updated["metadata"] - assert "is_silent_experiment" not in updated["metadata"] + assert SILENT_EXPERIMENT_METADATA_KEY not in updated["metadata"] assert "original_model_group" not in updated["metadata"] assert "_client_output_ceiling" not in updated["metadata"] assert "request_retry_count" not in updated["metadata"] diff --git a/tests/unit/test_router_silent_experiment.py b/tests/unit/test_router_silent_experiment.py index 97a0ba146cf..cea56219215 100644 --- a/tests/unit/test_router_silent_experiment.py +++ b/tests/unit/test_router_silent_experiment.py @@ -280,13 +280,24 @@ async def test_standard_logging_payload_marks_only_the_mirrored_calls_as_silent_ metadata_by_group: Final = {payload["model_group"]: payload["metadata"] for payload in payloads} assert metadata_by_group["shadow-a"]["is_silent_experiment"] is True assert metadata_by_group["shadow-b"]["is_silent_experiment"] is True - assert "is_silent_experiment" not in metadata_by_group["primary-model"] + assert metadata_by_group["primary-model"]["is_silent_experiment"] is False + + +def _spend_row_is_silent_experiment(call: dict[str, object]) -> object: + from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload + + now: Final = datetime.datetime.now(datetime.timezone.utc) + row: Final = get_logging_payload( + kwargs=call, + response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]), + start_time=now, + end_time=now, + ) + return json.loads(row["metadata"])["is_silent_experiment"] @pytest.mark.asyncio async def test_spend_rows_mark_only_the_mirrored_calls_when_the_caller_sends_litellm_metadata(recording_logger): - from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload - router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"])) await router.acompletion( model="primary-model", @@ -296,21 +307,29 @@ async def test_spend_rows_mark_only_the_mirrored_calls_when_the_caller_sends_lit ) await _wait_for_all_successes(recording_logger, expected=3) - now: Final = datetime.datetime.now(datetime.timezone.utc) silent_by_group: Final = { - call["standard_logging_object"]["model_group"]: json.loads( - get_logging_payload( - kwargs=call, - response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]), - start_time=now, - end_time=now, - )["metadata"] - )["is_silent_experiment"] + call["standard_logging_object"]["model_group"]: _spend_row_is_silent_experiment(call) for call in recording_logger.success_kwargs } assert silent_by_group == {"primary-model": False, "shadow-a": True, "shadow-b": True} +@pytest.mark.asyncio +async def test_a_caller_sent_is_silent_experiment_does_not_mark_a_real_call_as_mirrored(recording_logger): + router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"])) + await router.acompletion( + model="shadow-b", + messages=[{"role": "user", "content": "hi"}], + mock_response="pong", + metadata={"is_silent_experiment": True}, + ) + await _wait_for_all_successes(recording_logger, expected=1) + + call: Final = recording_logger.success_kwargs[0] + assert call["standard_logging_object"]["metadata"]["is_silent_experiment"] is False + assert _spend_row_is_silent_experiment(call) is False + + def test_multiple_shadow_targets_fan_out_sync(recording_logger): router = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"])) response = router.completion(