From f71b2cdfc96529bf4214d09412f8230c5fc7d2ef Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:07:45 -0700 Subject: [PATCH] fix(logging): mark mirrored calls with a router-only enum value so no client route can set is_silent_experiment The router stamps SilentExperimentMarker.MIRRORED, which client JSON can never produce, so batches, Responses WebSocket frames and pass-through need no strip. Langfuse layers the flag after its requester-metadata dedupe. --- litellm/integrations/langfuse/langfuse.py | 6 ++++ litellm/litellm_core_utils/litellm_logging.py | 4 +-- litellm/proxy/litellm_pre_call_utils.py | 2 -- .../pass_through_endpoints.py | 7 +---- .../spend_tracking/spend_tracking_utils.py | 4 +-- litellm/router.py | 5 ++-- litellm/types/router.py | 4 +++ tests/unit/integrations/test_langfuse.py | 30 +++++++++++++++++++ .../test_pass_through_endpoints.py | 12 ++++---- .../test_spend_tracking_utils.py | 6 ++-- .../unit/proxy/test_litellm_pre_call_utils.py | 3 -- tests/unit/test_router_silent_experiment.py | 27 +++++++++++++---- 12 files changed, 81 insertions(+), 29 deletions(-) diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index 86f80047e52..28c0d3ac8e2 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -832,6 +832,12 @@ class LangFuseLogger: ("vertex_location", vertex_location, bool(vertex_location)), ("aws_region_name", aws_region_name, bool(aws_region_name)), ("cache_hit", kwargs.get("cache_hit") or False, "cache_hit" in kwargs), + ( + "is_silent_experiment", + "is_silent_experiment" in allowlisted_metadata + and allowlisted_metadata["is_silent_experiment"] is True, + "is_silent_experiment" in allowlisted_metadata, + ), ) enrichments: Final[Mapping[str, object]] = { key: value for key, value, include in candidate_enrichments if include diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index ce0dc373f66..cf5c163fc22 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -155,7 +155,7 @@ from litellm.types.llms.openai import ( from litellm.types.mcp import MCPPostCallResponseObject from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.rerank import RerankResponse -from litellm.types.router import RouterRateLimitError +from litellm.types.router import RouterRateLimitError, SilentExperimentMarker from litellm.types.utils import ( DEPLOYMENT_SCOPED_PRICING_FIELDS, CachingDetails, @@ -6440,7 +6440,7 @@ class StandardLoggingPayloadSetup: is_silent_experiment=( metadata is not None and SILENT_EXPERIMENT_METADATA_KEY in metadata - and metadata[SILENT_EXPERIMENT_METADATA_KEY] is True + and metadata[SILENT_EXPERIMENT_METADATA_KEY] is SilentExperimentMarker.MIRRORED ), ) if isinstance(metadata, dict): diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 9124e4f50e9..09f53cecb72 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -43,7 +43,6 @@ from litellm.constants import ( SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY, SESSION_ID_OMITTED_METADATA_KEY, - SILENT_EXPERIMENT_METADATA_KEY, TYPESAFE_DANGER_ZONE_HEADER_PREFIX, VECTOR_STORE_CALLER_METADATA_KWARG, X_LITELLM_DISABLE_CALLBACKS, @@ -475,7 +474,6 @@ _ROUTER_RESERVED_METADATA_FIELDS: Final = frozenset( "request_retry_count", CLIENT_OUTPUT_CEILING_METADATA_KEY, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY, - SILENT_EXPERIMENT_METADATA_KEY, } ) _ALLOW_CLIENT_PRICING_OVERRIDE_METADATA_KEY: Final = "allow_client_pricing_override" diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 21da02cd572..fe7ac30e434 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -113,7 +113,6 @@ from litellm.proxy.common_utils.sse_keepalive import ( wrap_passthrough_sse_bytes_with_keepalive_pings, ) from litellm.proxy.litellm_pre_call_utils import ( # noqa: F401 # legacy module exports - _ROUTER_RESERVED_METADATA_FIELDS, # pyright: ignore[reportPrivateUsage] # pass-through applies the proxy's router-owned key strip LiteLLMProxyRequestSetup, _get_dynamic_logging_metadata, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _key_or_team_allows_client_pricing_override, # pyright: ignore[reportPrivateUsage] # reuse the proxy's pricing trust policy @@ -166,11 +165,7 @@ _PRICING_MODEL_OVERRIDE_PARAMS: Final = frozenset(("base_model",)) def _is_mergeable_client_metadata(key: str, value: object) -> bool: - return ( - not key.startswith("user_api_key_") - and key not in _ROUTER_RESERVED_METADATA_FIELDS - and (key != "model_info" or isinstance(value, Mapping)) - ) + return not key.startswith("user_api_key_") and (key != "model_info" or isinstance(value, Mapping)) # Global registry to track registered pass-through routes and prevent memory leaks diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 875762a7284..3bddb89b04a 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -75,7 +75,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( SETTLES_BACKGROUND_RESPONSE_KWARG, BackgroundResponseSettlement, ) -from litellm.types.router import DeploymentTypedDict, LiteLLM_Params +from litellm.types.router import DeploymentTypedDict, LiteLLM_Params, SilentExperimentMarker from litellm.types.utils import ( BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, PROMPT_CARRYING_GUARDRAIL_FIELDS, @@ -292,7 +292,7 @@ def _get_spend_logs_metadata( kubernetes_pod_routing=kubernetes_pod_routing, litellm_roi_estimator=metadata.get("litellm_roi_estimator") is True, is_silent_experiment=SILENT_EXPERIMENT_METADATA_KEY in metadata - and metadata[SILENT_EXPERIMENT_METADATA_KEY] is True, + and metadata[SILENT_EXPERIMENT_METADATA_KEY] is SilentExperimentMarker.MIRRORED, billed_retry_attempt=billed_retry_attempt, user_api_key_user_email=_stored_user_email(metadata), ) diff --git a/litellm/router.py b/litellm/router.py index 89a1124ff7b..2f61f274ab3 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -409,6 +409,7 @@ from litellm.types.router import ( RoutingPlugin, RoutingStrategy, SearchToolTypedDict, + SilentExperimentMarker, TaggedPreRoutingStrategy, holds_secret_pointer, ) @@ -3125,11 +3126,11 @@ class Router: silent_kwargs["metadata"].pop("litellm_parent_otel_span", None) silent_kwargs["metadata"]["is_silent_experiment"] = True - silent_kwargs["metadata"][SILENT_EXPERIMENT_METADATA_KEY] = True + silent_kwargs["metadata"][SILENT_EXPERIMENT_METADATA_KEY] = SilentExperimentMarker.MIRRORED if "litellm_metadata" in silent_kwargs and isinstance(silent_kwargs["litellm_metadata"], dict): silent_kwargs["litellm_metadata"] = { **silent_kwargs["litellm_metadata"], - SILENT_EXPERIMENT_METADATA_KEY: True, + SILENT_EXPERIMENT_METADATA_KEY: SilentExperimentMarker.MIRRORED, } # Pop logging objects and call IDs to ensure a fresh logging context diff --git a/litellm/types/router.py b/litellm/types/router.py index cd1389aea9d..2a4bd095255 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -66,6 +66,10 @@ from .utils import ( AllowedModelRegion: TypeAlias = Literal["eu", "us"] +class SilentExperimentMarker(str, enum.Enum): + MIRRORED = "mirrored" + + class FallbackSkipReason(str, enum.Enum): NOT_AUTHORIZED = "not_authorized" OVER_BUDGET = "over_budget" diff --git a/tests/unit/integrations/test_langfuse.py b/tests/unit/integrations/test_langfuse.py index 00d85b64566..462f82eeb52 100644 --- a/tests/unit/integrations/test_langfuse.py +++ b/tests/unit/integrations/test_langfuse.py @@ -830,6 +830,36 @@ class TestLangfuseUsageDetails(unittest.TestCase): assert generation_metadata["litellm_response_cost"] == 0.25 assert generation_metadata["api_base"] == "https://real-api-base" + def test_caller_sent_is_silent_experiment_cannot_hide_the_mirrored_call_flag(self): + payload = self._build_standard_logging_payload(trace_id="canary-trace-id") + payload["metadata"]["is_silent_experiment"] = True + payload["metadata"]["requester_metadata"] = {"is_silent_experiment": False} + kwargs = self._build_langfuse_kwargs(payload) + metadata = self._canary_request_metadata() + self.use_real_langfuse_client() + + with patch( + "litellm.integrations.langfuse.langfuse._add_prompt_to_generation_params", + side_effect=lambda generation_params, **kw: generation_params, + create=True, + ): + self.logger._log_langfuse_v2( + user_id="user-1", + metadata=metadata, + litellm_params={"metadata": metadata}, + output=None, + start_time=datetime.datetime(2024, 1, 1, 12, 0, 0), + end_time=datetime.datetime(2024, 1, 1, 12, 0, 1), + kwargs=kwargs, + optional_params={}, + input=None, + response_obj=None, + level="INFO", + litellm_call_id="canary-call-id", + ) + + assert self.exported_generation_metadata()["is_silent_experiment"] is True + def test_generation_metadata_carries_the_call_id_and_response_id(self): """ v2's generation id was ``time-_``, so a generation could diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 2ee5e17d6df..b7124236fbf 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -63,6 +63,7 @@ from litellm.proxy.pass_through_endpoints.success_handler import ( from litellm.proxy.route_llm_request import ProxyModelNotFoundError from litellm.types import utils as types_utils from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.router import SilentExperimentMarker from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, @@ -10339,14 +10340,15 @@ def test_init_kwargs_no_authenticated_identity_field_is_client_settable(mock_req assert survived == {}, f"client-supplied values survived for: {sorted(survived)}" +@pytest.mark.parametrize("planted", [True, SilentExperimentMarker.MIRRORED.value]) def test_init_kwargs_client_metadata_cannot_mark_a_pass_through_call_as_a_silent_experiment( - mock_request, mock_user_api_key_dict -): + planted: object, mock_user_api_key_dict: UserAPIKeyAuth +) -> None: from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload started: Final = datetime(2026, 1, 15, 12, 0, 0) result: Final = HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint( - request=mock_request(), + request=Request({"type": "http", "method": "POST", "path": "/test", "headers": [], "query_string": b""}), user_api_key_dict=mock_user_api_key_dict, passthrough_logging_payload=PassthroughStandardLoggingPayload(url="https://test.com", request_body={}), litellm_call_id="test-call-id", @@ -10360,8 +10362,8 @@ def test_init_kwargs_client_metadata_cannot_mark_a_pass_through_call_as_a_silent function_id="test-function-id", ), _parsed_body={ - "litellm_metadata": {SILENT_EXPERIMENT_METADATA_KEY: True, "client_key": "kept"}, - "metadata": {SILENT_EXPERIMENT_METADATA_KEY: True}, + "litellm_metadata": {SILENT_EXPERIMENT_METADATA_KEY: planted, "client_key": "kept"}, + "metadata": {SILENT_EXPERIMENT_METADATA_KEY: planted}, }, ) diff --git a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py index f5e95f87a52..ec3ba361b3f 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py @@ -17,6 +17,7 @@ import litellm.proxy.spend_tracking.spend_tracking_utils as spend_tracking_utils from litellm.constants import KUBERNETES_POD_ROUTING_KEY, LITELLM_TRUNCATED_PAYLOAD_FIELD, LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, LITTELM_CLI_SERVICE_ACCOUNT_NAME, LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, MAX_SPEND_LOG_MODEL_NAME_LENGTH, REDACTED_BY_LITELM_STRING, SESSION_ID_GENERATED_METADATA_KEY, SESSION_ID_OMITTED_METADATA_KEY, UNKNOWN_MODEL_SPEND_LOG_MODEL from litellm.constants import CALLER_SESSION_ID_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY, MANAGED_BACKGROUND_RESPONSE_METADATA_KEY from litellm.constants import SILENT_EXPERIMENT_METADATA_KEY +from litellm.types.router import SilentExperimentMarker from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup from litellm.llms.base_llm.ocr.transformation import OCRResponse, OCRUsageInfo from litellm.litellm_core_utils.safe_json_dumps import safe_dumps @@ -167,8 +168,9 @@ def test_get_logging_payload_includes_kubernetes_pod_routing( @pytest.mark.parametrize( ("silent_experiment_metadata", "expected"), [ - ({SILENT_EXPERIMENT_METADATA_KEY: True}, True), - ({SILENT_EXPERIMENT_METADATA_KEY: "true"}, False), + ({SILENT_EXPERIMENT_METADATA_KEY: SilentExperimentMarker.MIRRORED}, True), + ({SILENT_EXPERIMENT_METADATA_KEY: SilentExperimentMarker.MIRRORED.value}, False), + ({SILENT_EXPERIMENT_METADATA_KEY: True}, False), ({"is_silent_experiment": True}, False), ({}, False), ], diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index f1940fa2d2f..c2a5b01fe4e 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -23,7 +23,6 @@ from litellm.constants import ( SERVER_STREAMING_CLASSIFICATION_MARKER, SESSION_ID_GENERATED_METADATA_KEY, SESSION_ID_OMITTED_METADATA_KEY, - SILENT_EXPERIMENT_METADATA_KEY, ) from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs from litellm.litellm_core_utils.get_provider_specific_headers import ( @@ -8284,7 +8283,6 @@ _PLANTED_STAMPS = { "previous_models": [{"exception_type": "Timeout", "deployment_id": "spoofed", "attempted_retries": 101}], "_client_output_ceiling": {"api_base": "https://attacker.example"}, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: 10**9, - SILENT_EXPERIMENT_METADATA_KEY: True, "client_key": "client_value", } @@ -8314,7 +8312,6 @@ async def test_add_litellm_data_to_request_strips_router_reserved_stamps_from_bo assert "litellm_metadata" not in updated assert "attempted_fallbacks" not in updated["metadata"] - assert SILENT_EXPERIMENT_METADATA_KEY not in updated["metadata"] assert "original_model_group" not in updated["metadata"] assert "_client_output_ceiling" not in updated["metadata"] assert "request_retry_count" not in updated["metadata"] diff --git a/tests/unit/test_router_silent_experiment.py b/tests/unit/test_router_silent_experiment.py index cea56219215..c455153b3cf 100644 --- a/tests/unit/test_router_silent_experiment.py +++ b/tests/unit/test_router_silent_experiment.py @@ -12,11 +12,13 @@ import pytest import respx import litellm +from litellm.constants import SILENT_EXPERIMENT_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.router import Router from litellm.router import _silent_experiment_kwargs_snapshot from litellm.router import _silent_experiment_targets +from litellm.types.router import SilentExperimentMarker class _RecordingLogger(CustomLogger): @@ -265,7 +267,9 @@ async def test_multiple_shadow_targets_fan_out_async(recording_logger): @pytest.mark.asyncio -async def test_standard_logging_payload_marks_only_the_mirrored_calls_as_silent_experiments(recording_logger): +async def test_standard_logging_payload_marks_only_the_mirrored_calls_as_silent_experiments( + recording_logger: _RecordingLogger, +) -> None: router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"])) response: Final = await router.acompletion( model="primary-model", @@ -286,7 +290,7 @@ async def test_standard_logging_payload_marks_only_the_mirrored_calls_as_silent_ def _spend_row_is_silent_experiment(call: dict[str, object]) -> object: from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload - now: Final = datetime.datetime.now(datetime.timezone.utc) + now: Final = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc) row: Final = get_logging_payload( kwargs=call, response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]), @@ -297,7 +301,9 @@ def _spend_row_is_silent_experiment(call: dict[str, object]) -> object: @pytest.mark.asyncio -async def test_spend_rows_mark_only_the_mirrored_calls_when_the_caller_sends_litellm_metadata(recording_logger): +async def test_spend_rows_mark_only_the_mirrored_calls_when_the_caller_sends_litellm_metadata( + recording_logger: _RecordingLogger, +) -> None: router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"])) await router.acompletion( model="primary-model", @@ -315,13 +321,24 @@ async def test_spend_rows_mark_only_the_mirrored_calls_when_the_caller_sends_lit @pytest.mark.asyncio -async def test_a_caller_sent_is_silent_experiment_does_not_mark_a_real_call_as_mirrored(recording_logger): +@pytest.mark.parametrize( + ("bucket", "planted"), + [ + ("metadata", {"is_silent_experiment": True}), + ("metadata", {SILENT_EXPERIMENT_METADATA_KEY: SilentExperimentMarker.MIRRORED.value}), + ("litellm_metadata", {SILENT_EXPERIMENT_METADATA_KEY: SilentExperimentMarker.MIRRORED.value}), + ("litellm_metadata", {SILENT_EXPERIMENT_METADATA_KEY: True}), + ], +) +async def test_a_caller_sent_silent_experiment_flag_does_not_mark_a_real_call_as_mirrored( + recording_logger: _RecordingLogger, bucket: str, planted: dict[str, object] +) -> None: router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"])) await router.acompletion( model="shadow-b", messages=[{"role": "user", "content": "hi"}], mock_response="pong", - metadata={"is_silent_experiment": True}, + **{bucket: planted}, ) await _wait_for_all_successes(recording_logger, expected=1)