From 530a5094be98ae95eb8545d99e2006149b92f92e Mon Sep 17 00:00:00 2001 From: mika-lt <349045450+mika-lt@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:45:56 -0800 Subject: [PATCH] fix(router)!: stop a client's metadata flag from turning off silent_model mirroring (#52083) * fix(router)!: stop a client's metadata flag from turning off silent_model mirroring A request that sent metadata.is_silent_experiment true skipped every mirror the deployment's silent_model configured. The check was a recursion guard from when mirroring first shipped. The router now stops a mirror from mirroring again on its own, so the metadata check only ever acted on client input BREAKING CHANGE: clients can no longer opt out of an admin's traffic mirroring by sending is_silent_experiment in request metadata * test(router): assert a caller-sent silent experiment flag still mirrors in the router core tests The two core tests pinned the old opt-out, where a request that sent metadata.is_silent_experiment skipped the deployment's silent_model * test(router): check a mirrored call is never mirrored again on the sync path * test(router): name why the sync mirror test suppresses the endpoint attribute check * ci: start the zizmor check GitHub skipped at the last push --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/router_rust/invoke/endpoints.py | 4 -- .../unit/test_router/core/test_gaps_l2800.py | 27 +++++++++++--- tests/unit/test_router_silent_experiment.py | 37 +++++++++++++++++++ 3 files changed, 59 insertions(+), 9 deletions(-) diff --git a/litellm/router_rust/invoke/endpoints.py b/litellm/router_rust/invoke/endpoints.py index 6254a25dccd..b44e8f23f53 100644 --- a/litellm/router_rust/invoke/endpoints.py +++ b/litellm/router_rust/invoke/endpoints.py @@ -1333,8 +1333,6 @@ async def _run_silent(core: EndpointCore, silent_model: str, messages: object, k async def _silent_async(core: EndpointCore, silent_model: str, messages: object, kwargs: Mapping[str, object]) -> None: try: - if as_dict(kwargs.get("metadata")).get("is_silent_experiment", False): - return silent: Final = _silent_kwargs(kwargs) as_dict_metadata: Final = silent["metadata"] if isinstance(as_dict_metadata, dict): @@ -1346,8 +1344,6 @@ async def _silent_async(core: EndpointCore, silent_model: str, messages: object, def _silent_thread(core: EndpointCore, silent_model: str, messages: object, kwargs: Mapping[str, object]) -> None: try: - if as_dict(kwargs.get("metadata")).get("is_silent_experiment", False): - return silent: Final = _silent_kwargs(kwargs) metadata: Final = silent["metadata"] if isinstance(metadata, dict): diff --git a/tests/unit/test_router/core/test_gaps_l2800.py b/tests/unit/test_router/core/test_gaps_l2800.py index 196ce20d4b3..880e47d2ca9 100644 --- a/tests/unit/test_router/core/test_gaps_l2800.py +++ b/tests/unit/test_router/core/test_gaps_l2800.py @@ -157,22 +157,39 @@ def test_sync_completion_mirrors_to_the_silent_model_on_a_background_thread( assert "is_silent_experiment" not in _metadata(primary.kwargs) -async def test_a_request_already_marked_silent_does_not_start_another_experiment( +async def test_a_caller_sent_silent_experiment_flag_still_mirrors_to_the_silent_model( make_core: MakeCore, host: ScriptedHost ) -> None: core: Final = make_core(model_list=[deployment("g", "d1", silent_model="shadow"), deployment("shadow", "s1")]) await core.acompletion(model="g", messages=MESSAGES, metadata={"is_silent_experiment": True}) # pyright: ignore[reportAttributeAccessIssue] await _settle() - assert host.deployments_called() == ["d1"] + assert sorted(host.deployments_called()) == ["d1", "s1"] -def test_a_sync_request_already_marked_silent_does_not_start_another_experiment( +def test_a_caller_sent_silent_experiment_flag_still_mirrors_to_the_silent_model_on_the_sync_path( make_core: MakeCore, host: ScriptedHost ) -> None: core: Final = make_core(model_list=[deployment("g", "d1", silent_model="shadow"), deployment("shadow", "s1")]) core.completion(model="g", messages=MESSAGES, metadata={"is_silent_experiment": True}) # pyright: ignore[reportAttributeAccessIssue] - time.sleep(0.2) - assert host.deployments_called() == ["d1"] + _wait_for_calls(host, 2) + assert sorted(host.deployments_called()) == ["d1", "s1"] + + +def test_a_mirrored_call_is_not_mirrored_again_on_the_sync_path(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core( + model_list=[ + deployment("g", "d1", silent_model="shadow"), + deployment("shadow", "s1", silent_model="shadow-of-shadow"), + deployment("shadow-of-shadow", "s2"), + ] + ) + threads_before: Final = frozenset(threading.enumerate()) + core.completion( # pyright: ignore[reportAttributeAccessIssue] # endpoint method served by __getattr__ + model="g", messages=MESSAGES + ) + for thread in frozenset(threading.enumerate()) - threads_before: + thread.join(timeout=5) + assert sorted(host.deployments_called()) == ["d1", "s1"] async def test_silent_experiment_marks_its_own_metadata_when_the_request_metadata_cannot_be_deep_copied( diff --git a/tests/unit/test_router_silent_experiment.py b/tests/unit/test_router_silent_experiment.py index c7dd3bfa78c..9fde1ea5ac3 100644 --- a/tests/unit/test_router_silent_experiment.py +++ b/tests/unit/test_router_silent_experiment.py @@ -729,6 +729,43 @@ async def test_aembedding_with_silent_model_sends_provider_body_without_it( assert response.data[0]["embedding"] == [0.1, 0.2] +def _mirrored_groups_are_logged(logger: _RecordingLogger) -> bool: + return _silent_flag_by_group(logger).keys() >= {"primary-model", "shadow-a"} + + +def test_a_caller_sent_silent_experiment_flag_does_not_turn_off_mirroring_on_the_sync_path( + sync_and_async_recording_logger: _SyncAndAsyncRecordingLogger, +) -> None: + router: Final = Router(model_list=_streaming_model_list("shadow-a")) + response: Final = router.completion( + model="primary-model", + messages=[{"role": "user", "content": "hi"}], + mock_response="pong", + metadata={"is_silent_experiment": True}, + ) + assert response.choices[0].message.content == "pong" + sync_and_async_recording_logger.wait_until(lambda: _mirrored_groups_are_logged(sync_and_async_recording_logger)) + + assert _silent_flag_by_group(sync_and_async_recording_logger) == {"primary-model": None, "shadow-a": True} + + +@pytest.mark.asyncio +async def test_a_caller_sent_silent_experiment_flag_does_not_turn_off_mirroring_on_the_async_path( + recording_logger: _RecordingLogger, +) -> None: + router: Final = Router(model_list=_streaming_model_list("shadow-a")) + response: Final = await router.acompletion( + model="primary-model", + messages=[{"role": "user", "content": "hi"}], + mock_response="pong", + metadata={"is_silent_experiment": True}, + ) + assert response.choices[0].message.content == "pong" + await asyncio.to_thread(recording_logger.wait_until, lambda: _mirrored_groups_are_logged(recording_logger)) + + assert _silent_flag_by_group(recording_logger) == {"primary-model": None, "shadow-a": True} + + def _judged_router(shadow_outcome: str) -> Router: return Router( model_list=[