fix(router)!: stop a client's metadata flag from turning off silent_model mirroring (#52083)

* fix(router)!: stop a client's metadata flag from turning off silent_model mirroring

A request that sent metadata.is_silent_experiment true skipped every mirror the
deployment's silent_model configured. The check was a recursion guard from when
mirroring first shipped. The router now stops a mirror from mirroring again on
its own, so the metadata check only ever acted on client input

BREAKING CHANGE: clients can no longer opt out of an admin's traffic mirroring by
sending is_silent_experiment in request metadata

* test(router): assert a caller-sent silent experiment flag still mirrors in the router core tests

The two core tests pinned the old opt-out, where a request that sent
metadata.is_silent_experiment skipped the deployment's silent_model

* test(router): check a mirrored call is never mirrored again on the sync path

* test(router): name why the sync mirror test suppresses the endpoint attribute check

* ci: start the zizmor check GitHub skipped at the last push

---------

Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
mika-lt 2026-11-16 15:45:56 -08:00 • committed by GitHub
parent 8fadc09434
commit 530a5094be
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 59 additions and 9 deletions

View file

@ -1333,8 +1333,6 @@ async def _run_silent(core: EndpointCore, silent_model: str, messages: object, k
async def _silent_async(core: EndpointCore, silent_model: str, messages: object, kwargs: Mapping[str, object]) -> None:
try:
if as_dict(kwargs.get("metadata")).get("is_silent_experiment", False):
return
silent: Final = _silent_kwargs(kwargs)
as_dict_metadata: Final = silent["metadata"]
if isinstance(as_dict_metadata, dict):
@ -1346,8 +1344,6 @@ async def _silent_async(core: EndpointCore, silent_model: str, messages: object,
def _silent_thread(core: EndpointCore, silent_model: str, messages: object, kwargs: Mapping[str, object]) -> None:
try:
if as_dict(kwargs.get("metadata")).get("is_silent_experiment", False):
return
silent: Final = _silent_kwargs(kwargs)
metadata: Final = silent["metadata"]
if isinstance(metadata, dict):

View file

@ -157,22 +157,39 @@ def test_sync_completion_mirrors_to_the_silent_model_on_a_background_thread(
assert "is_silent_experiment" not in _metadata(primary.kwargs)
async def test_a_request_already_marked_silent_does_not_start_another_experiment(
async def test_a_caller_sent_silent_experiment_flag_still_mirrors_to_the_silent_model(
make_core: MakeCore, host: ScriptedHost
) -> None:
core: Final = make_core(model_list=[deployment("g", "d1", silent_model="shadow"), deployment("shadow", "s1")])
await core.acompletion(model="g", messages=MESSAGES, metadata={"is_silent_experiment": True}) # pyright: ignore[reportAttributeAccessIssue]
await _settle()
assert host.deployments_called() == ["d1"]
assert sorted(host.deployments_called()) == ["d1", "s1"]
def test_a_sync_request_already_marked_silent_does_not_start_another_experiment(
def test_a_caller_sent_silent_experiment_flag_still_mirrors_to_the_silent_model_on_the_sync_path(
make_core: MakeCore, host: ScriptedHost
) -> None:
core: Final = make_core(model_list=[deployment("g", "d1", silent_model="shadow"), deployment("shadow", "s1")])
core.completion(model="g", messages=MESSAGES, metadata={"is_silent_experiment": True}) # pyright: ignore[reportAttributeAccessIssue]
time.sleep(0.2)
assert host.deployments_called() == ["d1"]
_wait_for_calls(host, 2)
assert sorted(host.deployments_called()) == ["d1", "s1"]
def test_a_mirrored_call_is_not_mirrored_again_on_the_sync_path(make_core: MakeCore, host: ScriptedHost) -> None:
core: Final = make_core(
model_list=[
deployment("g", "d1", silent_model="shadow"),
deployment("shadow", "s1", silent_model="shadow-of-shadow"),
deployment("shadow-of-shadow", "s2"),
]
)
threads_before: Final = frozenset(threading.enumerate())
core.completion( # pyright: ignore[reportAttributeAccessIssue] # endpoint method served by __getattr__
model="g", messages=MESSAGES
)
for thread in frozenset(threading.enumerate()) - threads_before:
thread.join(timeout=5)
assert sorted(host.deployments_called()) == ["d1", "s1"]
async def test_silent_experiment_marks_its_own_metadata_when_the_request_metadata_cannot_be_deep_copied(

View file

@ -729,6 +729,43 @@ async def test_aembedding_with_silent_model_sends_provider_body_without_it(
assert response.data[0]["embedding"] == [0.1, 0.2]
def _mirrored_groups_are_logged(logger: _RecordingLogger) -> bool:
return _silent_flag_by_group(logger).keys() >= {"primary-model", "shadow-a"}
def test_a_caller_sent_silent_experiment_flag_does_not_turn_off_mirroring_on_the_sync_path(
sync_and_async_recording_logger: _SyncAndAsyncRecordingLogger,
) -> None:
router: Final = Router(model_list=_streaming_model_list("shadow-a"))
response: Final = router.completion(
model="primary-model",
messages=[{"role": "user", "content": "hi"}],
mock_response="pong",
metadata={"is_silent_experiment": True},
)
assert response.choices[0].message.content == "pong"
sync_and_async_recording_logger.wait_until(lambda: _mirrored_groups_are_logged(sync_and_async_recording_logger))
assert _silent_flag_by_group(sync_and_async_recording_logger) == {"primary-model": None, "shadow-a": True}
@pytest.mark.asyncio
async def test_a_caller_sent_silent_experiment_flag_does_not_turn_off_mirroring_on_the_async_path(
recording_logger: _RecordingLogger,
) -> None:
router: Final = Router(model_list=_streaming_model_list("shadow-a"))
response: Final = await router.acompletion(
model="primary-model",
messages=[{"role": "user", "content": "hi"}],
mock_response="pong",
metadata={"is_silent_experiment": True},
)
assert response.choices[0].message.content == "pong"
await asyncio.to_thread(recording_logger.wait_until, lambda: _mirrored_groups_are_logged(recording_logger))
assert _silent_flag_by_group(recording_logger) == {"primary-model": None, "shadow-a": True}
def _judged_router(shadow_outcome: str) -> Router:
return Router(
model_list=[