mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(router)!: stop a client's metadata flag from turning off silent_model mirroring
A request that sent metadata.is_silent_experiment true skipped every mirror the deployment's silent_model configured. The check was a recursion guard from when mirroring first shipped. The router now stops a mirror from mirroring again on its own, so the metadata check only ever acted on client input BREAKING CHANGE: clients can no longer opt out of an admin's traffic mirroring by sending is_silent_experiment in request metadata
This commit is contained in:
parent
feda51453a
commit
cda5eb2c7d
2 changed files with 34 additions and 19 deletions
|
|
@ -3394,10 +3394,6 @@ class Router:
|
|||
Run a silent experiment in the background (thread).
|
||||
"""
|
||||
try:
|
||||
# Prevent infinite recursion if silent model also has a silent model
|
||||
if kwargs.get("metadata", {}).get("is_silent_experiment", False):
|
||||
return
|
||||
|
||||
messages = copy.deepcopy(messages)
|
||||
|
||||
verbose_router_logger.info("Starting silent experiment for model %s", silent_model)
|
||||
|
|
@ -4582,10 +4578,6 @@ class Router:
|
|||
Run a silent experiment in the background.
|
||||
"""
|
||||
try:
|
||||
# Prevent infinite recursion if silent model also has a silent model
|
||||
if kwargs.get("metadata", {}).get("is_silent_experiment", False):
|
||||
return
|
||||
|
||||
messages = copy.deepcopy(messages)
|
||||
|
||||
verbose_router_logger.info("Starting silent experiment for model %s", silent_model)
|
||||
|
|
|
|||
|
|
@ -972,18 +972,41 @@ async def test_aembedding_with_silent_model_sends_provider_body_without_it(
|
|||
assert response.data[0]["embedding"] == [0.1, 0.2]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("run_silent_experiment", SILENT_EXPERIMENT_RUNNERS)
|
||||
def test_silent_experiment_does_not_launch_from_a_shadow_request(run_silent_experiment):
|
||||
router = Router(model_list=_streaming_model_list(["shadow-a"]))
|
||||
with patch.object(router, "acompletion", new_callable=AsyncMock, return_value=None) as acompletion:
|
||||
run_silent_experiment(
|
||||
router,
|
||||
silent_model="shadow-a",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
metadata={"is_silent_experiment": True},
|
||||
)
|
||||
def _mirrored_groups_are_logged(logger: _RecordingLogger) -> bool:
|
||||
return _silent_flag_by_group(logger).keys() >= {"primary-model", "shadow-a"}
|
||||
|
||||
acompletion.assert_not_awaited()
|
||||
|
||||
def test_a_caller_sent_silent_experiment_flag_does_not_turn_off_mirroring_on_the_sync_path(
|
||||
sync_and_async_recording_logger: _SyncAndAsyncRecordingLogger,
|
||||
) -> None:
|
||||
router: Final = Router(model_list=_streaming_model_list("shadow-a"))
|
||||
response: Final = router.completion(
|
||||
model="primary-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
mock_response="pong",
|
||||
metadata={"is_silent_experiment": True},
|
||||
)
|
||||
assert response.choices[0].message.content == "pong"
|
||||
sync_and_async_recording_logger.wait_until(lambda: _mirrored_groups_are_logged(sync_and_async_recording_logger))
|
||||
|
||||
assert _silent_flag_by_group(sync_and_async_recording_logger) == {"primary-model": None, "shadow-a": True}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_caller_sent_silent_experiment_flag_does_not_turn_off_mirroring_on_the_async_path(
|
||||
recording_logger: _RecordingLogger,
|
||||
) -> None:
|
||||
router: Final = Router(model_list=_streaming_model_list("shadow-a"))
|
||||
response: Final = await router.acompletion(
|
||||
model="primary-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
mock_response="pong",
|
||||
metadata={"is_silent_experiment": True},
|
||||
)
|
||||
assert response.choices[0].message.content == "pong"
|
||||
await asyncio.to_thread(recording_logger.wait_until, lambda: _mirrored_groups_are_logged(recording_logger))
|
||||
|
||||
assert _silent_flag_by_group(recording_logger) == {"primary-model": None, "shadow-a": True}
|
||||
|
||||
|
||||
def _judged_router(shadow_outcome: str) -> Router:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue