mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(router)!: stop a client's metadata flag from turning off silent_model mirroring (#52083)
* fix(router)!: stop a client's metadata flag from turning off silent_model mirroring A request that sent metadata.is_silent_experiment true skipped every mirror the deployment's silent_model configured. The check was a recursion guard from when mirroring first shipped. The router now stops a mirror from mirroring again on its own, so the metadata check only ever acted on client input BREAKING CHANGE: clients can no longer opt out of an admin's traffic mirroring by sending is_silent_experiment in request metadata * test(router): assert a caller-sent silent experiment flag still mirrors in the router core tests The two core tests pinned the old opt-out, where a request that sent metadata.is_silent_experiment skipped the deployment's silent_model * test(router): check a mirrored call is never mirrored again on the sync path * test(router): name why the sync mirror test suppresses the endpoint attribute check * ci: start the zizmor check GitHub skipped at the last push --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
parent
8fadc09434
commit
530a5094be
3 changed files with 59 additions and 9 deletions
|
|
@ -1333,8 +1333,6 @@ async def _run_silent(core: EndpointCore, silent_model: str, messages: object, k
|
|||
|
||||
async def _silent_async(core: EndpointCore, silent_model: str, messages: object, kwargs: Mapping[str, object]) -> None:
|
||||
try:
|
||||
if as_dict(kwargs.get("metadata")).get("is_silent_experiment", False):
|
||||
return
|
||||
silent: Final = _silent_kwargs(kwargs)
|
||||
as_dict_metadata: Final = silent["metadata"]
|
||||
if isinstance(as_dict_metadata, dict):
|
||||
|
|
@ -1346,8 +1344,6 @@ async def _silent_async(core: EndpointCore, silent_model: str, messages: object,
|
|||
|
||||
def _silent_thread(core: EndpointCore, silent_model: str, messages: object, kwargs: Mapping[str, object]) -> None:
|
||||
try:
|
||||
if as_dict(kwargs.get("metadata")).get("is_silent_experiment", False):
|
||||
return
|
||||
silent: Final = _silent_kwargs(kwargs)
|
||||
metadata: Final = silent["metadata"]
|
||||
if isinstance(metadata, dict):
|
||||
|
|
|
|||
|
|
@ -157,22 +157,39 @@ def test_sync_completion_mirrors_to_the_silent_model_on_a_background_thread(
|
|||
assert "is_silent_experiment" not in _metadata(primary.kwargs)
|
||||
|
||||
|
||||
async def test_a_request_already_marked_silent_does_not_start_another_experiment(
|
||||
async def test_a_caller_sent_silent_experiment_flag_still_mirrors_to_the_silent_model(
|
||||
make_core: MakeCore, host: ScriptedHost
|
||||
) -> None:
|
||||
core: Final = make_core(model_list=[deployment("g", "d1", silent_model="shadow"), deployment("shadow", "s1")])
|
||||
await core.acompletion(model="g", messages=MESSAGES, metadata={"is_silent_experiment": True}) # pyright: ignore[reportAttributeAccessIssue]
|
||||
await _settle()
|
||||
assert host.deployments_called() == ["d1"]
|
||||
assert sorted(host.deployments_called()) == ["d1", "s1"]
|
||||
|
||||
|
||||
def test_a_sync_request_already_marked_silent_does_not_start_another_experiment(
|
||||
def test_a_caller_sent_silent_experiment_flag_still_mirrors_to_the_silent_model_on_the_sync_path(
|
||||
make_core: MakeCore, host: ScriptedHost
|
||||
) -> None:
|
||||
core: Final = make_core(model_list=[deployment("g", "d1", silent_model="shadow"), deployment("shadow", "s1")])
|
||||
core.completion(model="g", messages=MESSAGES, metadata={"is_silent_experiment": True}) # pyright: ignore[reportAttributeAccessIssue]
|
||||
time.sleep(0.2)
|
||||
assert host.deployments_called() == ["d1"]
|
||||
_wait_for_calls(host, 2)
|
||||
assert sorted(host.deployments_called()) == ["d1", "s1"]
|
||||
|
||||
|
||||
def test_a_mirrored_call_is_not_mirrored_again_on_the_sync_path(make_core: MakeCore, host: ScriptedHost) -> None:
|
||||
core: Final = make_core(
|
||||
model_list=[
|
||||
deployment("g", "d1", silent_model="shadow"),
|
||||
deployment("shadow", "s1", silent_model="shadow-of-shadow"),
|
||||
deployment("shadow-of-shadow", "s2"),
|
||||
]
|
||||
)
|
||||
threads_before: Final = frozenset(threading.enumerate())
|
||||
core.completion( # pyright: ignore[reportAttributeAccessIssue] # endpoint method served by __getattr__
|
||||
model="g", messages=MESSAGES
|
||||
)
|
||||
for thread in frozenset(threading.enumerate()) - threads_before:
|
||||
thread.join(timeout=5)
|
||||
assert sorted(host.deployments_called()) == ["d1", "s1"]
|
||||
|
||||
|
||||
async def test_silent_experiment_marks_its_own_metadata_when_the_request_metadata_cannot_be_deep_copied(
|
||||
|
|
|
|||
|
|
@ -729,6 +729,43 @@ async def test_aembedding_with_silent_model_sends_provider_body_without_it(
|
|||
assert response.data[0]["embedding"] == [0.1, 0.2]
|
||||
|
||||
|
||||
def _mirrored_groups_are_logged(logger: _RecordingLogger) -> bool:
|
||||
return _silent_flag_by_group(logger).keys() >= {"primary-model", "shadow-a"}
|
||||
|
||||
|
||||
def test_a_caller_sent_silent_experiment_flag_does_not_turn_off_mirroring_on_the_sync_path(
|
||||
sync_and_async_recording_logger: _SyncAndAsyncRecordingLogger,
|
||||
) -> None:
|
||||
router: Final = Router(model_list=_streaming_model_list("shadow-a"))
|
||||
response: Final = router.completion(
|
||||
model="primary-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
mock_response="pong",
|
||||
metadata={"is_silent_experiment": True},
|
||||
)
|
||||
assert response.choices[0].message.content == "pong"
|
||||
sync_and_async_recording_logger.wait_until(lambda: _mirrored_groups_are_logged(sync_and_async_recording_logger))
|
||||
|
||||
assert _silent_flag_by_group(sync_and_async_recording_logger) == {"primary-model": None, "shadow-a": True}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_caller_sent_silent_experiment_flag_does_not_turn_off_mirroring_on_the_async_path(
|
||||
recording_logger: _RecordingLogger,
|
||||
) -> None:
|
||||
router: Final = Router(model_list=_streaming_model_list("shadow-a"))
|
||||
response: Final = await router.acompletion(
|
||||
model="primary-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
mock_response="pong",
|
||||
metadata={"is_silent_experiment": True},
|
||||
)
|
||||
assert response.choices[0].message.content == "pong"
|
||||
await asyncio.to_thread(recording_logger.wait_until, lambda: _mirrored_groups_are_logged(recording_logger))
|
||||
|
||||
assert _silent_flag_by_group(recording_logger) == {"primary-model": None, "shadow-a": True}
|
||||
|
||||
|
||||
def _judged_router(shadow_outcome: str) -> Router:
|
||||
return Router(
|
||||
model_list=[
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue