mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-31 06:56:59 +00:00
fix(router): keep the mirror flag when litellm_metadata is set and strip a client-sent is_silent_experiment
Spend rows read router metadata from litellm_metadata when a call carries it, so the router now sets the flag there too. The proxy drops a client-sent is_silent_experiment with the other router-owned keys, so real traffic cannot be recorded as mirrored.
This commit is contained in:
parent
a7e8895aa8
commit
f2d06470fd
4 changed files with 34 additions and 0 deletions
|
|
@ -469,6 +469,7 @@ _CLIENT_PRICING_METADATA_FIELDS: Final = frozenset({"model_info", "standard_logg
|
|||
_ROUTER_RESERVED_METADATA_FIELDS: Final = frozenset(
|
||||
{
|
||||
"attempted_fallbacks",
|
||||
"is_silent_experiment",
|
||||
"original_model_group",
|
||||
"previous_models",
|
||||
"request_retry_count",
|
||||
|
|
|
|||
|
|
@ -3124,6 +3124,8 @@ class Router:
|
|||
silent_kwargs["metadata"].pop("litellm_parent_otel_span", None)
|
||||
|
||||
silent_kwargs["metadata"]["is_silent_experiment"] = True
|
||||
if "litellm_metadata" in silent_kwargs and isinstance(silent_kwargs["litellm_metadata"], dict):
|
||||
silent_kwargs["litellm_metadata"] = {**silent_kwargs["litellm_metadata"], "is_silent_experiment": True}
|
||||
|
||||
# Pop logging objects and call IDs to ensure a fresh logging context
|
||||
# This prevents collisions in the Proxy's database (spend_logs)
|
||||
|
|
|
|||
|
|
@ -8278,6 +8278,7 @@ def _reserved_stamp_key(key_metadata: dict | None = None) -> UserAPIKeyAuth:
|
|||
|
||||
_PLANTED_STAMPS = {
|
||||
"attempted_fallbacks": 99,
|
||||
"is_silent_experiment": True,
|
||||
"original_model_group": "spoofed-group",
|
||||
"request_retry_count": -100,
|
||||
"previous_models": [{"exception_type": "Timeout", "deployment_id": "spoofed", "attempted_retries": 101}],
|
||||
|
|
@ -8312,6 +8313,7 @@ async def test_add_litellm_data_to_request_strips_router_reserved_stamps_from_bo
|
|||
|
||||
assert "litellm_metadata" not in updated
|
||||
assert "attempted_fallbacks" not in updated["metadata"]
|
||||
assert "is_silent_experiment" not in updated["metadata"]
|
||||
assert "original_model_group" not in updated["metadata"]
|
||||
assert "_client_output_ceiling" not in updated["metadata"]
|
||||
assert "request_retry_count" not in updated["metadata"]
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import asyncio
|
||||
import datetime
|
||||
import json
|
||||
import time
|
||||
from collections.abc import Callable, Mapping
|
||||
|
|
@ -282,6 +283,34 @@ async def test_standard_logging_payload_marks_only_the_mirrored_calls_as_silent_
|
|||
assert "is_silent_experiment" not in metadata_by_group["primary-model"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_spend_rows_mark_only_the_mirrored_calls_when_the_caller_sends_litellm_metadata(recording_logger):
|
||||
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
|
||||
|
||||
router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"]))
|
||||
await router.acompletion(
|
||||
model="primary-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
mock_response="pong",
|
||||
litellm_metadata={"user_api_key": "test-key"},
|
||||
)
|
||||
await _wait_for_all_successes(recording_logger, expected=3)
|
||||
|
||||
now: Final = datetime.datetime.now(datetime.timezone.utc)
|
||||
silent_by_group: Final = {
|
||||
call["standard_logging_object"]["model_group"]: json.loads(
|
||||
get_logging_payload(
|
||||
kwargs=call,
|
||||
response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]),
|
||||
start_time=now,
|
||||
end_time=now,
|
||||
)["metadata"]
|
||||
)["is_silent_experiment"]
|
||||
for call in recording_logger.success_kwargs
|
||||
}
|
||||
assert silent_by_group == {"primary-model": False, "shadow-a": True, "shadow-b": True}
|
||||
|
||||
|
||||
def test_multiple_shadow_targets_fan_out_sync(recording_logger):
|
||||
router = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"]))
|
||||
response = router.completion(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue