scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-31 06:56:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 244

fix(logging): mark mirrored calls with a router-only key so clients cannot set is_silent_experiment

Browse source 
Spend rows and the standard logging payload now derive is_silent_experiment from _litellm_silent_experiment, which only the router writes and the proxy and pass-through routes strip from client metadata. A caller's own is_silent_experiment key stays untouched, so it keeps its current effect on mirroring and on provider metadata, and unmirrored calls log false in both places.
This commit is contained in:
mateo-berri 2026-10-20 18:53:31 -07:00
parent f2d06470fd
commit d23c9fe320
10 changed files with 103 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
litellm/constants.py
View file

@ -1653,6 +1653,7 @@ ROUTING_REQUEST_TAGS_METADATA_KEY: Final = "_routing_request_tags"
BAGGAGE_MAPPED_TAGS_METADATA_KEY: Final = "_baggage_mapped_tags"
VECTOR_STORE_CALLER_METADATA_KWARG: Final = "litellm_vector_store_caller_metadata"
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted_tokens"
SILENT_EXPERIMENT_METADATA_KEY: Final = "_litellm_silent_experiment"
INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
BILLED_RETRY_ATTEMPT_METADATA_KEY: Final = "billed_retry_attempt"
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response"

8
litellm/litellm_core_utils/litellm_logging.py
View file

@ -47,6 +47,7 @@ from litellm.constants import (
LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL,
PROVIDER_REQUEST_ID_HEADERS,
REDACTED_BY_LITELLM,
SILENT_EXPERIMENT_METADATA_KEY,
)
from litellm.cost_calculator import (
RealtimeAPITokenUsageProcessor,
@ -305,7 +306,7 @@ else:
_in_memory_loggers: Final[list[CustomLogger]] = []
_STANDARD_LOGGING_METADATA_RESOLVED_KEYS: Final[frozenset[str]] = frozenset(
("used_client_oauth_token", "usage_object", KUBERNETES_POD_ROUTING_KEY)
("used_client_oauth_token", "usage_object", KUBERNETES_POD_ROUTING_KEY, "is_silent_experiment")
)
_STANDARD_LOGGING_METADATA_KEYS: Final[frozenset[str]] = (
frozenset(StandardLoggingMetadata.__annotations__.keys()) - _STANDARD_LOGGING_METADATA_RESOLVED_KEYS
@ -6436,6 +6437,11 @@ class StandardLoggingPayloadSetup:
proxy_stamped_used_client_oauth_token(metadata, litellm_params),
custom_llm_provider,
),
is_silent_experiment=(
metadata is not None
and SILENT_EXPERIMENT_METADATA_KEY in metadata
and metadata[SILENT_EXPERIMENT_METADATA_KEY] is True
),
)
if isinstance(metadata, dict):
for key in metadata.keys() & _STANDARD_LOGGING_METADATA_KEYS:

3
litellm/proxy/litellm_pre_call_utils.py
View file

@ -43,6 +43,7 @@ from litellm.constants import (
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
SESSION_ID_GENERATED_METADATA_KEY,
SESSION_ID_OMITTED_METADATA_KEY,
SILENT_EXPERIMENT_METADATA_KEY,
TYPESAFE_DANGER_ZONE_HEADER_PREFIX,
VECTOR_STORE_CALLER_METADATA_KWARG,
X_LITELLM_DISABLE_CALLBACKS,
@ -469,12 +470,12 @@ _CLIENT_PRICING_METADATA_FIELDS: Final = frozenset({"model_info", "standard_logg
_ROUTER_RESERVED_METADATA_FIELDS: Final = frozenset(
{
"attempted_fallbacks",
"is_silent_experiment",
"original_model_group",
"previous_models",
"request_retry_count",
CLIENT_OUTPUT_CEILING_METADATA_KEY,
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY,
SILENT_EXPERIMENT_METADATA_KEY,
}
)
_ALLOW_CLIENT_PRICING_OVERRIDE_METADATA_KEY: Final = "allow_client_pricing_override"

7
litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
View file

@ -113,6 +113,7 @@ from litellm.proxy.common_utils.sse_keepalive import (
wrap_passthrough_sse_bytes_with_keepalive_pings,
)
from litellm.proxy.litellm_pre_call_utils import ( # noqa: F401 # legacy module exports
_ROUTER_RESERVED_METADATA_FIELDS, # pyright: ignore[reportPrivateUsage] # pass-through applies the proxy's router-owned key strip
LiteLLMProxyRequestSetup,
_get_dynamic_logging_metadata, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
_key_or_team_allows_client_pricing_override, # pyright: ignore[reportPrivateUsage] # reuse the proxy's pricing trust policy
@ -165,7 +166,11 @@ _PRICING_MODEL_OVERRIDE_PARAMS: Final = frozenset(("base_model",))
def _is_mergeable_client_metadata(key: str, value: object) -> bool:
return not key.startswith("user_api_key_") and (key != "model_info" or isinstance(value, Mapping))
return (
not key.startswith("user_api_key_")
and key not in _ROUTER_RESERVED_METADATA_FIELDS
and (key != "model_info" or isinstance(value, Mapping))
)
# Global registry to track registered pass-through routes and prevent memory leaks

4
litellm/proxy/spend_tracking/spend_tracking_utils.py
View file

@ -31,6 +31,7 @@ from litellm.constants import (
NON_INFERENCE_CALL_TYPES,
REDACTED_BY_LITELM_STRING,
SESSION_ID_OMITTED_METADATA_KEY,
SILENT_EXPERIMENT_METADATA_KEY,
UNKNOWN_MODEL_SPEND_LOG_MODEL,
)
from litellm.constants import (
@ -290,7 +291,8 @@ def _get_spend_logs_metadata(
used_client_oauth_token=used_client_oauth_token,
kubernetes_pod_routing=kubernetes_pod_routing,
litellm_roi_estimator=metadata.get("litellm_roi_estimator") is True,
is_silent_experiment="is_silent_experiment" in metadata and metadata["is_silent_experiment"] is True,
is_silent_experiment=SILENT_EXPERIMENT_METADATA_KEY in metadata
and metadata[SILENT_EXPERIMENT_METADATA_KEY] is True,
billed_retry_attempt=billed_retry_attempt,
user_api_key_user_email=_stored_user_email(metadata),
)

7
litellm/router.py
View file

@ -78,6 +78,7 @@ from litellm.constants import (
ROUTING_REQUEST_TAGS_METADATA_KEY,
RUNTIME_UPDATABLE_ROUTER_SETTINGS,
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
SILENT_EXPERIMENT_METADATA_KEY,
)
from litellm.integrations.custom_guardrail import CustomGuardrail, is_guardrail_intervention
from litellm.integrations.custom_logger import CustomLogger
@ -3124,8 +3125,12 @@ class Router:
silent_kwargs["metadata"].pop("litellm_parent_otel_span", None)
silent_kwargs["metadata"]["is_silent_experiment"] = True
silent_kwargs["metadata"][SILENT_EXPERIMENT_METADATA_KEY] = True
if "litellm_metadata" in silent_kwargs and isinstance(silent_kwargs["litellm_metadata"], dict):
silent_kwargs["litellm_metadata"] = {**silent_kwargs["litellm_metadata"], "is_silent_experiment": True}
silent_kwargs["litellm_metadata"] = {
**silent_kwargs["litellm_metadata"],
SILENT_EXPERIMENT_METADATA_KEY: True,
}
# Pop logging objects and call IDs to ensure a fresh logging context
# This prevents collisions in the Proxy's database (spend_logs)

38
tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py
View file

@ -30,7 +30,7 @@ from starlette.datastructures import UploadFile as StarletteUploadFile
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS
from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS, SILENT_EXPERIMENT_METADATA_KEY
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@ -10338,6 +10338,42 @@ def test_init_kwargs_no_authenticated_identity_field_is_client_settable(mock_req
survived: Final = {key: metadata.get(key) for key in expected if metadata.get(key) != expected[key]}
assert survived == {}, f"client-supplied values survived for: {sorted(survived)}"
def test_init_kwargs_client_metadata_cannot_mark_a_pass_through_call_as_a_silent_experiment(
mock_request, mock_user_api_key_dict
):
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
started: Final = datetime(2026, 1, 15, 12, 0, 0)
result: Final = HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint(
request=mock_request(),
user_api_key_dict=mock_user_api_key_dict,
passthrough_logging_payload=PassthroughStandardLoggingPayload(url="https://test.com", request_body={}),
litellm_call_id="test-call-id",
logging_obj=LiteLLMLoggingObj(
model="test-model",
messages=[],
stream=False,
call_type="test-call-type",
start_time=started,
litellm_call_id="test-call-id",
function_id="test-function-id",
),
_parsed_body={
"litellm_metadata": {SILENT_EXPERIMENT_METADATA_KEY: True, "client_key": "kept"},
"metadata": {SILENT_EXPERIMENT_METADATA_KEY: True},
},
)
row: Final = get_logging_payload(
kwargs={"model": "test-model", "litellm_params": result["litellm_params"]},
response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]),
start_time=started,
end_time=started,
)
assert json.loads(row["metadata"])["is_silent_experiment"] is False
assert result["litellm_params"]["metadata"]["client_key"] == "kept"
@pytest.mark.asyncio
async def test_direct_passthrough_keeps_mapped_baggage_tags_outside_upstream_body():
from litellm.proxy.common_utils.http_parsing_utils import read_request_body

8
tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py
View file

@ -16,6 +16,7 @@ import litellm.constants as litellm_constants
import litellm.proxy.spend_tracking.spend_tracking_utils as spend_tracking_utils
from litellm.constants import KUBERNETES_POD_ROUTING_KEY, LITELLM_TRUNCATED_PAYLOAD_FIELD, LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, LITTELM_CLI_SERVICE_ACCOUNT_NAME, LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, MAX_SPEND_LOG_MODEL_NAME_LENGTH, REDACTED_BY_LITELM_STRING, SESSION_ID_GENERATED_METADATA_KEY, SESSION_ID_OMITTED_METADATA_KEY, UNKNOWN_MODEL_SPEND_LOG_MODEL
from litellm.constants import CALLER_SESSION_ID_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY, MANAGED_BACKGROUND_RESPONSE_METADATA_KEY
from litellm.constants import SILENT_EXPERIMENT_METADATA_KEY
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
from litellm.llms.base_llm.ocr.transformation import OCRResponse, OCRUsageInfo
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
@ -165,7 +166,12 @@ def test_get_logging_payload_includes_kubernetes_pod_routing(
@pytest.mark.parametrize(
("silent_experiment_metadata", "expected"),
[({"is_silent_experiment": True}, True), ({"is_silent_experiment": "true"}, False), ({}, False)],
[
({SILENT_EXPERIMENT_METADATA_KEY: True}, True),
({SILENT_EXPERIMENT_METADATA_KEY: "true"}, False),
({"is_silent_experiment": True}, False),
({}, False),
],
)
def test_get_logging_payload_records_whether_the_call_was_a_silent_experiment(
silent_experiment_metadata: dict[str, object], expected: bool

5
tests/unit/proxy/test_litellm_pre_call_utils.py
View file

@ -23,6 +23,7 @@ from litellm.constants import (
SERVER_STREAMING_CLASSIFICATION_MARKER,
SESSION_ID_GENERATED_METADATA_KEY,
SESSION_ID_OMITTED_METADATA_KEY,
SILENT_EXPERIMENT_METADATA_KEY,
)
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs
from litellm.litellm_core_utils.get_provider_specific_headers import (
@ -8278,12 +8279,12 @@ def _reserved_stamp_key(key_metadata: dict | None = None) -> UserAPIKeyAuth:
_PLANTED_STAMPS = {
"attempted_fallbacks": 99,
"is_silent_experiment": True,
"original_model_group": "spoofed-group",
"request_retry_count": -100,
"previous_models": [{"exception_type": "Timeout", "deployment_id": "spoofed", "attempted_retries": 101}],
"_client_output_ceiling": {"api_base": "https://attacker.example"},
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: 10**9,
SILENT_EXPERIMENT_METADATA_KEY: True,
"client_key": "client_value",
}
@ -8313,7 +8314,7 @@ async def test_add_litellm_data_to_request_strips_router_reserved_stamps_from_bo
assert "litellm_metadata" not in updated
assert "attempted_fallbacks" not in updated["metadata"]
assert "is_silent_experiment" not in updated["metadata"]
assert SILENT_EXPERIMENT_METADATA_KEY not in updated["metadata"]
assert "original_model_group" not in updated["metadata"]
assert "_client_output_ceiling" not in updated["metadata"]
assert "request_retry_count" not in updated["metadata"]

43
tests/unit/test_router_silent_experiment.py
View file

@ -280,13 +280,24 @@ async def test_standard_logging_payload_marks_only_the_mirrored_calls_as_silent_
metadata_by_group: Final = {payload["model_group"]: payload["metadata"] for payload in payloads}
assert metadata_by_group["shadow-a"]["is_silent_experiment"] is True
assert metadata_by_group["shadow-b"]["is_silent_experiment"] is True
assert "is_silent_experiment" not in metadata_by_group["primary-model"]
assert metadata_by_group["primary-model"]["is_silent_experiment"] is False
def _spend_row_is_silent_experiment(call: dict[str, object]) -> object:
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
now: Final = datetime.datetime.now(datetime.timezone.utc)
row: Final = get_logging_payload(
kwargs=call,
response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]),
start_time=now,
end_time=now,
)
return json.loads(row["metadata"])["is_silent_experiment"]
@pytest.mark.asyncio
async def test_spend_rows_mark_only_the_mirrored_calls_when_the_caller_sends_litellm_metadata(recording_logger):
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"]))
await router.acompletion(
model="primary-model",
@ -296,21 +307,29 @@ async def test_spend_rows_mark_only_the_mirrored_calls_when_the_caller_sends_lit
)
await _wait_for_all_successes(recording_logger, expected=3)
now: Final = datetime.datetime.now(datetime.timezone.utc)
silent_by_group: Final = {
call["standard_logging_object"]["model_group"]: json.loads(
get_logging_payload(
kwargs=call,
response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]),
start_time=now,
end_time=now,
)["metadata"]
)["is_silent_experiment"]
call["standard_logging_object"]["model_group"]: _spend_row_is_silent_experiment(call)
for call in recording_logger.success_kwargs
}
assert silent_by_group == {"primary-model": False, "shadow-a": True, "shadow-b": True}
@pytest.mark.asyncio
async def test_a_caller_sent_is_silent_experiment_does_not_mark_a_real_call_as_mirrored(recording_logger):
router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"]))
await router.acompletion(
model="shadow-b",
messages=[{"role": "user", "content": "hi"}],
mock_response="pong",
metadata={"is_silent_experiment": True},
)
await _wait_for_all_successes(recording_logger, expected=1)
call: Final = recording_logger.success_kwargs[0]
assert call["standard_logging_object"]["metadata"]["is_silent_experiment"] is False
assert _spend_row_is_silent_experiment(call) is False
def test_multiple_shadow_targets_fan_out_sync(recording_logger):
router = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"]))
response = router.completion(