This commit is contained in:
Mateo Wang 2026-10-27 02:21:16 +00:00 • committed by GitHub
commit 9e9aa1e3b6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 265 additions and 8 deletions

View file

@ -1660,6 +1660,7 @@ ROUTING_REQUEST_TAGS_METADATA_KEY: Final = "_routing_request_tags"
BAGGAGE_MAPPED_TAGS_METADATA_KEY: Final = "_baggage_mapped_tags"
VECTOR_STORE_CALLER_METADATA_KWARG: Final = "litellm_vector_store_caller_metadata"
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted_tokens"
SILENT_EXPERIMENT_METADATA_KEY: Final = "_litellm_silent_experiment"
INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
BILLED_RETRY_ATTEMPT_METADATA_KEY: Final = "billed_retry_attempt"
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response"

View file

@ -833,6 +833,12 @@ class LangFuseLogger:
("vertex_location", vertex_location, bool(vertex_location)),
("aws_region_name", aws_region_name, bool(aws_region_name)),
("cache_hit", kwargs.get("cache_hit") or False, "cache_hit" in kwargs),
(
"is_silent_experiment",
"is_silent_experiment" in allowlisted_metadata
and allowlisted_metadata["is_silent_experiment"] is True,
"is_silent_experiment" in allowlisted_metadata,
),
)
enrichments: Final[Mapping[str, object]] = {
key: value for key, value, include in candidate_enrichments if include

View file

@ -47,6 +47,7 @@ from litellm.constants import (
LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL,
PROVIDER_REQUEST_ID_HEADERS,
REDACTED_BY_LITELLM,
SILENT_EXPERIMENT_METADATA_KEY,
)
from litellm.cost_calculator import (
RealtimeAPITokenUsageProcessor,
@ -157,7 +158,7 @@ from litellm.types.llms.openai import (
from litellm.types.mcp import MCPPostCallResponseObject
from litellm.types.prompts.init_prompts import PromptSpec
from litellm.types.rerank import RerankResponse
from litellm.types.router import RouterRateLimitError
from litellm.types.router import RouterRateLimitError, SilentExperimentFlag, SilentExperimentMarker
from litellm.types.utils import (
DEPLOYMENT_SCOPED_PRICING_FIELDS,
CachingDetails,
@ -309,8 +310,9 @@ else:
_in_memory_loggers: Final[list[CustomLogger]] = []
_STANDARD_LOGGING_METADATA_RESOLVED_KEYS: Final[frozenset[str]] = frozenset(
("used_client_oauth_token", "usage_object", KUBERNETES_POD_ROUTING_KEY)
("used_client_oauth_token", "usage_object", KUBERNETES_POD_ROUTING_KEY, "is_silent_experiment")
)
_SILENT_EXPERIMENT_BUCKETS: Final = ("metadata", "litellm_metadata")
_STANDARD_LOGGING_METADATA_KEYS: Final[frozenset[str]] = (
frozenset(StandardLoggingMetadata.__annotations__.keys()) - _STANDARD_LOGGING_METADATA_RESOLVED_KEYS
)
@ -803,6 +805,7 @@ class Logging(LiteLLMLoggingBaseClass):
litellm_params = scrub_sensitive_keys_in_metadata(litellm_params)
self.litellm_params = litellm_params
self.is_silent_experiment = False
# Initialize cost breakdown field
self.cost_breakdown: CostBreakdown | None = None
@ -1118,6 +1121,7 @@ class Logging(LiteLLMLoggingBaseClass):
if model is not None:
self.model = model
self.user = user
self.is_silent_experiment = self.is_silent_experiment or is_silent_experiment_call(litellm_params)
self.litellm_params = {
**self.litellm_params,
**scrub_sensitive_keys_in_metadata(litellm_params),
@ -6523,6 +6527,7 @@ class StandardLoggingPayloadSetup:
start_time: dt_object | None = None,
response_id: str | None = None,
custom_llm_provider: str | None = None,
is_silent_experiment: bool = False,
) -> StandardLoggingMetadata:
"""
Clean and filter the metadata dictionary to include only the specified keys in StandardLoggingMetadata.
@ -6596,6 +6601,7 @@ class StandardLoggingPayloadSetup:
proxy_stamped_used_client_oauth_token(metadata, litellm_params),
custom_llm_provider,
),
**(SilentExperimentFlag(is_silent_experiment=True) if is_silent_experiment else SilentExperimentFlag()),
)
if isinstance(metadata, dict):
for key in metadata.keys() & _STANDARD_LOGGING_METADATA_KEYS:
@ -7398,6 +7404,7 @@ def get_standard_logging_object_payload(
start_time=start_time,
response_id=id,
custom_llm_provider=selected_provider if isinstance(selected_provider, str) else None,
is_silent_experiment=logging_obj.is_silent_experiment is True,
)
_request_body: Final = proxy_server_request.get("body", {})
end_user_id: Final = clean_metadata["user_api_key_end_user_id"] or _request_body.get(
@ -7692,7 +7699,9 @@ def get_standard_logging_metadata(
return clean_metadata
def scrub_sensitive_keys_in_metadata(litellm_params: dict | None):
def scrub_sensitive_keys_in_metadata(
litellm_params: dict[str, Any] | None, # mutable-ok: the langfuse masking function is moved onto these params
) -> dict[str, Any]: # mutable-ok: loggers write into the returned params and their metadata
if litellm_params is None:
litellm_params = {}
@ -7707,7 +7716,24 @@ def scrub_sensitive_keys_in_metadata(litellm_params: dict | None):
litellm_params["_langfuse_masking_function"] = masking_fn
litellm_params["metadata"] = metadata
return litellm_params
if not is_silent_experiment_call(litellm_params):
return litellm_params
return {
name: (
{key: value for key, value in bucket.items() if key != SILENT_EXPERIMENT_METADATA_KEY}
if name in _SILENT_EXPERIMENT_BUCKETS and _carries_silent_experiment_marker(bucket)
else bucket
)
for name, bucket in litellm_params.items()
}
def _carries_silent_experiment_marker(bucket: object) -> bool:
return isinstance(bucket, dict) and bucket.get(SILENT_EXPERIMENT_METADATA_KEY) is SilentExperimentMarker.MIRRORED
def is_silent_experiment_call(litellm_params: Mapping[str, object]) -> bool:
return any(_carries_silent_experiment_marker(litellm_params.get(name)) for name in _SILENT_EXPERIMENT_BUCKETS)
# integration helper function

View file

@ -3744,6 +3744,7 @@ class SpendLogsMetadata(TypedDict):
internal_call_origin: InternalCallOrigin | None
billed_retry_attempt: ReadOnly[NotRequired[bool | None]]
litellm_roi_estimator: ReadOnly[NotRequired[bool | None]]
is_silent_experiment: ReadOnly[NotRequired[bool | None]]
guardrail_information: list[StandardLoggingGuardrailInformation] | None
eval_information: Any | None
status: StandardLoggingPayloadStatus

View file

@ -80,7 +80,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import (
SETTLES_BACKGROUND_RESPONSE_KWARG,
BackgroundResponseSettlement,
)
from litellm.types.router import DeploymentTypedDict, LiteLLM_Params
from litellm.types.router import DeploymentTypedDict, LiteLLM_Params, SilentExperimentFlag
from litellm.types.utils import (
BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,
PROMPT_CARRYING_GUARDRAIL_FIELDS,
@ -194,6 +194,7 @@ _STAMPED_METADATA_KEYS: Final = frozenset(
"used_client_oauth_token",
KUBERNETES_POD_ROUTING_KEY,
"litellm_roi_estimator",
"is_silent_experiment",
BILLED_RETRY_ATTEMPT_METADATA_KEY,
"user_api_key_user_email",
)
@ -227,6 +228,7 @@ def _get_spend_logs_metadata(
azure_spillover: AzureSpillover | None = None,
used_client_oauth_token: bool | None = None,
kubernetes_pod_routing: StandardLoggingKubernetesPodRouting | None = None,
is_silent_experiment: bool = False,
) -> SpendLogsMetadata:
if metadata is None:
return SpendLogsMetadata(
@ -294,6 +296,7 @@ def _get_spend_logs_metadata(
used_client_oauth_token=used_client_oauth_token,
kubernetes_pod_routing=kubernetes_pod_routing,
litellm_roi_estimator=metadata.get("litellm_roi_estimator") is True,
**(SilentExperimentFlag(is_silent_experiment=True) if is_silent_experiment else SilentExperimentFlag()),
billed_retry_attempt=billed_retry_attempt,
user_api_key_user_email=_stored_user_email(metadata),
)
@ -738,6 +741,11 @@ def get_logging_payload(
# clean up litellm metadata
clean_metadata = _get_spend_logs_metadata(
persisted_metadata,
is_silent_experiment=(
standard_logging_payload is not None
and "is_silent_experiment" in standard_logging_payload["metadata"]
and standard_logging_payload["metadata"]["is_silent_experiment"]
),
applied_guardrails=(
standard_logging_payload["metadata"].get("applied_guardrails", None)
if standard_logging_payload is not None

View file

@ -79,6 +79,7 @@ from litellm.constants import (
ROUTING_REQUEST_TAGS_METADATA_KEY,
RUNTIME_UPDATABLE_ROUTER_SETTINGS,
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
SILENT_EXPERIMENT_METADATA_KEY,
)
from litellm.integrations.custom_guardrail import CustomGuardrail, is_guardrail_intervention
from litellm.integrations.custom_logger import CustomLogger
@ -413,6 +414,7 @@ from litellm.types.router import (
RoutingPlugin,
RoutingStrategy,
SearchToolTypedDict,
SilentExperimentMarker,
TaggedPreRoutingStrategy,
holds_secret_pointer,
)
@ -3249,6 +3251,12 @@ class Router:
silent_kwargs["metadata"].pop("litellm_parent_otel_span", None)
silent_kwargs["metadata"]["is_silent_experiment"] = True
silent_kwargs["metadata"][SILENT_EXPERIMENT_METADATA_KEY] = SilentExperimentMarker.MIRRORED
if "litellm_metadata" in silent_kwargs and isinstance(silent_kwargs["litellm_metadata"], dict):
silent_kwargs["litellm_metadata"] = {
**silent_kwargs["litellm_metadata"],
SILENT_EXPERIMENT_METADATA_KEY: SilentExperimentMarker.MIRRORED,
}
# Pop logging objects and call IDs to ensure a fresh logging context
# This prevents collisions in the Proxy's database (spend_logs)

View file

@ -66,6 +66,14 @@ from .utils import (
AllowedModelRegion: TypeAlias = Literal["eu", "us"]
class SilentExperimentMarker(str, enum.Enum):
MIRRORED = "mirrored"
class SilentExperimentFlag(TypedDict, total=False):
is_silent_experiment: ReadOnly[bool]
class FallbackSkipReason(str, enum.Enum):
NOT_AUTHORIZED = "not_authorized"
OVER_BUDGET = "over_budget"

View file

@ -3372,6 +3372,7 @@ class StandardLoggingMetadata(StandardLoggingUserAPIKeyMetadata):
team_alias: str | None
team_id: str | None
used_client_oauth_token: ReadOnly[bool | None]
is_silent_experiment: ReadOnly[NotRequired[bool]]
class AzureSpillover(TypedDict):

View file

@ -830,6 +830,31 @@ class TestLangfuseUsageDetails(unittest.TestCase):
assert generation_metadata["litellm_response_cost"] == 0.25
assert generation_metadata["api_base"] == "https://real-api-base"
def test_caller_sent_is_silent_experiment_cannot_hide_the_mirrored_call_flag(self):
payload = self._build_standard_logging_payload(trace_id="canary-trace-id")
payload["metadata"]["is_silent_experiment"] = True
payload["metadata"]["requester_metadata"] = {"is_silent_experiment": False}
kwargs = self._build_langfuse_kwargs(payload)
metadata = self._canary_request_metadata()
self.use_real_langfuse_client()
self.logger._log_langfuse_v2(
user_id="user-1",
metadata=metadata,
litellm_params={"metadata": metadata},
output=None,
start_time=datetime.datetime(2024, 1, 1, 12, 0, 0),
end_time=datetime.datetime(2024, 1, 1, 12, 0, 1),
kwargs=kwargs,
optional_params={},
input=None,
response_obj=None,
level="INFO",
litellm_call_id="canary-call-id",
)
assert self.exported_generation_metadata()["is_silent_experiment"] is True
def test_generation_metadata_carries_the_call_id_and_response_id(self):
"""
v2's generation id was ``time-<hh-mm-ss-us>_<response id>``, so a generation could

View file

@ -11857,7 +11857,7 @@ def test_get_additional_headers():
def all_fields_present(standard_logging_metadata: StandardLoggingMetadata):
for field in StandardLoggingMetadata.__annotations__.keys():
for field in StandardLoggingMetadata.__required_keys__:
assert field in standard_logging_metadata

View file

@ -30,7 +30,7 @@ from starlette.datastructures import UploadFile as StarletteUploadFile
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS
from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS, SILENT_EXPERIMENT_METADATA_KEY
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@ -63,6 +63,7 @@ from litellm.proxy.pass_through_endpoints.success_handler import (
from litellm.proxy.route_llm_request import ProxyModelNotFoundError
from litellm.types import utils as types_utils
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.router import SilentExperimentMarker
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY,
LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY,
@ -10339,6 +10340,43 @@ def test_init_kwargs_no_authenticated_identity_field_is_client_settable(mock_req
survived: Final = {key: metadata.get(key) for key in expected if metadata.get(key) != expected[key]}
assert survived == {}, f"client-supplied values survived for: {sorted(survived)}"
@pytest.mark.parametrize("planted", [True, SilentExperimentMarker.MIRRORED.value])
def test_init_kwargs_client_metadata_cannot_mark_a_pass_through_call_as_a_silent_experiment(
planted: object, mock_user_api_key_dict: UserAPIKeyAuth
) -> None:
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
started: Final = datetime(2026, 1, 15, 12, 0, 0)
result: Final = HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint(
request=Request({"type": "http", "method": "POST", "path": "/test", "headers": [], "query_string": b""}),
user_api_key_dict=mock_user_api_key_dict,
passthrough_logging_payload=PassthroughStandardLoggingPayload(url="https://test.com", request_body={}),
litellm_call_id="test-call-id",
logging_obj=LiteLLMLoggingObj(
model="test-model",
messages=[],
stream=False,
call_type="test-call-type",
start_time=started,
litellm_call_id="test-call-id",
function_id="test-function-id",
),
_parsed_body={
"litellm_metadata": {SILENT_EXPERIMENT_METADATA_KEY: planted, "client_key": "kept"},
"metadata": {SILENT_EXPERIMENT_METADATA_KEY: planted},
},
)
row: Final = get_logging_payload(
kwargs={"model": "test-model", "litellm_params": result["litellm_params"]},
response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]),
start_time=started,
end_time=started,
)
assert "is_silent_experiment" not in json.loads(row["metadata"])
assert result["litellm_params"]["metadata"]["client_key"] == "kept"
@pytest.mark.asyncio
async def test_direct_passthrough_keeps_mapped_baggage_tags_outside_upstream_body():
from litellm.proxy.common_utils.http_parsing_utils import read_request_body

View file

@ -6830,7 +6830,7 @@ def test_spend_logs_payload(model_id: str | None):
)
assert len(payload["request_id"]) > 0
expected_metadata_keys: Final = SpendLogsMetadata.__annotations__.keys()
expected_metadata_keys: Final = SpendLogsMetadata.__annotations__.keys() - {"is_silent_experiment"}
assert "metadata" in payload
assert isinstance(payload["metadata"], str)

View file

@ -1,4 +1,5 @@
import asyncio
import datetime
import json
import time
from collections.abc import Callable, Mapping
@ -11,11 +12,13 @@ import pytest
import respx
import litellm
from litellm.constants import SILENT_EXPERIMENT_METADATA_KEY
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.router import Router
from litellm.router import _silent_experiment_kwargs_snapshot
from litellm.router import _silent_experiment_targets
from litellm.types.router import SilentExperimentMarker
class _RecordingLogger(CustomLogger):
@ -263,6 +266,138 @@ async def test_multiple_shadow_targets_fan_out_async(recording_logger):
assert primary_successes[0]["litellm_params"]["metadata"]["model_group"] == "primary-model"
@pytest.mark.asyncio
async def test_standard_logging_payload_marks_only_the_mirrored_calls_as_silent_experiments(
recording_logger: _RecordingLogger,
) -> None:
router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"]))
response: Final = await router.acompletion(
model="primary-model",
messages=[{"role": "user", "content": "hi"}],
mock_response="pong",
)
assert response.choices[0].message.content == "pong"
await _wait_for_all_successes(recording_logger, expected=3)
payloads: Final = [call["standard_logging_object"] for call in recording_logger.success_kwargs]
assert sorted(payload["model_group"] for payload in payloads) == ["primary-model", "shadow-a", "shadow-b"]
metadata_by_group: Final = {payload["model_group"]: payload["metadata"] for payload in payloads}
assert metadata_by_group["shadow-a"]["is_silent_experiment"] is True
assert metadata_by_group["shadow-b"]["is_silent_experiment"] is True
assert "is_silent_experiment" not in metadata_by_group["primary-model"]
_SPEND_ROW_TIME: Final = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
def _spend_row_is_silent_experiment(call: Mapping[str, object]) -> object:
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
row: Final = get_logging_payload(
kwargs=dict(call),
response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]),
start_time=_SPEND_ROW_TIME,
end_time=_SPEND_ROW_TIME,
)
return json.loads(row["metadata"]).get("is_silent_experiment")
def _through_the_spend_collector(call: Mapping[str, object]) -> Mapping[str, object]:
from litellm.proxy.spend_tracking.spend_event import (
SpendEvent,
build_spend_event,
decode_spend_event,
spend_event_callback_args,
)
line: Final = build_spend_event(
call,
litellm.ModelResponse(id="chatcmpl-test", choices=[]),
_SPEND_ROW_TIME,
_SPEND_ROW_TIME,
store_bodies=False,
)
assert isinstance(line, bytes)
event: Final = decode_spend_event(line)
assert isinstance(event, SpendEvent)
return spend_event_callback_args(event).kwargs
@pytest.mark.asyncio
@pytest.mark.parametrize(
"spend_kwargs",
[pytest.param(lambda call: call, id="in_process"), pytest.param(_through_the_spend_collector, id="collector")],
)
async def test_spend_rows_mark_only_the_mirrored_calls_when_the_caller_sends_litellm_metadata(
recording_logger: _RecordingLogger,
spend_kwargs: Callable[[Mapping[str, object]], Mapping[str, object]],
) -> None:
router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"]))
await router.acompletion(
model="primary-model",
messages=[{"role": "user", "content": "hi"}],
mock_response="pong",
litellm_metadata={"user_api_key": "test-key"},
)
await _wait_for_all_successes(recording_logger, expected=3)
silent_by_group: Final = {
call["standard_logging_object"]["model_group"]: _spend_row_is_silent_experiment(spend_kwargs(call))
for call in recording_logger.success_kwargs
}
assert silent_by_group == {"primary-model": None, "shadow-a": True, "shadow-b": True}
@pytest.mark.asyncio
async def test_logging_callbacks_never_receive_the_router_mirror_marker(recording_logger: _RecordingLogger) -> None:
router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"]))
await router.acompletion(
model="primary-model",
messages=[{"role": "user", "content": "hi"}],
mock_response="pong",
litellm_metadata={"user_api_key": "test-key"},
)
await _wait_for_all_successes(recording_logger, expected=3)
request_metadata_keys: Final = {
call["standard_logging_object"]["model_group"]: {
*call["litellm_params"]["metadata"],
*call["litellm_params"]["litellm_metadata"],
}
for call in recording_logger.success_kwargs
}
assert sorted(request_metadata_keys) == ["primary-model", "shadow-a", "shadow-b"]
assert all(SILENT_EXPERIMENT_METADATA_KEY not in keys for keys in request_metadata_keys.values())
assert all("is_silent_experiment" in request_metadata_keys[group] for group in ("shadow-a", "shadow-b"))
@pytest.mark.asyncio
@pytest.mark.parametrize(
("bucket", "planted"),
[
("metadata", {"is_silent_experiment": True}),
("metadata", {SILENT_EXPERIMENT_METADATA_KEY: SilentExperimentMarker.MIRRORED.value}),
("litellm_metadata", {SILENT_EXPERIMENT_METADATA_KEY: SilentExperimentMarker.MIRRORED.value}),
("litellm_metadata", {SILENT_EXPERIMENT_METADATA_KEY: True}),
],
)
async def test_a_caller_sent_silent_experiment_flag_does_not_mark_a_real_call_as_mirrored(
recording_logger: _RecordingLogger, bucket: str, planted: dict[str, object]
) -> None:
router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"]))
await router.acompletion(
model="shadow-b",
messages=[{"role": "user", "content": "hi"}],
mock_response="pong",
**{bucket: planted},
)
await _wait_for_all_successes(recording_logger, expected=1)
call: Final = recording_logger.success_kwargs[0]
assert "is_silent_experiment" not in call["standard_logging_object"]["metadata"]
assert _spend_row_is_silent_experiment(call) is None
def test_multiple_shadow_targets_fan_out_sync(recording_logger):
router = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"]))
response = router.completion(