scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-30 06:43:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 249

fix(logging): mark mirrored calls with a router-only enum value so no client route can set is_silent_experiment

Browse source 
The router stamps SilentExperimentMarker.MIRRORED, which client JSON can never produce, so batches, Responses WebSocket frames and pass-through need no strip. Langfuse layers the flag after its requester-metadata dedupe.
This commit is contained in:
mateo-berri 2026-10-20 21:07:45 -07:00
parent 1fb6b63a93
commit f71b2cdfc9
12 changed files with 81 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
litellm/integrations/langfuse/langfuse.py
View file

@ -832,6 +832,12 @@ class LangFuseLogger:
("vertex_location", vertex_location, bool(vertex_location)),
("aws_region_name", aws_region_name, bool(aws_region_name)),
("cache_hit", kwargs.get("cache_hit") or False, "cache_hit" in kwargs),
(
"is_silent_experiment",
"is_silent_experiment" in allowlisted_metadata
and allowlisted_metadata["is_silent_experiment"] is True,
"is_silent_experiment" in allowlisted_metadata,
),
)
enrichments: Final[Mapping[str, object]] = {
key: value for key, value, include in candidate_enrichments if include

4
litellm/litellm_core_utils/litellm_logging.py
View file

@ -155,7 +155,7 @@ from litellm.types.llms.openai import (
from litellm.types.mcp import MCPPostCallResponseObject
from litellm.types.prompts.init_prompts import PromptSpec
from litellm.types.rerank import RerankResponse
from litellm.types.router import RouterRateLimitError
from litellm.types.router import RouterRateLimitError, SilentExperimentMarker
from litellm.types.utils import (
DEPLOYMENT_SCOPED_PRICING_FIELDS,
CachingDetails,
@ -6440,7 +6440,7 @@ class StandardLoggingPayloadSetup:
is_silent_experiment=(
metadata is not None
and SILENT_EXPERIMENT_METADATA_KEY in metadata
and metadata[SILENT_EXPERIMENT_METADATA_KEY] is True
and metadata[SILENT_EXPERIMENT_METADATA_KEY] is SilentExperimentMarker.MIRRORED
),
)
if isinstance(metadata, dict):

2
litellm/proxy/litellm_pre_call_utils.py
View file

@ -43,7 +43,6 @@ from litellm.constants import (
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
SESSION_ID_GENERATED_METADATA_KEY,
SESSION_ID_OMITTED_METADATA_KEY,
SILENT_EXPERIMENT_METADATA_KEY,
TYPESAFE_DANGER_ZONE_HEADER_PREFIX,
VECTOR_STORE_CALLER_METADATA_KWARG,
X_LITELLM_DISABLE_CALLBACKS,
@ -475,7 +474,6 @@ _ROUTER_RESERVED_METADATA_FIELDS: Final = frozenset(
"request_retry_count",
CLIENT_OUTPUT_CEILING_METADATA_KEY,
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY,
SILENT_EXPERIMENT_METADATA_KEY,
}
)
_ALLOW_CLIENT_PRICING_OVERRIDE_METADATA_KEY: Final = "allow_client_pricing_override"

7
litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
View file

@ -113,7 +113,6 @@ from litellm.proxy.common_utils.sse_keepalive import (
wrap_passthrough_sse_bytes_with_keepalive_pings,
)
from litellm.proxy.litellm_pre_call_utils import ( # noqa: F401 # legacy module exports
_ROUTER_RESERVED_METADATA_FIELDS, # pyright: ignore[reportPrivateUsage] # pass-through applies the proxy's router-owned key strip
LiteLLMProxyRequestSetup,
_get_dynamic_logging_metadata, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
_key_or_team_allows_client_pricing_override, # pyright: ignore[reportPrivateUsage] # reuse the proxy's pricing trust policy
@ -166,11 +165,7 @@ _PRICING_MODEL_OVERRIDE_PARAMS: Final = frozenset(("base_model",))
def _is_mergeable_client_metadata(key: str, value: object) -> bool:
return (
not key.startswith("user_api_key_")
and key not in _ROUTER_RESERVED_METADATA_FIELDS
and (key != "model_info" or isinstance(value, Mapping))
)
return not key.startswith("user_api_key_") and (key != "model_info" or isinstance(value, Mapping))
# Global registry to track registered pass-through routes and prevent memory leaks

4
litellm/proxy/spend_tracking/spend_tracking_utils.py
View file

@ -75,7 +75,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import (
SETTLES_BACKGROUND_RESPONSE_KWARG,
BackgroundResponseSettlement,
)
from litellm.types.router import DeploymentTypedDict, LiteLLM_Params
from litellm.types.router import DeploymentTypedDict, LiteLLM_Params, SilentExperimentMarker
from litellm.types.utils import (
BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,
PROMPT_CARRYING_GUARDRAIL_FIELDS,
@ -292,7 +292,7 @@ def _get_spend_logs_metadata(
kubernetes_pod_routing=kubernetes_pod_routing,
litellm_roi_estimator=metadata.get("litellm_roi_estimator") is True,
is_silent_experiment=SILENT_EXPERIMENT_METADATA_KEY in metadata
and metadata[SILENT_EXPERIMENT_METADATA_KEY] is True,
and metadata[SILENT_EXPERIMENT_METADATA_KEY] is SilentExperimentMarker.MIRRORED,
billed_retry_attempt=billed_retry_attempt,
user_api_key_user_email=_stored_user_email(metadata),
)

5
litellm/router.py
View file

@ -409,6 +409,7 @@ from litellm.types.router import (
RoutingPlugin,
RoutingStrategy,
SearchToolTypedDict,
SilentExperimentMarker,
TaggedPreRoutingStrategy,
holds_secret_pointer,
)
@ -3125,11 +3126,11 @@ class Router:
silent_kwargs["metadata"].pop("litellm_parent_otel_span", None)
silent_kwargs["metadata"]["is_silent_experiment"] = True
silent_kwargs["metadata"][SILENT_EXPERIMENT_METADATA_KEY] = True
silent_kwargs["metadata"][SILENT_EXPERIMENT_METADATA_KEY] = SilentExperimentMarker.MIRRORED
if "litellm_metadata" in silent_kwargs and isinstance(silent_kwargs["litellm_metadata"], dict):
silent_kwargs["litellm_metadata"] = {
**silent_kwargs["litellm_metadata"],
SILENT_EXPERIMENT_METADATA_KEY: True,
SILENT_EXPERIMENT_METADATA_KEY: SilentExperimentMarker.MIRRORED,
}
# Pop logging objects and call IDs to ensure a fresh logging context

4
litellm/types/router.py
View file

@ -66,6 +66,10 @@ from .utils import (
AllowedModelRegion: TypeAlias = Literal["eu", "us"]
class SilentExperimentMarker(str, enum.Enum):
MIRRORED = "mirrored"
class FallbackSkipReason(str, enum.Enum):
NOT_AUTHORIZED = "not_authorized"
OVER_BUDGET = "over_budget"

30
tests/unit/integrations/test_langfuse.py
View file

@ -830,6 +830,36 @@ class TestLangfuseUsageDetails(unittest.TestCase):
assert generation_metadata["litellm_response_cost"] == 0.25
assert generation_metadata["api_base"] == "https://real-api-base"
def test_caller_sent_is_silent_experiment_cannot_hide_the_mirrored_call_flag(self):
payload = self._build_standard_logging_payload(trace_id="canary-trace-id")
payload["metadata"]["is_silent_experiment"] = True
payload["metadata"]["requester_metadata"] = {"is_silent_experiment": False}
kwargs = self._build_langfuse_kwargs(payload)
metadata = self._canary_request_metadata()
self.use_real_langfuse_client()
with patch(
"litellm.integrations.langfuse.langfuse._add_prompt_to_generation_params",
side_effect=lambda generation_params, **kw: generation_params,
create=True,
):
self.logger._log_langfuse_v2(
user_id="user-1",
metadata=metadata,
litellm_params={"metadata": metadata},
output=None,
start_time=datetime.datetime(2024, 1, 1, 12, 0, 0),
end_time=datetime.datetime(2024, 1, 1, 12, 0, 1),
kwargs=kwargs,
optional_params={},
input=None,
response_obj=None,
level="INFO",
litellm_call_id="canary-call-id",
)
assert self.exported_generation_metadata()["is_silent_experiment"] is True
def test_generation_metadata_carries_the_call_id_and_response_id(self):
"""
v2's generation id was ``time-<hh-mm-ss-us>_<response id>``, so a generation could

12
tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py
View file

@ -63,6 +63,7 @@ from litellm.proxy.pass_through_endpoints.success_handler import (
from litellm.proxy.route_llm_request import ProxyModelNotFoundError
from litellm.types import utils as types_utils
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.router import SilentExperimentMarker
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY,
LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY,
@ -10339,14 +10340,15 @@ def test_init_kwargs_no_authenticated_identity_field_is_client_settable(mock_req
assert survived == {}, f"client-supplied values survived for: {sorted(survived)}"
@pytest.mark.parametrize("planted", [True, SilentExperimentMarker.MIRRORED.value])
def test_init_kwargs_client_metadata_cannot_mark_a_pass_through_call_as_a_silent_experiment(
mock_request, mock_user_api_key_dict
):
planted: object, mock_user_api_key_dict: UserAPIKeyAuth
) -> None:
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
started: Final = datetime(2026, 1, 15, 12, 0, 0)
result: Final = HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint(
request=mock_request(),
request=Request({"type": "http", "method": "POST", "path": "/test", "headers": [], "query_string": b""}),
user_api_key_dict=mock_user_api_key_dict,
passthrough_logging_payload=PassthroughStandardLoggingPayload(url="https://test.com", request_body={}),
litellm_call_id="test-call-id",
@ -10360,8 +10362,8 @@ def test_init_kwargs_client_metadata_cannot_mark_a_pass_through_call_as_a_silent
function_id="test-function-id",
),
_parsed_body={
"litellm_metadata": {SILENT_EXPERIMENT_METADATA_KEY: True, "client_key": "kept"},
"metadata": {SILENT_EXPERIMENT_METADATA_KEY: True},
"litellm_metadata": {SILENT_EXPERIMENT_METADATA_KEY: planted, "client_key": "kept"},
"metadata": {SILENT_EXPERIMENT_METADATA_KEY: planted},
},
)

6
tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py
View file

@ -17,6 +17,7 @@ import litellm.proxy.spend_tracking.spend_tracking_utils as spend_tracking_utils
from litellm.constants import KUBERNETES_POD_ROUTING_KEY, LITELLM_TRUNCATED_PAYLOAD_FIELD, LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, LITTELM_CLI_SERVICE_ACCOUNT_NAME, LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, MAX_SPEND_LOG_MODEL_NAME_LENGTH, REDACTED_BY_LITELM_STRING, SESSION_ID_GENERATED_METADATA_KEY, SESSION_ID_OMITTED_METADATA_KEY, UNKNOWN_MODEL_SPEND_LOG_MODEL
from litellm.constants import CALLER_SESSION_ID_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY, MANAGED_BACKGROUND_RESPONSE_METADATA_KEY
from litellm.constants import SILENT_EXPERIMENT_METADATA_KEY
from litellm.types.router import SilentExperimentMarker
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
from litellm.llms.base_llm.ocr.transformation import OCRResponse, OCRUsageInfo
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
@ -167,8 +168,9 @@ def test_get_logging_payload_includes_kubernetes_pod_routing(
@pytest.mark.parametrize(
("silent_experiment_metadata", "expected"),
[
({SILENT_EXPERIMENT_METADATA_KEY: True}, True),
({SILENT_EXPERIMENT_METADATA_KEY: "true"}, False),
({SILENT_EXPERIMENT_METADATA_KEY: SilentExperimentMarker.MIRRORED}, True),
({SILENT_EXPERIMENT_METADATA_KEY: SilentExperimentMarker.MIRRORED.value}, False),
({SILENT_EXPERIMENT_METADATA_KEY: True}, False),
({"is_silent_experiment": True}, False),
({}, False),
],

3
tests/unit/proxy/test_litellm_pre_call_utils.py
View file

@ -23,7 +23,6 @@ from litellm.constants import (
SERVER_STREAMING_CLASSIFICATION_MARKER,
SESSION_ID_GENERATED_METADATA_KEY,
SESSION_ID_OMITTED_METADATA_KEY,
SILENT_EXPERIMENT_METADATA_KEY,
)
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs
from litellm.litellm_core_utils.get_provider_specific_headers import (
@ -8284,7 +8283,6 @@ _PLANTED_STAMPS = {
"previous_models": [{"exception_type": "Timeout", "deployment_id": "spoofed", "attempted_retries": 101}],
"_client_output_ceiling": {"api_base": "https://attacker.example"},
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: 10**9,
SILENT_EXPERIMENT_METADATA_KEY: True,
"client_key": "client_value",
}
@ -8314,7 +8312,6 @@ async def test_add_litellm_data_to_request_strips_router_reserved_stamps_from_bo
assert "litellm_metadata" not in updated
assert "attempted_fallbacks" not in updated["metadata"]
assert SILENT_EXPERIMENT_METADATA_KEY not in updated["metadata"]
assert "original_model_group" not in updated["metadata"]
assert "_client_output_ceiling" not in updated["metadata"]
assert "request_retry_count" not in updated["metadata"]

27
tests/unit/test_router_silent_experiment.py
View file

@ -12,11 +12,13 @@ import pytest
import respx
import litellm
from litellm.constants import SILENT_EXPERIMENT_METADATA_KEY
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.router import Router
from litellm.router import _silent_experiment_kwargs_snapshot
from litellm.router import _silent_experiment_targets
from litellm.types.router import SilentExperimentMarker
class _RecordingLogger(CustomLogger):
@ -265,7 +267,9 @@ async def test_multiple_shadow_targets_fan_out_async(recording_logger):
@pytest.mark.asyncio
async def test_standard_logging_payload_marks_only_the_mirrored_calls_as_silent_experiments(recording_logger):
async def test_standard_logging_payload_marks_only_the_mirrored_calls_as_silent_experiments(
recording_logger: _RecordingLogger,
) -> None:
router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"]))
response: Final = await router.acompletion(
model="primary-model",
@ -286,7 +290,7 @@ async def test_standard_logging_payload_marks_only_the_mirrored_calls_as_silent_
def _spend_row_is_silent_experiment(call: dict[str, object]) -> object:
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
now: Final = datetime.datetime.now(datetime.timezone.utc)
now: Final = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
row: Final = get_logging_payload(
kwargs=call,
response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]),
@ -297,7 +301,9 @@ def _spend_row_is_silent_experiment(call: dict[str, object]) -> object:
@pytest.mark.asyncio
async def test_spend_rows_mark_only_the_mirrored_calls_when_the_caller_sends_litellm_metadata(recording_logger):
async def test_spend_rows_mark_only_the_mirrored_calls_when_the_caller_sends_litellm_metadata(
recording_logger: _RecordingLogger,
) -> None:
router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"]))
await router.acompletion(
model="primary-model",
@ -315,13 +321,24 @@ async def test_spend_rows_mark_only_the_mirrored_calls_when_the_caller_sends_lit
@pytest.mark.asyncio
async def test_a_caller_sent_is_silent_experiment_does_not_mark_a_real_call_as_mirrored(recording_logger):
@pytest.mark.parametrize(
("bucket", "planted"),
[
("metadata", {"is_silent_experiment": True}),
("metadata", {SILENT_EXPERIMENT_METADATA_KEY: SilentExperimentMarker.MIRRORED.value}),
("litellm_metadata", {SILENT_EXPERIMENT_METADATA_KEY: SilentExperimentMarker.MIRRORED.value}),
("litellm_metadata", {SILENT_EXPERIMENT_METADATA_KEY: True}),
],
)
async def test_a_caller_sent_silent_experiment_flag_does_not_mark_a_real_call_as_mirrored(
recording_logger: _RecordingLogger, bucket: str, planted: dict[str, object]
) -> None:
router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"]))
await router.acompletion(
model="shadow-b",
messages=[{"role": "user", "content": "hi"}],
mock_response="pong",
metadata={"is_silent_experiment": True},
**{bucket: planted},
)
await _wait_for_all_successes(recording_logger, expected=1)