mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge bc14914095 into 3b8ad37296
This commit is contained in:
commit
9e9aa1e3b6
13 changed files with 265 additions and 8 deletions
|
|
@ -1660,6 +1660,7 @@ ROUTING_REQUEST_TAGS_METADATA_KEY: Final = "_routing_request_tags"
|
|||
BAGGAGE_MAPPED_TAGS_METADATA_KEY: Final = "_baggage_mapped_tags"
|
||||
VECTOR_STORE_CALLER_METADATA_KWARG: Final = "litellm_vector_store_caller_metadata"
|
||||
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted_tokens"
|
||||
SILENT_EXPERIMENT_METADATA_KEY: Final = "_litellm_silent_experiment"
|
||||
INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
|
||||
BILLED_RETRY_ATTEMPT_METADATA_KEY: Final = "billed_retry_attempt"
|
||||
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response"
|
||||
|
|
|
|||
|
|
@ -833,6 +833,12 @@ class LangFuseLogger:
|
|||
("vertex_location", vertex_location, bool(vertex_location)),
|
||||
("aws_region_name", aws_region_name, bool(aws_region_name)),
|
||||
("cache_hit", kwargs.get("cache_hit") or False, "cache_hit" in kwargs),
|
||||
(
|
||||
"is_silent_experiment",
|
||||
"is_silent_experiment" in allowlisted_metadata
|
||||
and allowlisted_metadata["is_silent_experiment"] is True,
|
||||
"is_silent_experiment" in allowlisted_metadata,
|
||||
),
|
||||
)
|
||||
enrichments: Final[Mapping[str, object]] = {
|
||||
key: value for key, value, include in candidate_enrichments if include
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ from litellm.constants import (
|
|||
LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL,
|
||||
PROVIDER_REQUEST_ID_HEADERS,
|
||||
REDACTED_BY_LITELLM,
|
||||
SILENT_EXPERIMENT_METADATA_KEY,
|
||||
)
|
||||
from litellm.cost_calculator import (
|
||||
RealtimeAPITokenUsageProcessor,
|
||||
|
|
@ -157,7 +158,7 @@ from litellm.types.llms.openai import (
|
|||
from litellm.types.mcp import MCPPostCallResponseObject
|
||||
from litellm.types.prompts.init_prompts import PromptSpec
|
||||
from litellm.types.rerank import RerankResponse
|
||||
from litellm.types.router import RouterRateLimitError
|
||||
from litellm.types.router import RouterRateLimitError, SilentExperimentFlag, SilentExperimentMarker
|
||||
from litellm.types.utils import (
|
||||
DEPLOYMENT_SCOPED_PRICING_FIELDS,
|
||||
CachingDetails,
|
||||
|
|
@ -309,8 +310,9 @@ else:
|
|||
_in_memory_loggers: Final[list[CustomLogger]] = []
|
||||
|
||||
_STANDARD_LOGGING_METADATA_RESOLVED_KEYS: Final[frozenset[str]] = frozenset(
|
||||
("used_client_oauth_token", "usage_object", KUBERNETES_POD_ROUTING_KEY)
|
||||
("used_client_oauth_token", "usage_object", KUBERNETES_POD_ROUTING_KEY, "is_silent_experiment")
|
||||
)
|
||||
_SILENT_EXPERIMENT_BUCKETS: Final = ("metadata", "litellm_metadata")
|
||||
_STANDARD_LOGGING_METADATA_KEYS: Final[frozenset[str]] = (
|
||||
frozenset(StandardLoggingMetadata.__annotations__.keys()) - _STANDARD_LOGGING_METADATA_RESOLVED_KEYS
|
||||
)
|
||||
|
|
@ -803,6 +805,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
litellm_params = scrub_sensitive_keys_in_metadata(litellm_params)
|
||||
|
||||
self.litellm_params = litellm_params
|
||||
self.is_silent_experiment = False
|
||||
|
||||
# Initialize cost breakdown field
|
||||
self.cost_breakdown: CostBreakdown | None = None
|
||||
|
|
@ -1118,6 +1121,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
if model is not None:
|
||||
self.model = model
|
||||
self.user = user
|
||||
self.is_silent_experiment = self.is_silent_experiment or is_silent_experiment_call(litellm_params)
|
||||
self.litellm_params = {
|
||||
**self.litellm_params,
|
||||
**scrub_sensitive_keys_in_metadata(litellm_params),
|
||||
|
|
@ -6523,6 +6527,7 @@ class StandardLoggingPayloadSetup:
|
|||
start_time: dt_object | None = None,
|
||||
response_id: str | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
is_silent_experiment: bool = False,
|
||||
) -> StandardLoggingMetadata:
|
||||
"""
|
||||
Clean and filter the metadata dictionary to include only the specified keys in StandardLoggingMetadata.
|
||||
|
|
@ -6596,6 +6601,7 @@ class StandardLoggingPayloadSetup:
|
|||
proxy_stamped_used_client_oauth_token(metadata, litellm_params),
|
||||
custom_llm_provider,
|
||||
),
|
||||
**(SilentExperimentFlag(is_silent_experiment=True) if is_silent_experiment else SilentExperimentFlag()),
|
||||
)
|
||||
if isinstance(metadata, dict):
|
||||
for key in metadata.keys() & _STANDARD_LOGGING_METADATA_KEYS:
|
||||
|
|
@ -7398,6 +7404,7 @@ def get_standard_logging_object_payload(
|
|||
start_time=start_time,
|
||||
response_id=id,
|
||||
custom_llm_provider=selected_provider if isinstance(selected_provider, str) else None,
|
||||
is_silent_experiment=logging_obj.is_silent_experiment is True,
|
||||
)
|
||||
_request_body: Final = proxy_server_request.get("body", {})
|
||||
end_user_id: Final = clean_metadata["user_api_key_end_user_id"] or _request_body.get(
|
||||
|
|
@ -7692,7 +7699,9 @@ def get_standard_logging_metadata(
|
|||
return clean_metadata
|
||||
|
||||
|
||||
def scrub_sensitive_keys_in_metadata(litellm_params: dict | None):
|
||||
def scrub_sensitive_keys_in_metadata(
|
||||
litellm_params: dict[str, Any] | None, # mutable-ok: the langfuse masking function is moved onto these params
|
||||
) -> dict[str, Any]: # mutable-ok: loggers write into the returned params and their metadata
|
||||
if litellm_params is None:
|
||||
litellm_params = {}
|
||||
|
||||
|
|
@ -7707,7 +7716,24 @@ def scrub_sensitive_keys_in_metadata(litellm_params: dict | None):
|
|||
litellm_params["_langfuse_masking_function"] = masking_fn
|
||||
litellm_params["metadata"] = metadata
|
||||
|
||||
return litellm_params
|
||||
if not is_silent_experiment_call(litellm_params):
|
||||
return litellm_params
|
||||
return {
|
||||
name: (
|
||||
{key: value for key, value in bucket.items() if key != SILENT_EXPERIMENT_METADATA_KEY}
|
||||
if name in _SILENT_EXPERIMENT_BUCKETS and _carries_silent_experiment_marker(bucket)
|
||||
else bucket
|
||||
)
|
||||
for name, bucket in litellm_params.items()
|
||||
}
|
||||
|
||||
|
||||
def _carries_silent_experiment_marker(bucket: object) -> bool:
|
||||
return isinstance(bucket, dict) and bucket.get(SILENT_EXPERIMENT_METADATA_KEY) is SilentExperimentMarker.MIRRORED
|
||||
|
||||
|
||||
def is_silent_experiment_call(litellm_params: Mapping[str, object]) -> bool:
|
||||
return any(_carries_silent_experiment_marker(litellm_params.get(name)) for name in _SILENT_EXPERIMENT_BUCKETS)
|
||||
|
||||
|
||||
# integration helper function
|
||||
|
|
|
|||
|
|
@ -3744,6 +3744,7 @@ class SpendLogsMetadata(TypedDict):
|
|||
internal_call_origin: InternalCallOrigin | None
|
||||
billed_retry_attempt: ReadOnly[NotRequired[bool | None]]
|
||||
litellm_roi_estimator: ReadOnly[NotRequired[bool | None]]
|
||||
is_silent_experiment: ReadOnly[NotRequired[bool | None]]
|
||||
guardrail_information: list[StandardLoggingGuardrailInformation] | None
|
||||
eval_information: Any | None
|
||||
status: StandardLoggingPayloadStatus
|
||||
|
|
|
|||
|
|
@ -80,7 +80,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import (
|
|||
SETTLES_BACKGROUND_RESPONSE_KWARG,
|
||||
BackgroundResponseSettlement,
|
||||
)
|
||||
from litellm.types.router import DeploymentTypedDict, LiteLLM_Params
|
||||
from litellm.types.router import DeploymentTypedDict, LiteLLM_Params, SilentExperimentFlag
|
||||
from litellm.types.utils import (
|
||||
BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,
|
||||
PROMPT_CARRYING_GUARDRAIL_FIELDS,
|
||||
|
|
@ -194,6 +194,7 @@ _STAMPED_METADATA_KEYS: Final = frozenset(
|
|||
"used_client_oauth_token",
|
||||
KUBERNETES_POD_ROUTING_KEY,
|
||||
"litellm_roi_estimator",
|
||||
"is_silent_experiment",
|
||||
BILLED_RETRY_ATTEMPT_METADATA_KEY,
|
||||
"user_api_key_user_email",
|
||||
)
|
||||
|
|
@ -227,6 +228,7 @@ def _get_spend_logs_metadata(
|
|||
azure_spillover: AzureSpillover | None = None,
|
||||
used_client_oauth_token: bool | None = None,
|
||||
kubernetes_pod_routing: StandardLoggingKubernetesPodRouting | None = None,
|
||||
is_silent_experiment: bool = False,
|
||||
) -> SpendLogsMetadata:
|
||||
if metadata is None:
|
||||
return SpendLogsMetadata(
|
||||
|
|
@ -294,6 +296,7 @@ def _get_spend_logs_metadata(
|
|||
used_client_oauth_token=used_client_oauth_token,
|
||||
kubernetes_pod_routing=kubernetes_pod_routing,
|
||||
litellm_roi_estimator=metadata.get("litellm_roi_estimator") is True,
|
||||
**(SilentExperimentFlag(is_silent_experiment=True) if is_silent_experiment else SilentExperimentFlag()),
|
||||
billed_retry_attempt=billed_retry_attempt,
|
||||
user_api_key_user_email=_stored_user_email(metadata),
|
||||
)
|
||||
|
|
@ -738,6 +741,11 @@ def get_logging_payload(
|
|||
# clean up litellm metadata
|
||||
clean_metadata = _get_spend_logs_metadata(
|
||||
persisted_metadata,
|
||||
is_silent_experiment=(
|
||||
standard_logging_payload is not None
|
||||
and "is_silent_experiment" in standard_logging_payload["metadata"]
|
||||
and standard_logging_payload["metadata"]["is_silent_experiment"]
|
||||
),
|
||||
applied_guardrails=(
|
||||
standard_logging_payload["metadata"].get("applied_guardrails", None)
|
||||
if standard_logging_payload is not None
|
||||
|
|
|
|||
|
|
@ -79,6 +79,7 @@ from litellm.constants import (
|
|||
ROUTING_REQUEST_TAGS_METADATA_KEY,
|
||||
RUNTIME_UPDATABLE_ROUTER_SETTINGS,
|
||||
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
|
||||
SILENT_EXPERIMENT_METADATA_KEY,
|
||||
)
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail, is_guardrail_intervention
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
|
|
@ -413,6 +414,7 @@ from litellm.types.router import (
|
|||
RoutingPlugin,
|
||||
RoutingStrategy,
|
||||
SearchToolTypedDict,
|
||||
SilentExperimentMarker,
|
||||
TaggedPreRoutingStrategy,
|
||||
holds_secret_pointer,
|
||||
)
|
||||
|
|
@ -3249,6 +3251,12 @@ class Router:
|
|||
silent_kwargs["metadata"].pop("litellm_parent_otel_span", None)
|
||||
|
||||
silent_kwargs["metadata"]["is_silent_experiment"] = True
|
||||
silent_kwargs["metadata"][SILENT_EXPERIMENT_METADATA_KEY] = SilentExperimentMarker.MIRRORED
|
||||
if "litellm_metadata" in silent_kwargs and isinstance(silent_kwargs["litellm_metadata"], dict):
|
||||
silent_kwargs["litellm_metadata"] = {
|
||||
**silent_kwargs["litellm_metadata"],
|
||||
SILENT_EXPERIMENT_METADATA_KEY: SilentExperimentMarker.MIRRORED,
|
||||
}
|
||||
|
||||
# Pop logging objects and call IDs to ensure a fresh logging context
|
||||
# This prevents collisions in the Proxy's database (spend_logs)
|
||||
|
|
|
|||
|
|
@ -66,6 +66,14 @@ from .utils import (
|
|||
AllowedModelRegion: TypeAlias = Literal["eu", "us"]
|
||||
|
||||
|
||||
class SilentExperimentMarker(str, enum.Enum):
|
||||
MIRRORED = "mirrored"
|
||||
|
||||
|
||||
class SilentExperimentFlag(TypedDict, total=False):
|
||||
is_silent_experiment: ReadOnly[bool]
|
||||
|
||||
|
||||
class FallbackSkipReason(str, enum.Enum):
|
||||
NOT_AUTHORIZED = "not_authorized"
|
||||
OVER_BUDGET = "over_budget"
|
||||
|
|
|
|||
|
|
@ -3372,6 +3372,7 @@ class StandardLoggingMetadata(StandardLoggingUserAPIKeyMetadata):
|
|||
team_alias: str | None
|
||||
team_id: str | None
|
||||
used_client_oauth_token: ReadOnly[bool | None]
|
||||
is_silent_experiment: ReadOnly[NotRequired[bool]]
|
||||
|
||||
|
||||
class AzureSpillover(TypedDict):
|
||||
|
|
|
|||
|
|
@ -830,6 +830,31 @@ class TestLangfuseUsageDetails(unittest.TestCase):
|
|||
assert generation_metadata["litellm_response_cost"] == 0.25
|
||||
assert generation_metadata["api_base"] == "https://real-api-base"
|
||||
|
||||
def test_caller_sent_is_silent_experiment_cannot_hide_the_mirrored_call_flag(self):
|
||||
payload = self._build_standard_logging_payload(trace_id="canary-trace-id")
|
||||
payload["metadata"]["is_silent_experiment"] = True
|
||||
payload["metadata"]["requester_metadata"] = {"is_silent_experiment": False}
|
||||
kwargs = self._build_langfuse_kwargs(payload)
|
||||
metadata = self._canary_request_metadata()
|
||||
self.use_real_langfuse_client()
|
||||
|
||||
self.logger._log_langfuse_v2(
|
||||
user_id="user-1",
|
||||
metadata=metadata,
|
||||
litellm_params={"metadata": metadata},
|
||||
output=None,
|
||||
start_time=datetime.datetime(2024, 1, 1, 12, 0, 0),
|
||||
end_time=datetime.datetime(2024, 1, 1, 12, 0, 1),
|
||||
kwargs=kwargs,
|
||||
optional_params={},
|
||||
input=None,
|
||||
response_obj=None,
|
||||
level="INFO",
|
||||
litellm_call_id="canary-call-id",
|
||||
)
|
||||
|
||||
assert self.exported_generation_metadata()["is_silent_experiment"] is True
|
||||
|
||||
def test_generation_metadata_carries_the_call_id_and_response_id(self):
|
||||
"""
|
||||
v2's generation id was ``time-<hh-mm-ss-us>_<response id>``, so a generation could
|
||||
|
|
|
|||
|
|
@ -11857,7 +11857,7 @@ def test_get_additional_headers():
|
|||
|
||||
|
||||
def all_fields_present(standard_logging_metadata: StandardLoggingMetadata):
|
||||
for field in StandardLoggingMetadata.__annotations__.keys():
|
||||
for field in StandardLoggingMetadata.__required_keys__:
|
||||
assert field in standard_logging_metadata
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ from starlette.datastructures import UploadFile as StarletteUploadFile
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS
|
||||
from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS, SILENT_EXPERIMENT_METADATA_KEY
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
|
@ -63,6 +63,7 @@ from litellm.proxy.pass_through_endpoints.success_handler import (
|
|||
from litellm.proxy.route_llm_request import ProxyModelNotFoundError
|
||||
from litellm.types import utils as types_utils
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
from litellm.types.router import SilentExperimentMarker
|
||||
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
|
||||
LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY,
|
||||
LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY,
|
||||
|
|
@ -10339,6 +10340,43 @@ def test_init_kwargs_no_authenticated_identity_field_is_client_settable(mock_req
|
|||
survived: Final = {key: metadata.get(key) for key in expected if metadata.get(key) != expected[key]}
|
||||
assert survived == {}, f"client-supplied values survived for: {sorted(survived)}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("planted", [True, SilentExperimentMarker.MIRRORED.value])
|
||||
def test_init_kwargs_client_metadata_cannot_mark_a_pass_through_call_as_a_silent_experiment(
|
||||
planted: object, mock_user_api_key_dict: UserAPIKeyAuth
|
||||
) -> None:
|
||||
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
|
||||
|
||||
started: Final = datetime(2026, 1, 15, 12, 0, 0)
|
||||
result: Final = HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint(
|
||||
request=Request({"type": "http", "method": "POST", "path": "/test", "headers": [], "query_string": b""}),
|
||||
user_api_key_dict=mock_user_api_key_dict,
|
||||
passthrough_logging_payload=PassthroughStandardLoggingPayload(url="https://test.com", request_body={}),
|
||||
litellm_call_id="test-call-id",
|
||||
logging_obj=LiteLLMLoggingObj(
|
||||
model="test-model",
|
||||
messages=[],
|
||||
stream=False,
|
||||
call_type="test-call-type",
|
||||
start_time=started,
|
||||
litellm_call_id="test-call-id",
|
||||
function_id="test-function-id",
|
||||
),
|
||||
_parsed_body={
|
||||
"litellm_metadata": {SILENT_EXPERIMENT_METADATA_KEY: planted, "client_key": "kept"},
|
||||
"metadata": {SILENT_EXPERIMENT_METADATA_KEY: planted},
|
||||
},
|
||||
)
|
||||
|
||||
row: Final = get_logging_payload(
|
||||
kwargs={"model": "test-model", "litellm_params": result["litellm_params"]},
|
||||
response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]),
|
||||
start_time=started,
|
||||
end_time=started,
|
||||
)
|
||||
assert "is_silent_experiment" not in json.loads(row["metadata"])
|
||||
assert result["litellm_params"]["metadata"]["client_key"] == "kept"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_direct_passthrough_keeps_mapped_baggage_tags_outside_upstream_body():
|
||||
from litellm.proxy.common_utils.http_parsing_utils import read_request_body
|
||||
|
|
|
|||
|
|
@ -6830,7 +6830,7 @@ def test_spend_logs_payload(model_id: str | None):
|
|||
)
|
||||
|
||||
assert len(payload["request_id"]) > 0
|
||||
expected_metadata_keys: Final = SpendLogsMetadata.__annotations__.keys()
|
||||
expected_metadata_keys: Final = SpendLogsMetadata.__annotations__.keys() - {"is_silent_experiment"}
|
||||
|
||||
assert "metadata" in payload
|
||||
assert isinstance(payload["metadata"], str)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import asyncio
|
||||
import datetime
|
||||
import json
|
||||
import time
|
||||
from collections.abc import Callable, Mapping
|
||||
|
|
@ -11,11 +12,13 @@ import pytest
|
|||
import respx
|
||||
|
||||
import litellm
|
||||
from litellm.constants import SILENT_EXPERIMENT_METADATA_KEY
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.router import Router
|
||||
from litellm.router import _silent_experiment_kwargs_snapshot
|
||||
from litellm.router import _silent_experiment_targets
|
||||
from litellm.types.router import SilentExperimentMarker
|
||||
|
||||
|
||||
class _RecordingLogger(CustomLogger):
|
||||
|
|
@ -263,6 +266,138 @@ async def test_multiple_shadow_targets_fan_out_async(recording_logger):
|
|||
assert primary_successes[0]["litellm_params"]["metadata"]["model_group"] == "primary-model"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_standard_logging_payload_marks_only_the_mirrored_calls_as_silent_experiments(
|
||||
recording_logger: _RecordingLogger,
|
||||
) -> None:
|
||||
router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"]))
|
||||
response: Final = await router.acompletion(
|
||||
model="primary-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
mock_response="pong",
|
||||
)
|
||||
assert response.choices[0].message.content == "pong"
|
||||
await _wait_for_all_successes(recording_logger, expected=3)
|
||||
|
||||
payloads: Final = [call["standard_logging_object"] for call in recording_logger.success_kwargs]
|
||||
assert sorted(payload["model_group"] for payload in payloads) == ["primary-model", "shadow-a", "shadow-b"]
|
||||
metadata_by_group: Final = {payload["model_group"]: payload["metadata"] for payload in payloads}
|
||||
assert metadata_by_group["shadow-a"]["is_silent_experiment"] is True
|
||||
assert metadata_by_group["shadow-b"]["is_silent_experiment"] is True
|
||||
assert "is_silent_experiment" not in metadata_by_group["primary-model"]
|
||||
|
||||
|
||||
_SPEND_ROW_TIME: Final = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
|
||||
|
||||
|
||||
def _spend_row_is_silent_experiment(call: Mapping[str, object]) -> object:
|
||||
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
|
||||
|
||||
row: Final = get_logging_payload(
|
||||
kwargs=dict(call),
|
||||
response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]),
|
||||
start_time=_SPEND_ROW_TIME,
|
||||
end_time=_SPEND_ROW_TIME,
|
||||
)
|
||||
return json.loads(row["metadata"]).get("is_silent_experiment")
|
||||
|
||||
|
||||
def _through_the_spend_collector(call: Mapping[str, object]) -> Mapping[str, object]:
|
||||
from litellm.proxy.spend_tracking.spend_event import (
|
||||
SpendEvent,
|
||||
build_spend_event,
|
||||
decode_spend_event,
|
||||
spend_event_callback_args,
|
||||
)
|
||||
|
||||
line: Final = build_spend_event(
|
||||
call,
|
||||
litellm.ModelResponse(id="chatcmpl-test", choices=[]),
|
||||
_SPEND_ROW_TIME,
|
||||
_SPEND_ROW_TIME,
|
||||
store_bodies=False,
|
||||
)
|
||||
assert isinstance(line, bytes)
|
||||
event: Final = decode_spend_event(line)
|
||||
assert isinstance(event, SpendEvent)
|
||||
return spend_event_callback_args(event).kwargs
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"spend_kwargs",
|
||||
[pytest.param(lambda call: call, id="in_process"), pytest.param(_through_the_spend_collector, id="collector")],
|
||||
)
|
||||
async def test_spend_rows_mark_only_the_mirrored_calls_when_the_caller_sends_litellm_metadata(
|
||||
recording_logger: _RecordingLogger,
|
||||
spend_kwargs: Callable[[Mapping[str, object]], Mapping[str, object]],
|
||||
) -> None:
|
||||
router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"]))
|
||||
await router.acompletion(
|
||||
model="primary-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
mock_response="pong",
|
||||
litellm_metadata={"user_api_key": "test-key"},
|
||||
)
|
||||
await _wait_for_all_successes(recording_logger, expected=3)
|
||||
|
||||
silent_by_group: Final = {
|
||||
call["standard_logging_object"]["model_group"]: _spend_row_is_silent_experiment(spend_kwargs(call))
|
||||
for call in recording_logger.success_kwargs
|
||||
}
|
||||
assert silent_by_group == {"primary-model": None, "shadow-a": True, "shadow-b": True}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_logging_callbacks_never_receive_the_router_mirror_marker(recording_logger: _RecordingLogger) -> None:
|
||||
router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"]))
|
||||
await router.acompletion(
|
||||
model="primary-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
mock_response="pong",
|
||||
litellm_metadata={"user_api_key": "test-key"},
|
||||
)
|
||||
await _wait_for_all_successes(recording_logger, expected=3)
|
||||
|
||||
request_metadata_keys: Final = {
|
||||
call["standard_logging_object"]["model_group"]: {
|
||||
*call["litellm_params"]["metadata"],
|
||||
*call["litellm_params"]["litellm_metadata"],
|
||||
}
|
||||
for call in recording_logger.success_kwargs
|
||||
}
|
||||
assert sorted(request_metadata_keys) == ["primary-model", "shadow-a", "shadow-b"]
|
||||
assert all(SILENT_EXPERIMENT_METADATA_KEY not in keys for keys in request_metadata_keys.values())
|
||||
assert all("is_silent_experiment" in request_metadata_keys[group] for group in ("shadow-a", "shadow-b"))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("bucket", "planted"),
|
||||
[
|
||||
("metadata", {"is_silent_experiment": True}),
|
||||
("metadata", {SILENT_EXPERIMENT_METADATA_KEY: SilentExperimentMarker.MIRRORED.value}),
|
||||
("litellm_metadata", {SILENT_EXPERIMENT_METADATA_KEY: SilentExperimentMarker.MIRRORED.value}),
|
||||
("litellm_metadata", {SILENT_EXPERIMENT_METADATA_KEY: True}),
|
||||
],
|
||||
)
|
||||
async def test_a_caller_sent_silent_experiment_flag_does_not_mark_a_real_call_as_mirrored(
|
||||
recording_logger: _RecordingLogger, bucket: str, planted: dict[str, object]
|
||||
) -> None:
|
||||
router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"]))
|
||||
await router.acompletion(
|
||||
model="shadow-b",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
mock_response="pong",
|
||||
**{bucket: planted},
|
||||
)
|
||||
await _wait_for_all_successes(recording_logger, expected=1)
|
||||
|
||||
call: Final = recording_logger.success_kwargs[0]
|
||||
assert "is_silent_experiment" not in call["standard_logging_object"]["metadata"]
|
||||
assert _spend_row_is_silent_experiment(call) is None
|
||||
|
||||
|
||||
def test_multiple_shadow_targets_fan_out_sync(recording_logger):
|
||||
router = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"]))
|
||||
response = router.completion(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue