scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-10 03:28:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 85

feat(spend-logs): record when a spend log row is the auto-router's own classifier call (#35300)

Browse source 
The complexity router's classifier sub-call copies the parent request's metadata
verbatim, so its spend log row carries the caller's key, team and user and is
indistinguishable from traffic the caller actually sent. Nothing on the row says
otherwise: call_type is "acompletion" either way, model_group is overwritten to the
classifier's own model group so the row never looks auto-routed, and routing_decision
is absent exactly as it is on an ordinary request.

Record the fact the system already knows at call time. internal_call_origin is
declared on SpendLogsMetadata, which is the allowlist _get_spend_logs_metadata
projects onto, and stamped in _classifier_call_metadata; both classifier paths
already route through that one function and it feeds the metadata and
litellm_metadata buckets alike, so every request surface is covered at one site.
The key is reserved rather than caller-supplied, so it joins routing_decision in the
untrusted-metadata strip and a caller cannot label their own traffic as router
overhead.

The classifier call also inherited no session identity, so the router minted a fresh
trace id and the row landed in a session of its own. Forwarding the parent's session
puts it in the trace of the request that triggered it, which is where an operator
looks for what the routing cost.
This commit is contained in:
tin-berri 2026-07-30 19:26:42 -07:00 • committed by GitHub
parent c8bec20443
commit 2dbcb9a999
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 143 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
litellm/constants.py
View file

@ -1297,6 +1297,7 @@ X_LITELLM_DISABLE_CALLBACKS = "x-litellm-disable-callbacks"
LITELLM_METADATA_FIELD = "litellm_metadata"
OLD_LITELLM_METADATA_FIELD = "metadata"
RETURN_RAW_MODEL_NAME_METADATA_KEY = "_complexity_router_return_raw_model_name"
INTERNAL_CALL_ORIGIN_METADATA_KEY = "internal_call_origin"
LITELLM_TRUNCATED_PAYLOAD_FIELD = "litellm_truncated"
LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE = (
"Truncation is a DB storage safeguard. "

2
litellm/proxy/_types.py
View file

@ -44,6 +44,7 @@ from litellm.types.utils import (
EmbeddingResponse,
GenericBudgetConfigType,
ImageResponse,
InternalCallOrigin,
LiteLLMPydanticObjectBase,
ModelResponse,
ProviderField,
@ -3304,6 +3305,7 @@ class SpendLogsMetadata(TypedDict):
mcp_tool_call_metadata: Optional[StandardLoggingMCPToolCall]
vector_store_request_metadata: Optional[List[StandardLoggingVectorStoreRequest]]
routing_decision: StandardLoggingRoutingDecision | None
internal_call_origin: InternalCallOrigin | None
guardrail_information: Optional[List[StandardLoggingGuardrailInformation]]
eval_information: Optional[Any]
status: StandardLoggingPayloadStatus

7
litellm/proxy/litellm_pre_call_utils.py
View file

@ -13,7 +13,11 @@ from starlette.datastructures import Headers
import litellm
from litellm._logging import verbose_logger, verbose_proxy_logger
from litellm._service_logger import ServiceLogging
from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS, PRE_CALL_EXECUTED_GUARDRAILS_KEY
from litellm.constants import (
INTERNAL_CALL_ORIGIN_METADATA_KEY,
LITELLM_PROXY_MASTER_KEY_ALIAS,
PRE_CALL_EXECUTED_GUARDRAILS_KEY,
)
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
iter_client_callback_metadata_dicts,
@ -199,6 +203,7 @@ _UNTRUSTED_METADATA_CONTROL_FIELDS = (
"applied_policies",
"policy_sources",
"routing_decision",
INTERNAL_CALL_ORIGIN_METADATA_KEY,
"standard_logging_object",
"proxy_server_request",
"secret_fields",

1
litellm/proxy/spend_tracking/spend_tracking_utils.py
View file

@ -109,6 +109,7 @@ def _get_spend_logs_metadata(
model_map_information=None,
usage_object=None,
guardrail_information=None,
internal_call_origin=None,
eval_information=None,
cold_storage_object_key=cold_storage_object_key,
litellm_overhead_time_ms=None,

12
litellm/router_strategy/complexity_router/complexity_router.py
View file

@ -25,10 +25,11 @@ from typing import TYPE_CHECKING, Any, Literal, NamedTuple, Union, cast
from pydantic import BaseModel
from litellm._logging import verbose_router_logger
from litellm.constants import RETURN_RAW_MODEL_NAME_METADATA_KEY
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY, RETURN_RAW_MODEL_NAME_METADATA_KEY
from litellm.integrations.custom_logger import CustomLogger
from litellm.llms.base_llm.base_utils import type_to_response_format_param
from litellm.types.utils import (
AUTOROUTER_CLASSIFIER_CALL_ORIGIN,
ModelResponse,
RoutingDecisionCause,
StandardLoggingRoutingDecision,
@ -116,7 +117,12 @@ def _classifier_call_metadata(metadata: dict[str, Any] | None) -> dict[str, Any]
k: _sanitize_user_api_key_auth(v) if k == "user_api_key_auth" else v
for k, v in metadata.items()
if k not in _BUDGET_RESERVATION_METADATA_KEYS
}
} | {INTERNAL_CALL_ORIGIN_METADATA_KEY: AUTOROUTER_CLASSIFIER_CALL_ORIGIN}
def _parent_session_kwargs(request_kwargs: Mapping[str, Any] | None) -> Mapping[str, Any]:
kwargs = request_kwargs or {}
return {k: kwargs[k] for k in ("litellm_session_id", "litellm_trace_id") if kwargs.get(k) is not None}
def _effective_turn_off_message_logging(request_kwargs: Mapping[str, Any] | None) -> bool | None:
@ -734,6 +740,7 @@ class ComplexityRouter(CustomLogger):
metadata=metadata,
proxy_server_request=proxy_server_request,
turn_off_message_logging=turn_off_message_logging,
**_parent_session_kwargs(request_kwargs),
)
content = response.choices[0].message.content
if not content:
@ -1186,6 +1193,7 @@ class ComplexityRouter(CustomLogger):
litellm_metadata=litellm_metadata,
proxy_server_request=proxy_server_request,
turn_off_message_logging=turn_off_message_logging,
**_parent_session_kwargs(request_kwargs),
)
)[0]
route_choice = await routelayer.acall(vector=query_vector)

7
litellm/types/utils.py
View file

@ -2703,6 +2703,13 @@ RoutingDecisionCause = Literal[
]
InternalCallOrigin = Literal["autorouter_classifier"]
"""Which internal litellm feature originated a billed sub-call, so a spend log row
records that it is not traffic the caller sent."""
AUTOROUTER_CLASSIFIER_CALL_ORIGIN: InternalCallOrigin = "autorouter_classifier"
class StandardLoggingRoutingDecision(TypedDict, total=False):
"""Per-request provenance for a pre-routing strategy (auto-router) decision."""

6
tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py
View file

@ -2396,7 +2396,7 @@ class TestSpendLogsPayload:
"model": "gpt-4o",
"user": "",
"team_id": "",
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
"cache_key": "Cache OFF",
"spend": 0.00022500000000000002,
"total_tokens": 30,
@ -2492,7 +2492,7 @@ class TestSpendLogsPayload:
"model": "claude-4-sonnet-20250514",
"user": "",
"team_id": "",
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
"cache_key": "Cache OFF",
"spend": 0.01383,
"total_tokens": 2598,
@ -2586,7 +2586,7 @@ class TestSpendLogsPayload:
"model": "claude-4-sonnet-20250514",
"user": "",
"team_id": "",
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
"cache_key": "Cache OFF",
"spend": 0.01383,
"total_tokens": 2598,

43
tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
View file

@ -2916,3 +2916,46 @@ def test_no_routing_decision_key_defaults_to_none_in_spend_log_metadata():
)
metadata = json.loads(payload["metadata"])
assert metadata["routing_decision"] is None
@pytest.mark.parametrize("bucket", ["metadata", "litellm_metadata"])
def test_internal_call_origin_survives_into_spend_log_metadata(bucket):
"""The origin is only useful if it reaches the row the Logs UI reads.
_get_spend_logs_metadata projects onto SpendLogsMetadata.__annotations__, so an
undeclared key is dropped silently. Both buckets are covered because the resolver
returns litellm_metadata when present and metadata otherwise, and the classifier
sub-call populates whichever the parent route used.
"""
payload = get_logging_payload(
kwargs={
"model": "gpt-4o-mini",
"litellm_params": {
bucket: {
"user_api_key": "test-key",
"internal_call_origin": "autorouter_classifier",
}
},
},
response_obj=litellm.ModelResponse(id="chatcmpl-classifier", choices=[], usage=litellm.Usage()),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
metadata = json.loads(payload["metadata"])
assert metadata["internal_call_origin"] == "autorouter_classifier"
def test_user_traffic_carries_no_internal_call_origin():
"""The negative class the badge depends on: an ordinary request must be
distinguishable from a classifier call, not merely unlabelled by accident."""
payload = get_logging_payload(
kwargs={
"model": "gpt-4o-mini",
"litellm_params": {"metadata": {"user_api_key": "test-key"}},
},
response_obj=litellm.ModelResponse(id="chatcmpl-user-traffic", choices=[], usage=litellm.Usage()),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
metadata = json.loads(payload["metadata"])
assert metadata["internal_call_origin"] is None

2
tests/test_litellm/proxy/test_litellm_pre_call_utils.py
View file

@ -672,6 +672,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields():
"applied_policies": ["spoofed-policy"],
"policy_sources": {"spoofed-policy": "request"},
"routing_decision": {"cause": "forged", "routed_model": "spoofed"},
"internal_call_origin": "autorouter_classifier",
"_guardrail_pipelines": [{"name": "spoofed"}],
"_pipeline_managed_guardrails": ["evaded"],
"safe_user_metadata": "kept",
@ -714,6 +715,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields():
"applied_policies",
"policy_sources",
"routing_decision",
"internal_call_origin",
"_guardrail_pipelines",
"_pipeline_managed_guardrails",
}

77
tests/test_litellm/router_strategy/test_complexity_router.py
View file

@ -1422,7 +1422,7 @@ class TestLLMClassifier:
request_metadata = {"user_api_key": "sk-abc", "user_api_key_team_id": "team-1"}
await llm_complexity_router.aclassify("hi", request_kwargs={"litellm_metadata": request_metadata})
call_kwargs = mock_router_instance.acompletion.call_args.kwargs
assert call_kwargs["metadata"] == request_metadata
assert call_kwargs["metadata"] == {**request_metadata, "internal_call_origin": "autorouter_classifier"}
@pytest.mark.asyncio
async def test_aclassify_forwards_metadata_key_used_by_chat_completions(
@ -1440,7 +1440,7 @@ class TestLLMClassifier:
request_metadata = {"user_api_key": "sk-abc", "user_api_key_team_id": "team-1"}
await llm_complexity_router.aclassify("hi", request_kwargs={"metadata": request_metadata})
call_kwargs = mock_router_instance.acompletion.call_args.kwargs
assert call_kwargs["metadata"] == request_metadata
assert call_kwargs["metadata"] == {**request_metadata, "internal_call_origin": "autorouter_classifier"}
@pytest.mark.asyncio
async def test_aclassify_captures_request_body_in_proxy_server_request(
@ -1555,12 +1555,38 @@ class TestLLMClassifier:
"user_api_key": "sk-abc",
"user_api_key_team_id": "team-1",
"user_api_key_auth": {"models": ["gpt-4o"]},
"internal_call_origin": "autorouter_classifier",
}
assert request_metadata["user_api_key_auth"] == {
"models": ["gpt-4o"],
"budget_reservation": {"reserved_cost": 1.0},
}
@pytest.mark.asyncio
@pytest.mark.parametrize(
"parent_kwargs, expected",
[
({"litellm_trace_id": "trace-1"}, {"litellm_trace_id": "trace-1"}),
({"litellm_session_id": "sess-1"}, {"litellm_session_id": "sess-1"}),
(
{"litellm_session_id": "sess-1", "litellm_trace_id": "trace-1"},
{"litellm_session_id": "sess-1", "litellm_trace_id": "trace-1"},
),
({}, {}),
],
)
async def test_aclassify_chains_classifier_call_into_parent_session(
self, llm_complexity_router, mock_router_instance, parent_kwargs, expected
):
"""Without the parent's session identity the router mints a fresh trace id for the
sub-call, so the classifier's spend row lands in a session of its own and never
appears in the trace of the request that triggered it."""
mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "SIMPLE"}'))
await llm_complexity_router.aclassify("hi", request_kwargs={"metadata": {}, **parent_kwargs})
call_kwargs = mock_router_instance.acompletion.call_args.kwargs
for key in ("litellm_session_id", "litellm_trace_id"):
assert call_kwargs.get(key) == expected.get(key)
@pytest.mark.asyncio
async def test_aclassify_falls_back_to_heuristic_on_llm_exception(
self, llm_complexity_router, mock_router_instance
@ -1608,7 +1634,7 @@ class TestLLMClassifier:
assert result is not None
assert result.model == "o1-preview" # REASONING tier model
call_kwargs = mock_router_instance.acompletion.call_args.kwargs
assert call_kwargs["metadata"] == request_metadata
assert call_kwargs["metadata"] == {**request_metadata, "internal_call_origin": "autorouter_classifier"}
class TestRouterPreRoutingAliasOverrides:
@ -2285,8 +2311,9 @@ class TestSemanticKeywordTierRules:
)
assert result is not None
assert fake_router.async_embedding_kwargs, "expected an embedding call for the prompt"
assert fake_router.async_embedding_kwargs[0]["metadata"] == caller_metadata
assert fake_router.async_embedding_kwargs[0]["litellm_metadata"] == caller_litellm_metadata
origin = {"internal_call_origin": "autorouter_classifier"}
assert fake_router.async_embedding_kwargs[0]["metadata"] == {**caller_metadata, **origin}
assert fake_router.async_embedding_kwargs[0]["litellm_metadata"] == {**caller_litellm_metadata, **origin}
@pytest.mark.asyncio
async def test_semantic_embedding_call_captures_request_body_in_proxy_server_request(self, basic_config):
@ -2395,6 +2422,7 @@ class TestSemanticKeywordTierRules:
"user_api_key_hash": "hash-abc",
"user_api_key_team_id": "team-1",
"user_api_key_auth": {"models": ["voyage-3-5"]},
"internal_call_origin": "autorouter_classifier",
}
assert fake_router.async_embedding_kwargs[0]["metadata"] == expected
assert fake_router.async_embedding_kwargs[0]["litellm_metadata"] == expected
@ -2730,15 +2758,46 @@ class TestSubCallMetadataSanitization:
assert sanitized["user_api_key_auth"] is not None
assert _get_budget_reservation_from_metadata(sanitized) is None
def test_returns_empty_dict_for_missing_metadata(self):
def test_absent_parent_bucket_stays_empty(self):
"""An absent bucket must not be materialized just to carry the origin.
The embedding path passes both buckets, and get_litellm_metadata_from_kwargs
prefers litellm_metadata whenever it is truthy, backfilling only user_api_key*
keys from metadata. Returning an origin-only dict here would make a chat
completions parent's empty litellm_metadata win and silently drop
requester_ip_address, tags and spend_logs_metadata from the classifier's row."""
from litellm.router_strategy.complexity_router.complexity_router import (
_classifier_call_metadata,
)
for absent in (None, {}):
result = _classifier_call_metadata(absent)
assert result == {}
assert isinstance(result, dict)
assert _classifier_call_metadata(absent) == {}
def test_classifier_buckets_keep_non_spend_fields_on_a_chat_completions_parent(self):
"""Drives the real resolver over the buckets the embedding classifier builds."""
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs
from litellm.router_strategy.complexity_router.complexity_router import (
_classifier_call_metadata,
)
parent = {
"user_api_key": "sk-abc",
"requester_ip_address": "10.0.0.1",
"spend_logs_metadata": {"team_note": "keep me"},
"tags": ["prod"],
}
resolved = get_litellm_metadata_from_kwargs(
{
"litellm_params": {
"metadata": _classifier_call_metadata(parent),
"litellm_metadata": _classifier_call_metadata(None),
}
}
)
assert resolved["internal_call_origin"] == "autorouter_classifier"
assert resolved["requester_ip_address"] == "10.0.0.1"
assert resolved["spend_logs_metadata"] == {"team_note": "keep me"}
assert resolved["tags"] == ["prod"]
def test_sanitized_auth_keeps_access_group_fields_and_leaves_original_untouched(self):
from litellm.proxy._types import UserAPIKeyAuth