scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-14 23:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 102

merge(litellm_internal_staging): sync latest staging; ratchet budgets to genuine merged-tree counts

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
shivam 2026-07-31 02:40:06 +00:00
commit 475ebe5baf
13 changed files with 156 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
basedpyright-code-budget.json
View file

@ -1,6 +1,6 @@
{
"reportAny": {
"limit": 33129
"limit": 31903
},
"reportArgumentType": {
"limit": 2645
@ -24,7 +24,7 @@
"limit": 42
},
"reportExplicitAny": {
"limit": 10227
"limit": 10214
},
"reportFunctionMemberAccess": {
"limit": 11
@ -99,7 +99,7 @@
"limit": 0
},
"reportUnknownArgumentType": {
"limit": 45498
"limit": 45366
},
"reportUnknownLambdaType": {
"limit": 113
@ -117,7 +117,7 @@
"limit": 177
},
"reportUnnecessaryComparison": {
"limit": 1023
"limit": 1021
},
"reportUnnecessaryContains": {
"limit": 7
@ -135,7 +135,7 @@
"limit": 33
},
"reportUnusedFunction": {
"limit": 206
"limit": 204
},
"reportUnusedImport": {
"limit": 1003

1
litellm/constants.py
View file

@ -1299,6 +1299,7 @@ X_LITELLM_DISABLE_CALLBACKS = "x-litellm-disable-callbacks"
LITELLM_METADATA_FIELD = "litellm_metadata"
OLD_LITELLM_METADATA_FIELD = "metadata"
RETURN_RAW_MODEL_NAME_METADATA_KEY = "_complexity_router_return_raw_model_name"
INTERNAL_CALL_ORIGIN_METADATA_KEY = "internal_call_origin"
LITELLM_TRUNCATED_PAYLOAD_FIELD = "litellm_truncated"
LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE = (
"Truncation is a DB storage safeguard. "

2
litellm/proxy/_types.py
View file

@ -48,6 +48,7 @@ from litellm.types.utils import (
EmbeddingResponse,
GenericBudgetConfigType,
ImageResponse,
InternalCallOrigin,
LiteLLMPydanticObjectBase,
ModelResponse,
ProviderField,
@ -3309,6 +3310,7 @@ class SpendLogsMetadata(TypedDict):
mcp_tool_call_metadata: Optional[StandardLoggingMCPToolCall]
vector_store_request_metadata: Optional[List[StandardLoggingVectorStoreRequest]]
routing_decision: StandardLoggingRoutingDecision | None
internal_call_origin: InternalCallOrigin | None
guardrail_information: Optional[List[StandardLoggingGuardrailInformation]]
eval_information: Optional[Any]
status: StandardLoggingPayloadStatus

7
litellm/proxy/litellm_pre_call_utils.py
View file

@ -14,7 +14,11 @@ from starlette.datastructures import Headers
import litellm
from litellm._logging import verbose_logger, verbose_proxy_logger
from litellm._service_logger import ServiceLogging
from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS, PRE_CALL_EXECUTED_GUARDRAILS_KEY
from litellm.constants import (
INTERNAL_CALL_ORIGIN_METADATA_KEY,
LITELLM_PROXY_MASTER_KEY_ALIAS,
PRE_CALL_EXECUTED_GUARDRAILS_KEY,
)
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
iter_client_callback_metadata_dicts,
@ -203,6 +207,7 @@ _UNTRUSTED_METADATA_CONTROL_FIELDS = (
"applied_policies",
"policy_sources",
"routing_decision",
INTERNAL_CALL_ORIGIN_METADATA_KEY,
"standard_logging_object",
"proxy_server_request",
"secret_fields",

1
litellm/proxy/spend_tracking/spend_tracking_utils.py
View file

@ -109,6 +109,7 @@ def _get_spend_logs_metadata(
model_map_information=None,
usage_object=None,
guardrail_information=None,
internal_call_origin=None,
eval_information=None,
cold_storage_object_key=cold_storage_object_key,
litellm_overhead_time_ms=None,

12
litellm/router_strategy/complexity_router/complexity_router.py
View file

@ -25,10 +25,11 @@ from typing import TYPE_CHECKING, Any, Literal, NamedTuple, Union, cast
from pydantic import BaseModel
from litellm._logging import verbose_router_logger
from litellm.constants import RETURN_RAW_MODEL_NAME_METADATA_KEY
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY, RETURN_RAW_MODEL_NAME_METADATA_KEY
from litellm.integrations.custom_logger import CustomLogger
from litellm.llms.base_llm.base_utils import type_to_response_format_param
from litellm.types.utils import (
AUTOROUTER_CLASSIFIER_CALL_ORIGIN,
ModelResponse,
RoutingDecisionCause,
StandardLoggingRoutingDecision,
@ -116,7 +117,12 @@ def _classifier_call_metadata(metadata: dict[str, Any] | None) -> dict[str, Any]
k: _sanitize_user_api_key_auth(v) if k == "user_api_key_auth" else v
for k, v in metadata.items()
if k not in _BUDGET_RESERVATION_METADATA_KEYS
}
} | {INTERNAL_CALL_ORIGIN_METADATA_KEY: AUTOROUTER_CLASSIFIER_CALL_ORIGIN}
def _parent_session_kwargs(request_kwargs: Mapping[str, Any] | None) -> Mapping[str, Any]:
kwargs = request_kwargs or {}
return {k: kwargs[k] for k in ("litellm_session_id", "litellm_trace_id") if kwargs.get(k) is not None}
def _effective_turn_off_message_logging(request_kwargs: Mapping[str, Any] | None) -> bool | None:
@ -734,6 +740,7 @@ class ComplexityRouter(CustomLogger):
metadata=metadata,
proxy_server_request=proxy_server_request,
turn_off_message_logging=turn_off_message_logging,
**_parent_session_kwargs(request_kwargs),
)
content = response.choices[0].message.content
if not content:
@ -1186,6 +1193,7 @@ class ComplexityRouter(CustomLogger):
litellm_metadata=litellm_metadata,
proxy_server_request=proxy_server_request,
turn_off_message_logging=turn_off_message_logging,
**_parent_session_kwargs(request_kwargs),
)
)[0]
route_choice = await routelayer.acall(vector=query_vector)

7
litellm/types/utils.py
View file

@ -2703,6 +2703,13 @@ RoutingDecisionCause = Literal[
]
InternalCallOrigin = Literal["autorouter_classifier"]
"""Which internal litellm feature originated a billed sub-call, so a spend log row
records that it is not traffic the caller sent."""
AUTOROUTER_CLASSIFIER_CALL_ORIGIN: InternalCallOrigin = "autorouter_classifier"
class StandardLoggingRoutingDecision(TypedDict, total=False):
"""Per-request provenance for a pre-routing strategy (auto-router) decision."""

14
ruff-strict-budget.json
View file

@ -24,7 +24,7 @@
"limit": 130
},
"ANN401": {
"limit": 2017
"limit": 2010
},
"ASYNC230": {
"limit": 14
@ -135,7 +135,7 @@
"limit": 30
},
"PERF401": {
"limit": 144
"limit": 142
},
"PERF402": {
"limit": 9
@ -306,7 +306,7 @@
"limit": 9
},
"TID251": {
"limit": 2653
"limit": 2652
},
"TRY002": {
"limit": 547
@ -315,16 +315,16 @@
"limit": 98
},
"TRY201": {
"limit": 424
"limit": 420
},
"TRY203": {
"limit": 123
"limit": 121
},
"TRY300": {
"limit": 883
"limit": 879
},
"UP006": {
"limit": 12168
"limit": 12147
},
"UP007": {
"limit": 2526

6
tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py
View file

@ -2396,7 +2396,7 @@ class TestSpendLogsPayload:
"model": "gpt-4o",
"user": "",
"team_id": "",
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
"cache_key": "Cache OFF",
"spend": 0.00022500000000000002,
"total_tokens": 30,
@ -2492,7 +2492,7 @@ class TestSpendLogsPayload:
"model": "claude-4-sonnet-20250514",
"user": "",
"team_id": "",
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
"cache_key": "Cache OFF",
"spend": 0.01383,
"total_tokens": 2598,
@ -2586,7 +2586,7 @@ class TestSpendLogsPayload:
"model": "claude-4-sonnet-20250514",
"user": "",
"team_id": "",
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
"cache_key": "Cache OFF",
"spend": 0.01383,
"total_tokens": 2598,

43
tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
View file

@ -2916,3 +2916,46 @@ def test_no_routing_decision_key_defaults_to_none_in_spend_log_metadata():
)
metadata = json.loads(payload["metadata"])
assert metadata["routing_decision"] is None
@pytest.mark.parametrize("bucket", ["metadata", "litellm_metadata"])
def test_internal_call_origin_survives_into_spend_log_metadata(bucket):
"""The origin is only useful if it reaches the row the Logs UI reads.
_get_spend_logs_metadata projects onto SpendLogsMetadata.__annotations__, so an
undeclared key is dropped silently. Both buckets are covered because the resolver
returns litellm_metadata when present and metadata otherwise, and the classifier
sub-call populates whichever the parent route used.
"""
payload = get_logging_payload(
kwargs={
"model": "gpt-4o-mini",
"litellm_params": {
bucket: {
"user_api_key": "test-key",
"internal_call_origin": "autorouter_classifier",
}
},
},
response_obj=litellm.ModelResponse(id="chatcmpl-classifier", choices=[], usage=litellm.Usage()),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
metadata = json.loads(payload["metadata"])
assert metadata["internal_call_origin"] == "autorouter_classifier"
def test_user_traffic_carries_no_internal_call_origin():
"""The negative class the badge depends on: an ordinary request must be
distinguishable from a classifier call, not merely unlabelled by accident."""
payload = get_logging_payload(
kwargs={
"model": "gpt-4o-mini",
"litellm_params": {"metadata": {"user_api_key": "test-key"}},
},
response_obj=litellm.ModelResponse(id="chatcmpl-user-traffic", choices=[], usage=litellm.Usage()),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
metadata = json.loads(payload["metadata"])
assert metadata["internal_call_origin"] is None

2
tests/test_litellm/proxy/test_litellm_pre_call_utils.py
View file

@ -647,6 +647,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields():
"applied_policies": ["spoofed-policy"],
"policy_sources": {"spoofed-policy": "request"},
"routing_decision": {"cause": "forged", "routed_model": "spoofed"},
"internal_call_origin": "autorouter_classifier",
"_guardrail_pipelines": [{"name": "spoofed"}],
"_pipeline_managed_guardrails": ["evaded"],
"safe_user_metadata": "kept",
@ -689,6 +690,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields():
"applied_policies",
"policy_sources",
"routing_decision",
"internal_call_origin",
"_guardrail_pipelines",
"_pipeline_managed_guardrails",
}

77
tests/test_litellm/router_strategy/test_complexity_router.py
View file

@ -1422,7 +1422,7 @@ class TestLLMClassifier:
request_metadata = {"user_api_key": "sk-abc", "user_api_key_team_id": "team-1"}
await llm_complexity_router.aclassify("hi", request_kwargs={"litellm_metadata": request_metadata})
call_kwargs = mock_router_instance.acompletion.call_args.kwargs
assert call_kwargs["metadata"] == request_metadata
assert call_kwargs["metadata"] == {**request_metadata, "internal_call_origin": "autorouter_classifier"}
@pytest.mark.asyncio
async def test_aclassify_forwards_metadata_key_used_by_chat_completions(
@ -1440,7 +1440,7 @@ class TestLLMClassifier:
request_metadata = {"user_api_key": "sk-abc", "user_api_key_team_id": "team-1"}
await llm_complexity_router.aclassify("hi", request_kwargs={"metadata": request_metadata})
call_kwargs = mock_router_instance.acompletion.call_args.kwargs
assert call_kwargs["metadata"] == request_metadata
assert call_kwargs["metadata"] == {**request_metadata, "internal_call_origin": "autorouter_classifier"}
@pytest.mark.asyncio
async def test_aclassify_captures_request_body_in_proxy_server_request(
@ -1555,12 +1555,38 @@ class TestLLMClassifier:
"user_api_key": "sk-abc",
"user_api_key_team_id": "team-1",
"user_api_key_auth": {"models": ["gpt-4o"]},
"internal_call_origin": "autorouter_classifier",
}
assert request_metadata["user_api_key_auth"] == {
"models": ["gpt-4o"],
"budget_reservation": {"reserved_cost": 1.0},
}
@pytest.mark.asyncio
@pytest.mark.parametrize(
"parent_kwargs, expected",
[
({"litellm_trace_id": "trace-1"}, {"litellm_trace_id": "trace-1"}),
({"litellm_session_id": "sess-1"}, {"litellm_session_id": "sess-1"}),
(
{"litellm_session_id": "sess-1", "litellm_trace_id": "trace-1"},
{"litellm_session_id": "sess-1", "litellm_trace_id": "trace-1"},
),
({}, {}),
],
)
async def test_aclassify_chains_classifier_call_into_parent_session(
self, llm_complexity_router, mock_router_instance, parent_kwargs, expected
):
"""Without the parent's session identity the router mints a fresh trace id for the
sub-call, so the classifier's spend row lands in a session of its own and never
appears in the trace of the request that triggered it."""
mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "SIMPLE"}'))
await llm_complexity_router.aclassify("hi", request_kwargs={"metadata": {}, **parent_kwargs})
call_kwargs = mock_router_instance.acompletion.call_args.kwargs
for key in ("litellm_session_id", "litellm_trace_id"):
assert call_kwargs.get(key) == expected.get(key)
@pytest.mark.asyncio
async def test_aclassify_falls_back_to_heuristic_on_llm_exception(
self, llm_complexity_router, mock_router_instance
@ -1608,7 +1634,7 @@ class TestLLMClassifier:
assert result is not None
assert result.model == "o1-preview" # REASONING tier model
call_kwargs = mock_router_instance.acompletion.call_args.kwargs
assert call_kwargs["metadata"] == request_metadata
assert call_kwargs["metadata"] == {**request_metadata, "internal_call_origin": "autorouter_classifier"}
class TestRouterPreRoutingAliasOverrides:
@ -2285,8 +2311,9 @@ class TestSemanticKeywordTierRules:
)
assert result is not None
assert fake_router.async_embedding_kwargs, "expected an embedding call for the prompt"
assert fake_router.async_embedding_kwargs[0]["metadata"] == caller_metadata
assert fake_router.async_embedding_kwargs[0]["litellm_metadata"] == caller_litellm_metadata
origin = {"internal_call_origin": "autorouter_classifier"}
assert fake_router.async_embedding_kwargs[0]["metadata"] == {**caller_metadata, **origin}
assert fake_router.async_embedding_kwargs[0]["litellm_metadata"] == {**caller_litellm_metadata, **origin}
@pytest.mark.asyncio
async def test_semantic_embedding_call_captures_request_body_in_proxy_server_request(self, basic_config):
@ -2395,6 +2422,7 @@ class TestSemanticKeywordTierRules:
"user_api_key_hash": "hash-abc",
"user_api_key_team_id": "team-1",
"user_api_key_auth": {"models": ["voyage-3-5"]},
"internal_call_origin": "autorouter_classifier",
}
assert fake_router.async_embedding_kwargs[0]["metadata"] == expected
assert fake_router.async_embedding_kwargs[0]["litellm_metadata"] == expected
@ -2730,15 +2758,46 @@ class TestSubCallMetadataSanitization:
assert sanitized["user_api_key_auth"] is not None
assert _get_budget_reservation_from_metadata(sanitized) is None
def test_returns_empty_dict_for_missing_metadata(self):
def test_absent_parent_bucket_stays_empty(self):
"""An absent bucket must not be materialized just to carry the origin.
The embedding path passes both buckets, and get_litellm_metadata_from_kwargs
prefers litellm_metadata whenever it is truthy, backfilling only user_api_key*
keys from metadata. Returning an origin-only dict here would make a chat
completions parent's empty litellm_metadata win and silently drop
requester_ip_address, tags and spend_logs_metadata from the classifier's row."""
from litellm.router_strategy.complexity_router.complexity_router import (
_classifier_call_metadata,
)
for absent in (None, {}):
result = _classifier_call_metadata(absent)
assert result == {}
assert isinstance(result, dict)
assert _classifier_call_metadata(absent) == {}
def test_classifier_buckets_keep_non_spend_fields_on_a_chat_completions_parent(self):
"""Drives the real resolver over the buckets the embedding classifier builds."""
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs
from litellm.router_strategy.complexity_router.complexity_router import (
_classifier_call_metadata,
)
parent = {
"user_api_key": "sk-abc",
"requester_ip_address": "10.0.0.1",
"spend_logs_metadata": {"team_note": "keep me"},
"tags": ["prod"],
}
resolved = get_litellm_metadata_from_kwargs(
{
"litellm_params": {
"metadata": _classifier_call_metadata(parent),
"litellm_metadata": _classifier_call_metadata(None),
}
}
)
assert resolved["internal_call_origin"] == "autorouter_classifier"
assert resolved["requester_ip_address"] == "10.0.0.1"
assert resolved["spend_logs_metadata"] == {"team_note": "keep me"}
assert resolved["tags"] == ["prod"]
def test_sanitized_auth_keeps_access_group_fields_and_leaves_original_untouched(self):
from litellm.proxy._types import UserAPIKeyAuth

2
type-discipline-budget.json
View file

@ -3,7 +3,7 @@
"limit": 23253
},
"LIT002": {
"limit": 27449
"limit": 27452
},
"LIT003": {
"limit": 292