diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index c99b1c15fe2..b249bfb091e 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -1,6 +1,6 @@ { "reportAny": { - "limit": 33129 + "limit": 31903 }, "reportArgumentType": { "limit": 2645 @@ -24,7 +24,7 @@ "limit": 42 }, "reportExplicitAny": { - "limit": 10227 + "limit": 10214 }, "reportFunctionMemberAccess": { "limit": 11 @@ -99,7 +99,7 @@ "limit": 0 }, "reportUnknownArgumentType": { - "limit": 45498 + "limit": 45366 }, "reportUnknownLambdaType": { "limit": 113 @@ -117,7 +117,7 @@ "limit": 177 }, "reportUnnecessaryComparison": { - "limit": 1023 + "limit": 1021 }, "reportUnnecessaryContains": { "limit": 7 @@ -135,7 +135,7 @@ "limit": 33 }, "reportUnusedFunction": { - "limit": 206 + "limit": 204 }, "reportUnusedImport": { "limit": 1003 diff --git a/litellm/constants.py b/litellm/constants.py index 170106f01e8..6d2d5b49323 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1299,6 +1299,7 @@ X_LITELLM_DISABLE_CALLBACKS = "x-litellm-disable-callbacks" LITELLM_METADATA_FIELD = "litellm_metadata" OLD_LITELLM_METADATA_FIELD = "metadata" RETURN_RAW_MODEL_NAME_METADATA_KEY = "_complexity_router_return_raw_model_name" +INTERNAL_CALL_ORIGIN_METADATA_KEY = "internal_call_origin" LITELLM_TRUNCATED_PAYLOAD_FIELD = "litellm_truncated" LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE = ( "Truncation is a DB storage safeguard. " diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 8682df060cc..e2434a2fbe7 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -48,6 +48,7 @@ from litellm.types.utils import ( EmbeddingResponse, GenericBudgetConfigType, ImageResponse, + InternalCallOrigin, LiteLLMPydanticObjectBase, ModelResponse, ProviderField, @@ -3309,6 +3310,7 @@ class SpendLogsMetadata(TypedDict): mcp_tool_call_metadata: Optional[StandardLoggingMCPToolCall] vector_store_request_metadata: Optional[List[StandardLoggingVectorStoreRequest]] routing_decision: StandardLoggingRoutingDecision | None + internal_call_origin: InternalCallOrigin | None guardrail_information: Optional[List[StandardLoggingGuardrailInformation]] eval_information: Optional[Any] status: StandardLoggingPayloadStatus diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 2ca1f10687f..05fa88d06bc 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -14,7 +14,11 @@ from starlette.datastructures import Headers import litellm from litellm._logging import verbose_logger, verbose_proxy_logger from litellm._service_logger import ServiceLogging -from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS, PRE_CALL_EXECUTED_GUARDRAILS_KEY +from litellm.constants import ( + INTERNAL_CALL_ORIGIN_METADATA_KEY, + LITELLM_PROXY_MASTER_KEY_ALIAS, + PRE_CALL_EXECUTED_GUARDRAILS_KEY, +) from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( iter_client_callback_metadata_dicts, @@ -203,6 +207,7 @@ _UNTRUSTED_METADATA_CONTROL_FIELDS = ( "applied_policies", "policy_sources", "routing_decision", + INTERNAL_CALL_ORIGIN_METADATA_KEY, "standard_logging_object", "proxy_server_request", "secret_fields", diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index a6105b6dff9..a6a67d57582 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -109,6 +109,7 @@ def _get_spend_logs_metadata( model_map_information=None, usage_object=None, guardrail_information=None, + internal_call_origin=None, eval_information=None, cold_storage_object_key=cold_storage_object_key, litellm_overhead_time_ms=None, diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 3f30a38b1df..b43fe0da4ca 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -25,10 +25,11 @@ from typing import TYPE_CHECKING, Any, Literal, NamedTuple, Union, cast from pydantic import BaseModel from litellm._logging import verbose_router_logger -from litellm.constants import RETURN_RAW_MODEL_NAME_METADATA_KEY +from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY, RETURN_RAW_MODEL_NAME_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger from litellm.llms.base_llm.base_utils import type_to_response_format_param from litellm.types.utils import ( + AUTOROUTER_CLASSIFIER_CALL_ORIGIN, ModelResponse, RoutingDecisionCause, StandardLoggingRoutingDecision, @@ -116,7 +117,12 @@ def _classifier_call_metadata(metadata: dict[str, Any] | None) -> dict[str, Any] k: _sanitize_user_api_key_auth(v) if k == "user_api_key_auth" else v for k, v in metadata.items() if k not in _BUDGET_RESERVATION_METADATA_KEYS - } + } | {INTERNAL_CALL_ORIGIN_METADATA_KEY: AUTOROUTER_CLASSIFIER_CALL_ORIGIN} + + +def _parent_session_kwargs(request_kwargs: Mapping[str, Any] | None) -> Mapping[str, Any]: + kwargs = request_kwargs or {} + return {k: kwargs[k] for k in ("litellm_session_id", "litellm_trace_id") if kwargs.get(k) is not None} def _effective_turn_off_message_logging(request_kwargs: Mapping[str, Any] | None) -> bool | None: @@ -734,6 +740,7 @@ class ComplexityRouter(CustomLogger): metadata=metadata, proxy_server_request=proxy_server_request, turn_off_message_logging=turn_off_message_logging, + **_parent_session_kwargs(request_kwargs), ) content = response.choices[0].message.content if not content: @@ -1186,6 +1193,7 @@ class ComplexityRouter(CustomLogger): litellm_metadata=litellm_metadata, proxy_server_request=proxy_server_request, turn_off_message_logging=turn_off_message_logging, + **_parent_session_kwargs(request_kwargs), ) )[0] route_choice = await routelayer.acall(vector=query_vector) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 668143d5950..c77183a61b8 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2703,6 +2703,13 @@ RoutingDecisionCause = Literal[ ] +InternalCallOrigin = Literal["autorouter_classifier"] +"""Which internal litellm feature originated a billed sub-call, so a spend log row +records that it is not traffic the caller sent.""" + +AUTOROUTER_CLASSIFIER_CALL_ORIGIN: InternalCallOrigin = "autorouter_classifier" + + class StandardLoggingRoutingDecision(TypedDict, total=False): """Per-request provenance for a pre-routing strategy (auto-router) decision.""" diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index b58054872a9..0bb67216216 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -24,7 +24,7 @@ "limit": 130 }, "ANN401": { - "limit": 2017 + "limit": 2010 }, "ASYNC230": { "limit": 14 @@ -135,7 +135,7 @@ "limit": 30 }, "PERF401": { - "limit": 144 + "limit": 142 }, "PERF402": { "limit": 9 @@ -306,7 +306,7 @@ "limit": 9 }, "TID251": { - "limit": 2653 + "limit": 2652 }, "TRY002": { "limit": 547 @@ -315,16 +315,16 @@ "limit": 98 }, "TRY201": { - "limit": 424 + "limit": 420 }, "TRY203": { - "limit": 123 + "limit": 121 }, "TRY300": { - "limit": 883 + "limit": 879 }, "UP006": { - "limit": 12168 + "limit": 12147 }, "UP007": { "limit": 2526 diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 795a99ec266..aa20c3f6ed4 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -2396,7 +2396,7 @@ class TestSpendLogsPayload: "model": "gpt-4o", "user": "", "team_id": "", - "metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}', + "metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}', "cache_key": "Cache OFF", "spend": 0.00022500000000000002, "total_tokens": 30, @@ -2492,7 +2492,7 @@ class TestSpendLogsPayload: "model": "claude-4-sonnet-20250514", "user": "", "team_id": "", - "metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', + "metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', "cache_key": "Cache OFF", "spend": 0.01383, "total_tokens": 2598, @@ -2586,7 +2586,7 @@ class TestSpendLogsPayload: "model": "claude-4-sonnet-20250514", "user": "", "team_id": "", - "metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', + "metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', "cache_key": "Cache OFF", "spend": 0.01383, "total_tokens": 2598, diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index c6f2a6f1792..9eb45c399db 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -2916,3 +2916,46 @@ def test_no_routing_decision_key_defaults_to_none_in_spend_log_metadata(): ) metadata = json.loads(payload["metadata"]) assert metadata["routing_decision"] is None + + +@pytest.mark.parametrize("bucket", ["metadata", "litellm_metadata"]) +def test_internal_call_origin_survives_into_spend_log_metadata(bucket): + """The origin is only useful if it reaches the row the Logs UI reads. + + _get_spend_logs_metadata projects onto SpendLogsMetadata.__annotations__, so an + undeclared key is dropped silently. Both buckets are covered because the resolver + returns litellm_metadata when present and metadata otherwise, and the classifier + sub-call populates whichever the parent route used. + """ + payload = get_logging_payload( + kwargs={ + "model": "gpt-4o-mini", + "litellm_params": { + bucket: { + "user_api_key": "test-key", + "internal_call_origin": "autorouter_classifier", + } + }, + }, + response_obj=litellm.ModelResponse(id="chatcmpl-classifier", choices=[], usage=litellm.Usage()), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + metadata = json.loads(payload["metadata"]) + assert metadata["internal_call_origin"] == "autorouter_classifier" + + +def test_user_traffic_carries_no_internal_call_origin(): + """The negative class the badge depends on: an ordinary request must be + distinguishable from a classifier call, not merely unlabelled by accident.""" + payload = get_logging_payload( + kwargs={ + "model": "gpt-4o-mini", + "litellm_params": {"metadata": {"user_api_key": "test-key"}}, + }, + response_obj=litellm.ModelResponse(id="chatcmpl-user-traffic", choices=[], usage=litellm.Usage()), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + metadata = json.loads(payload["metadata"]) + assert metadata["internal_call_origin"] is None diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index 43542f6496e..baf54f1ba3e 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -647,6 +647,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields(): "applied_policies": ["spoofed-policy"], "policy_sources": {"spoofed-policy": "request"}, "routing_decision": {"cause": "forged", "routed_model": "spoofed"}, + "internal_call_origin": "autorouter_classifier", "_guardrail_pipelines": [{"name": "spoofed"}], "_pipeline_managed_guardrails": ["evaded"], "safe_user_metadata": "kept", @@ -689,6 +690,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields(): "applied_policies", "policy_sources", "routing_decision", + "internal_call_origin", "_guardrail_pipelines", "_pipeline_managed_guardrails", } diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index b82c6792e88..2b4e882675f 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -1422,7 +1422,7 @@ class TestLLMClassifier: request_metadata = {"user_api_key": "sk-abc", "user_api_key_team_id": "team-1"} await llm_complexity_router.aclassify("hi", request_kwargs={"litellm_metadata": request_metadata}) call_kwargs = mock_router_instance.acompletion.call_args.kwargs - assert call_kwargs["metadata"] == request_metadata + assert call_kwargs["metadata"] == {**request_metadata, "internal_call_origin": "autorouter_classifier"} @pytest.mark.asyncio async def test_aclassify_forwards_metadata_key_used_by_chat_completions( @@ -1440,7 +1440,7 @@ class TestLLMClassifier: request_metadata = {"user_api_key": "sk-abc", "user_api_key_team_id": "team-1"} await llm_complexity_router.aclassify("hi", request_kwargs={"metadata": request_metadata}) call_kwargs = mock_router_instance.acompletion.call_args.kwargs - assert call_kwargs["metadata"] == request_metadata + assert call_kwargs["metadata"] == {**request_metadata, "internal_call_origin": "autorouter_classifier"} @pytest.mark.asyncio async def test_aclassify_captures_request_body_in_proxy_server_request( @@ -1555,12 +1555,38 @@ class TestLLMClassifier: "user_api_key": "sk-abc", "user_api_key_team_id": "team-1", "user_api_key_auth": {"models": ["gpt-4o"]}, + "internal_call_origin": "autorouter_classifier", } assert request_metadata["user_api_key_auth"] == { "models": ["gpt-4o"], "budget_reservation": {"reserved_cost": 1.0}, } + @pytest.mark.asyncio + @pytest.mark.parametrize( + "parent_kwargs, expected", + [ + ({"litellm_trace_id": "trace-1"}, {"litellm_trace_id": "trace-1"}), + ({"litellm_session_id": "sess-1"}, {"litellm_session_id": "sess-1"}), + ( + {"litellm_session_id": "sess-1", "litellm_trace_id": "trace-1"}, + {"litellm_session_id": "sess-1", "litellm_trace_id": "trace-1"}, + ), + ({}, {}), + ], + ) + async def test_aclassify_chains_classifier_call_into_parent_session( + self, llm_complexity_router, mock_router_instance, parent_kwargs, expected + ): + """Without the parent's session identity the router mints a fresh trace id for the + sub-call, so the classifier's spend row lands in a session of its own and never + appears in the trace of the request that triggered it.""" + mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "SIMPLE"}')) + await llm_complexity_router.aclassify("hi", request_kwargs={"metadata": {}, **parent_kwargs}) + call_kwargs = mock_router_instance.acompletion.call_args.kwargs + for key in ("litellm_session_id", "litellm_trace_id"): + assert call_kwargs.get(key) == expected.get(key) + @pytest.mark.asyncio async def test_aclassify_falls_back_to_heuristic_on_llm_exception( self, llm_complexity_router, mock_router_instance @@ -1608,7 +1634,7 @@ class TestLLMClassifier: assert result is not None assert result.model == "o1-preview" # REASONING tier model call_kwargs = mock_router_instance.acompletion.call_args.kwargs - assert call_kwargs["metadata"] == request_metadata + assert call_kwargs["metadata"] == {**request_metadata, "internal_call_origin": "autorouter_classifier"} class TestRouterPreRoutingAliasOverrides: @@ -2285,8 +2311,9 @@ class TestSemanticKeywordTierRules: ) assert result is not None assert fake_router.async_embedding_kwargs, "expected an embedding call for the prompt" - assert fake_router.async_embedding_kwargs[0]["metadata"] == caller_metadata - assert fake_router.async_embedding_kwargs[0]["litellm_metadata"] == caller_litellm_metadata + origin = {"internal_call_origin": "autorouter_classifier"} + assert fake_router.async_embedding_kwargs[0]["metadata"] == {**caller_metadata, **origin} + assert fake_router.async_embedding_kwargs[0]["litellm_metadata"] == {**caller_litellm_metadata, **origin} @pytest.mark.asyncio async def test_semantic_embedding_call_captures_request_body_in_proxy_server_request(self, basic_config): @@ -2395,6 +2422,7 @@ class TestSemanticKeywordTierRules: "user_api_key_hash": "hash-abc", "user_api_key_team_id": "team-1", "user_api_key_auth": {"models": ["voyage-3-5"]}, + "internal_call_origin": "autorouter_classifier", } assert fake_router.async_embedding_kwargs[0]["metadata"] == expected assert fake_router.async_embedding_kwargs[0]["litellm_metadata"] == expected @@ -2730,15 +2758,46 @@ class TestSubCallMetadataSanitization: assert sanitized["user_api_key_auth"] is not None assert _get_budget_reservation_from_metadata(sanitized) is None - def test_returns_empty_dict_for_missing_metadata(self): + def test_absent_parent_bucket_stays_empty(self): + """An absent bucket must not be materialized just to carry the origin. + + The embedding path passes both buckets, and get_litellm_metadata_from_kwargs + prefers litellm_metadata whenever it is truthy, backfilling only user_api_key* + keys from metadata. Returning an origin-only dict here would make a chat + completions parent's empty litellm_metadata win and silently drop + requester_ip_address, tags and spend_logs_metadata from the classifier's row.""" from litellm.router_strategy.complexity_router.complexity_router import ( _classifier_call_metadata, ) for absent in (None, {}): - result = _classifier_call_metadata(absent) - assert result == {} - assert isinstance(result, dict) + assert _classifier_call_metadata(absent) == {} + + def test_classifier_buckets_keep_non_spend_fields_on_a_chat_completions_parent(self): + """Drives the real resolver over the buckets the embedding classifier builds.""" + from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs + from litellm.router_strategy.complexity_router.complexity_router import ( + _classifier_call_metadata, + ) + + parent = { + "user_api_key": "sk-abc", + "requester_ip_address": "10.0.0.1", + "spend_logs_metadata": {"team_note": "keep me"}, + "tags": ["prod"], + } + resolved = get_litellm_metadata_from_kwargs( + { + "litellm_params": { + "metadata": _classifier_call_metadata(parent), + "litellm_metadata": _classifier_call_metadata(None), + } + } + ) + assert resolved["internal_call_origin"] == "autorouter_classifier" + assert resolved["requester_ip_address"] == "10.0.0.1" + assert resolved["spend_logs_metadata"] == {"team_note": "keep me"} + assert resolved["tags"] == ["prod"] def test_sanitized_auth_keeps_access_group_fields_and_leaves_original_untouched(self): from litellm.proxy._types import UserAPIKeyAuth diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 18e35e13d92..b76c6f6c4ae 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -3,7 +3,7 @@ "limit": 23253 }, "LIT002": { - "limit": 27449 + "limit": 27452 }, "LIT003": { "limit": 292