diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 2967fc2a505..a37cb194757 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -57,7 +57,7 @@ "limit": 5601 }, "reportMissingTypeArgument": { - "limit": 15288 + "limit": 15287 }, "reportMissingTypeStubs": { "limit": 40 @@ -105,10 +105,10 @@ "limit": 109 }, "reportUnknownMemberType": { - "limit": 38324 + "limit": 38323 }, "reportUnknownParameterType": { - "limit": 19625 + "limit": 19624 }, "reportUnknownVariableType": { "limit": 29861 diff --git a/enterprise/pyproject.toml b/enterprise/pyproject.toml index 8360c0a077d..b6f482ccd86 100644 --- a/enterprise/pyproject.toml +++ b/enterprise/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-enterprise" -version = "0.1.63" +version = "0.1.64" description = "Package for LiteLLM Enterprise features" readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.1.63" +version = "0.1.64" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-enterprise==", diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index 0944f99ad54..97e9eb66bf2 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-proxy-extras" -version = "0.4.92" +version = "0.4.93" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.4.92" +version = "0.4.93" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-proxy-extras==", diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 7368de1e968..d70f947469a 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -932,7 +932,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): return OpenAiResponsesToChatCompletionStreamIterator(streaming_response, sync_stream, json_mode) def _convert_content_str_to_input_text(self, content: str, role: str) -> dict[str, object]: - if role == "user" or role == "system" or role == "tool": + if role in ("user", "system", "developer", "tool"): return {"type": "input_text", "text": content} else: return {"type": "output_text", "text": content} diff --git a/litellm/constants.py b/litellm/constants.py index ef9329b9dfc..1063b6ddeeb 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -7,6 +7,7 @@ from litellm.litellm_core_utils.env_utils import get_env_int, get_env_int_in_ran DEFAULT_HEALTH_CHECK_PROMPT: Final = str(os.getenv("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm")) AZURE_DEFAULT_RESPONSES_API_VERSION: Final = str(os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview")) +AZURE_OPENAI_AUDIO_PROVIDERS: Final = frozenset({"azure", "azure_ai"}) ROUTER_MAX_FALLBACKS: Final = int(os.getenv("ROUTER_MAX_FALLBACKS", 5)) ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS: Final = 2000 RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset( @@ -1450,6 +1451,7 @@ SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: Final = "_session_deployment_affin CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags" INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin" SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated" +SESSION_ID_OMITTED_METADATA_KEY: Final = "litellm_session_id_omitted" LITELLM_TRUNCATED_PAYLOAD_FIELD: Final = "litellm_truncated" LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE: Final = ( "Truncation is a DB storage safeguard. " diff --git a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py index 5504756ceb8..bf99035a6b1 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py +++ b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py @@ -5,6 +5,8 @@ Helper utilities for tracking the cost of built-in tools. from collections.abc import Mapping from typing import Final, Literal +from pydantic import ValidationError + import litellm from litellm.constants import OPENAI_FILE_SEARCH_COST_PER_1K_CALLS from litellm.litellm_core_utils.llm_cost_calc.utils import ( @@ -13,6 +15,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( from litellm.types.llms.openai import ( FileSearchTool, ResponsesAPIResponse, + ResponsesToolUsage, WebSearchOptions, ) from litellm.types.utils import ( @@ -32,6 +35,17 @@ def _output_item_type(output_item: object) -> str | None: return item_type if isinstance(item_type, str) else None +def _reported_web_search_requests(response_object: ResponsesAPIResponse) -> int | None: + tool_usage: Final = getattr(response_object, "tool_usage", None) + if tool_usage is None: + return None + try: + web_search: Final = ResponsesToolUsage.model_validate(tool_usage).web_search + except ValidationError: + return None + return None if web_search is None else web_search.num_requests + + def _usage_reports_server_side_web_search_calls(usage: Usage) -> bool: details: Final = getattr(usage, "server_side_tool_usage_details", None) if not isinstance(details, Mapping): @@ -182,15 +196,19 @@ class StandardBuiltInToolCostTracking: Providers that report a request count in usage (gemini, anthropic, xai, vertex) are handled by get_cost_for_web_search_request and never reach here. This path prices per call, so it must count - the web_search_call items. Chat-completions responses only expose url_citation annotations with no - count, so they floor to a single billable search. + the web_search_call items, unless the response reports the billable count itself + (Bedrock's tool_usage.web_search.num_requests, which excludes open_page fetches). Chat-completions + responses only expose url_citation annotations with no count, so they floor to a single billable search. """ - if isinstance(response_object, ResponsesAPIResponse): - count = sum( - 1 for output_item in response_object.output if _output_item_type(output_item) == "web_search_call" - ) - return max(count, 1) - return 1 + if not isinstance(response_object, ResponsesAPIResponse): + return 1 + reported: Final = _reported_web_search_requests(response_object) + if reported is not None: + return reported + count: Final = sum( + 1 for output_item in response_object.output if _output_item_type(output_item) == "web_search_call" + ) + return max(count, 1) @staticmethod def _handle_file_search_cost( diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index b34c416cd40..21587af73aa 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -428,7 +428,7 @@ def _coerce_off_peak_rate(value: object, default: float) -> float: return default -def _apply_off_peak_pricing( +def apply_off_peak_pricing( model_info: ModelInfo, current_time: datetime | None, prompt_base_cost: float, @@ -462,7 +462,7 @@ def _apply_off_peak_to_base_costs( has no field for them. """ prompt, completion, cache_creation, cache_creation_above_1hr, cache_read = base_costs - off_peak_prompt, off_peak_completion, off_peak_cache_read = _apply_off_peak_pricing( + off_peak_prompt, off_peak_completion, off_peak_cache_read = apply_off_peak_pricing( model_info, current_time, prompt, completion, cache_read ) return (off_peak_prompt, off_peak_completion, cache_creation, cache_creation_above_1hr, off_peak_cache_read) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 355faa41e68..6ae17bac6ff 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -2337,6 +2337,9 @@ class CustomStreamWrapper: else: self.sent_last_chunk = True processed_chunk: Final = self.finish_reason_handler() + if self.stream_options is None: + usage: Final = calculate_total_usage(chunks=self.chunks) + processed_chunk._hidden_params["usage"] = usage # pyright: ignore[reportPrivateUsage] # sync parity # see sync __next__'s sibling branch: deliberately do NOT restore # here - this chunk is still this call's own data, and restoring # before returning it would corrupt the caller's own log diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 7fe9d3dec52..f2d405e9a17 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -15,6 +15,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( filter_value_from_dict, ) from litellm.llms.azure.common_utils import BaseAzureLLM +from litellm.llms.azure_ai.common_utils import is_foundry_model_inference_base from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj from litellm.llms.openai.common_utils import drop_params_from_unprocessable_entity_error from litellm.llms.openai.openai import OpenAIConfig @@ -207,20 +208,18 @@ class AzureAIStudioConfig(OpenAIConfig): message["content"] = texts return stripped_messages - def _is_azure_openai_model(self, model: str, api_base: str | None) -> bool: - try: - if "/" in model: - model = model.split("/", 1)[1] - if ( - model in litellm.open_ai_chat_completion_models - or model in litellm.open_ai_text_completion_models - or model in litellm.open_ai_embedding_models - ): - return True + def _is_foundry_model_inference_base(self, api_base: str) -> bool: + return is_foundry_model_inference_base(api_base) - except Exception: + def _is_azure_openai_model(self, model: str, api_base: str | None) -> bool: + if api_base is None or self._is_foundry_model_inference_base(api_base): return False - return False + stripped_model: Final = model.split("/", 1)[1] if "/" in model else model + return ( + stripped_model in litellm.open_ai_chat_completion_models + or stripped_model in litellm.open_ai_text_completion_models + or stripped_model in litellm.open_ai_embedding_models + ) def _get_openai_compatible_provider_info( self, diff --git a/litellm/llms/azure_ai/common_utils.py b/litellm/llms/azure_ai/common_utils.py index 26a90157455..aa34bab5b2e 100644 --- a/litellm/llms/azure_ai/common_utils.py +++ b/litellm/llms/azure_ai/common_utils.py @@ -1,5 +1,6 @@ from collections.abc import Mapping from typing import Final, Literal +from urllib.parse import urlparse import litellm from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter @@ -10,6 +11,14 @@ from litellm.types.router import GenericLiteLLMParams AzureAIApiKeyHeader = Literal["Authorization", "api-key", "Api-Key", "Ocp-Apim-Subscription-Key"] +def is_foundry_model_inference_base(api_base: str) -> bool: + parsed: Final = urlparse(api_base) + host: Final = parsed.hostname + if host is None or not host.endswith(".services.ai.azure.com"): + return False + return "/openai/deployments" not in parsed.path + + def get_azure_ai_entra_token(litellm_params: Mapping[str, object] | None = None) -> str | None: """ Resolve an Entra ID / OAuth access token for an Azure AI Foundry deployment. diff --git a/litellm/llms/azure_ai/embed/handler.py b/litellm/llms/azure_ai/embed/handler.py index 65c3997c099..c65edbf56e6 100644 --- a/litellm/llms/azure_ai/embed/handler.py +++ b/litellm/llms/azure_ai/embed/handler.py @@ -1,8 +1,10 @@ from typing import Final +from urllib.parse import urlsplit, urlunsplit from openai import OpenAI import litellm +from litellm.llms.azure_ai.common_utils import is_foundry_model_inference_base from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, HTTPHandler, @@ -16,6 +18,16 @@ from litellm.utils import convert_to_model_response_object from .cohere_transformation import AzureAICohereConfig +def _foundry_models_route_base(api_base: str | None) -> str | None: + if api_base is None or not is_foundry_model_inference_base(api_base): + return api_base + parts: Final = urlsplit(api_base) + path: Final = parts.path.rstrip("/") + if path.endswith("/models"): + return api_base + return urlunsplit((parts.scheme, parts.netloc, f"{path}/models", parts.query, parts.fragment)) + + class AzureAIEmbedding(OpenAIChatCompletion): def _process_response( self, @@ -214,6 +226,7 @@ class AzureAIEmbedding(OpenAIChatCompletion): assemble result in-order, and return """ + resolved_api_base: Final = _foundry_models_route_base(api_base) if aembedding is True: return self.async_embedding( model, @@ -223,7 +236,7 @@ class AzureAIEmbedding(OpenAIChatCompletion): model_response, optional_params, api_key, - api_base, + resolved_api_base, client, ) @@ -245,7 +258,7 @@ class AzureAIEmbedding(OpenAIChatCompletion): model_response=model_response, optional_params=optional_params, api_key=api_key, - api_base=api_base, + api_base=resolved_api_base, client=client, ) @@ -262,7 +275,7 @@ class AzureAIEmbedding(OpenAIChatCompletion): model_response, optional_params, api_key, - api_base, + resolved_api_base, client=(client if client is not None and isinstance(client, OpenAI) else None), aembedding=aembedding, shared_session=shared_session, diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 71a598a6fe7..2896d889f96 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -99,9 +99,12 @@ from litellm.types.containers.main import ( ) from litellm.types.files import StreamingMediaUploadConfig, TwoStepFileUploadConfig from litellm.types.integrations.custom_logger import ( + NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES, AgenticLoopPlan, AgenticLoopRequestPatch, AgenticLoopSafetyError, + converted_stream_requested, + is_interception_internal_key, ) from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, @@ -2760,6 +2763,7 @@ class BaseLLMHTTPHandler: ) if self._has_agentic_completion_hook(logging_obj): + agentic_kwargs: Final = dict(litellm_params) # mutable-ok: agentic hooks mutate kwargs in place final_response: Final = run_async_function( self._call_agentic_completion_hooks, response=initial_response, @@ -2770,10 +2774,19 @@ class BaseLLMHTTPHandler: logging_obj=logging_obj, stream=False, custom_llm_provider=custom_llm_provider, - kwargs=dict(litellm_params), + kwargs=agentic_kwargs, api_surface="responses", ) - return final_response if final_response is not None else initial_response + result: Final = final_response if final_response is not None else initial_response + if converted_stream_requested(agentic_kwargs) and not agentic_kwargs.get("_agentic_loop_depth"): + return self._wrap_responses_response_as_fake_stream( + result=result, + model=model, + responses_api_provider_config=responses_api_provider_config, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + ) + return result return initial_response @@ -2939,6 +2952,7 @@ class BaseLLMHTTPHandler: logging_obj=logging_obj, ) + agentic_kwargs: Final = dict(litellm_params) # mutable-ok: agentic hooks mutate kwargs in place final_response: Final = await self._call_agentic_completion_hooks( response=initial_response, model=model, @@ -2948,15 +2962,12 @@ class BaseLLMHTTPHandler: logging_obj=logging_obj, stream=False, custom_llm_provider=custom_llm_provider, - kwargs=dict(litellm_params), + kwargs=agentic_kwargs, api_surface="responses", ) result: Final = final_response if final_response is not None else initial_response - interception_converted_stream: Final = litellm_params.get( - "_code_interpreter_interception_converted_stream" - ) or litellm_params.get("_websearch_interception_converted_stream") - if interception_converted_stream and not litellm_params.get("_agentic_loop_depth"): + if converted_stream_requested(agentic_kwargs) and not agentic_kwargs.get("_agentic_loop_depth"): return self._wrap_responses_response_as_fake_stream( result=result, model=model, @@ -5420,8 +5431,7 @@ class BaseLLMHTTPHandler: kwargs_for_followup: Final = { k: v for k, v in kwargs.items() - if not k.startswith("_websearch_interception") - and not k.startswith("_compression_interception") + if not is_interception_internal_key(k, prefixes=NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES) and k != "_code_interpreter_interception_converted_stream" and k not in internal_keys and k not in optional_params diff --git a/litellm/llms/dashscope/cost_calculator.py b/litellm/llms/dashscope/cost_calculator.py index dd5bee1fe8b..d8eb1f9f8d7 100644 --- a/litellm/llms/dashscope/cost_calculator.py +++ b/litellm/llms/dashscope/cost_calculator.py @@ -7,11 +7,13 @@ cached, cache-creation, output, reasoning) is billed at that one tier's rate. See https://help.aliyun.com/zh/model-studio/billing-for-model-studio """ -from dataclasses import dataclass +from dataclasses import dataclass, replace +from datetime import datetime from typing import Final from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import select_tier_for_input, tier_rate from litellm.litellm_core_utils.llm_cost_calc.utils import ( + apply_off_peak_pricing, parse_completion_tokens_details, parse_prompt_tokens_details, ) @@ -32,6 +34,19 @@ class TokenBreakdown: return self.text_tokens + self.cached_tokens + self.cache_creation_tokens +@dataclass(frozen=True, slots=True) +class TokenRates: + input_rate: float + cache_read_rate: float + cache_creation_rate: float + output_rate: float + reasoning_rate: float | None + + @property + def billed_reasoning_rate(self) -> float: + return self.output_rate if self.reasoning_rate is None else self.reasoning_rate + + def _extract_token_breakdown(usage: Usage) -> TokenBreakdown: prompt_details: Final = parse_prompt_tokens_details(usage) cached_tokens: Final = prompt_details["cache_hit_tokens"] @@ -57,69 +72,75 @@ def _flat_rate(model_info: ModelInfo, cost_key: str, fallback_cost_key: str) -> return float(value) -def _calculate_prompt_cost( - breakdown: TokenBreakdown, - model_info: ModelInfo, - tier: dict | None, -) -> float: - if tier is not None: - return ( - (breakdown.text_tokens * tier_rate(tier, "input_cost_per_token")) - + (breakdown.cached_tokens * tier_rate(tier, "cache_read_input_token_cost", "input_cost_per_token")) - + ( - breakdown.cache_creation_tokens - * tier_rate(tier, "cache_creation_input_token_cost", "input_cost_per_token") - ) - ) - - input_cost: Final = float(model_info.get("input_cost_per_token") or 0.0) - cache_read_cost: Final = _flat_rate(model_info, "cache_read_input_token_cost", "input_cost_per_token") - cache_creation_cost: Final = _flat_rate(model_info, "cache_creation_input_token_cost", "input_cost_per_token") - - return ( - (breakdown.text_tokens * input_cost) - + (breakdown.cached_tokens * cache_read_cost) - + (breakdown.cache_creation_tokens * cache_creation_cost) +def _flat_rates(model_info: ModelInfo) -> TokenRates: + reasoning_rate: Final = model_info.get("output_cost_per_reasoning_token") + return TokenRates( + input_rate=float(model_info.get("input_cost_per_token") or 0.0), + cache_read_rate=_flat_rate(model_info, "cache_read_input_token_cost", "input_cost_per_token"), + cache_creation_rate=_flat_rate(model_info, "cache_creation_input_token_cost", "input_cost_per_token"), + output_rate=float(model_info.get("output_cost_per_token") or 0.0), + reasoning_rate=None if reasoning_rate is None else float(reasoning_rate), ) -def _calculate_completion_cost( - breakdown: TokenBreakdown, - model_info: ModelInfo, - tier: dict | None, -) -> float: +def _tier_rates(model_info: ModelInfo, tier: dict) -> TokenRates: # A tier that declares output rates keeps the request on them, all-or-nothing. A tier table # spelling out only input rates would serve every completion for free, so there the model's # own output rates stand in - tier_declares_output: Final = tier is not None and "output_cost_per_token" in tier - output_cost: Final = ( - tier_rate(tier, "output_cost_per_token") - if tier_declares_output - else float(model_info.get("output_cost_per_token") or 0.0) - ) - tier_declares_reasoning: Final = tier is not None and "output_cost_per_reasoning_token" in tier - model_reasoning_rate: Final = None if tier_declares_output else model_info.get("output_cost_per_reasoning_token") - reasoning_cost: Final = ( - tier_rate(tier, "output_cost_per_reasoning_token", "output_cost_per_token") - if tier_declares_reasoning - else float(model_reasoning_rate) - if model_reasoning_rate is not None - else output_cost + flat_rates: Final = _flat_rates(model_info) + tier_declares_output: Final = "output_cost_per_token" in tier + tier_declares_reasoning: Final = "output_cost_per_reasoning_token" in tier + return TokenRates( + input_rate=tier_rate(tier, "input_cost_per_token"), + cache_read_rate=tier_rate(tier, "cache_read_input_token_cost", "input_cost_per_token"), + cache_creation_rate=tier_rate(tier, "cache_creation_input_token_cost", "input_cost_per_token"), + output_rate=tier_rate(tier, "output_cost_per_token") if tier_declares_output else flat_rates.output_rate, + reasoning_rate=( + tier_rate(tier, "output_cost_per_reasoning_token") + if tier_declares_reasoning + else None + if tier_declares_output + else flat_rates.reasoning_rate + ), ) - return (breakdown.completion_tokens * output_cost) + (breakdown.reasoning_tokens * reasoning_cost) + +def _off_peak_rates(model_info: ModelInfo, current_time: datetime | None, rates: TokenRates) -> TokenRates: + input_rate, output_rate, cache_read_rate = apply_off_peak_pricing( + model_info, current_time, rates.input_rate, rates.output_rate, rates.cache_read_rate + ) + return replace(rates, input_rate=input_rate, output_rate=output_rate, cache_read_rate=cache_read_rate) -def cost_per_token(model: str, usage: Usage, custom_llm_provider: str = "dashscope") -> tuple[float, float]: +def _bill(breakdown: TokenBreakdown, rates: TokenRates) -> tuple[float, float]: + prompt_cost: Final = ( + (breakdown.text_tokens * rates.input_rate) + + (breakdown.cached_tokens * rates.cache_read_rate) + + (breakdown.cache_creation_tokens * rates.cache_creation_rate) + ) + completion_cost: Final = (breakdown.completion_tokens * rates.output_rate) + ( + breakdown.reasoning_tokens * rates.billed_reasoning_rate + ) + return prompt_cost, completion_cost + + +def cost_per_token( + model: str, + usage: Usage, + custom_llm_provider: str = "dashscope", + current_time: datetime | None = None, +) -> tuple[float, float]: """ Calculate cost per token for Dashscope models. - Supports both tiered and flat pricing with cached and reasoning tokens. + Supports both tiered and flat pricing with cached and reasoning tokens, and swaps in the + model's off_peak_pricing rates while one of its windows is open. Args: model: Model name without provider prefix usage: LiteLLM Usage block custom_llm_provider: The provider id the request resolved to; dashscope or one of its brand aliases + current_time: The moment the request is billed at; defaults to now, UTC Returns: Tuple[float, float] - (prompt_cost_in_usd, completion_cost_in_usd) @@ -133,8 +154,7 @@ def cost_per_token(model: str, usage: Usage, custom_llm_provider: str = "dashsco if tiered_pricing else None ) + standard_rates: Final = _flat_rates(model_info) if tier is None else _tier_rates(model_info, tier) + rates: Final = _off_peak_rates(model_info, current_time, standard_rates) - prompt_cost: Final = _calculate_prompt_cost(breakdown=breakdown, model_info=model_info, tier=tier) - completion_cost: Final = _calculate_completion_cost(breakdown=breakdown, model_info=model_info, tier=tier) - - return prompt_cost, completion_cost + return _bill(breakdown, rates) diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index bcd4ea43243..2db6d78a218 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -11,6 +11,7 @@ import time import uuid from collections.abc import AsyncIterator, Iterator, Mapping from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, Optional +from urllib.parse import urlsplit import httpx import openai @@ -43,6 +44,14 @@ _OPENAI_INIT_PARAMS: Final[tuple[str, ...]] = _get_client_init_params(OpenAI) _AZURE_OPENAI_INIT_PARAMS: Final[tuple[str, ...]] = _get_client_init_params(AzureOpenAI) +_OPENAI_API_HOST: Final[str] = "api.openai.com" + + +def is_openai_backed_api_base(api_base: str) -> bool: + hostname: Final = urlsplit(api_base).hostname + return hostname is not None and (hostname == _OPENAI_API_HOST or hostname.endswith(f".{_OPENAI_API_HOST}")) + + class OpenAIError(BaseLLMException): def __init__( self, diff --git a/litellm/llms/openai/image_generation/gpt_transformation.py b/litellm/llms/openai/image_generation/gpt_transformation.py index 05494c497ca..090b2eba387 100644 --- a/litellm/llms/openai/image_generation/gpt_transformation.py +++ b/litellm/llms/openai/image_generation/gpt_transformation.py @@ -82,8 +82,8 @@ class GPTImageGenerationConfig(BaseImageGenerationConfig): ) # set optional params - image_response.size = optional_params.get("size", "1024x1024") # default is always 1024x1024 - image_response.quality = optional_params.get("quality", "high") # always hd for dall-e-3 - image_response.output_format = optional_params.get("response_format", "png") # always png for dall-e-3 + image_response.size = image_response.size or optional_params.get("size", "1024x1024") + image_response.quality = image_response.quality or optional_params.get("quality", "high") + image_response.output_format = image_response.output_format or optional_params.get("output_format", "png") return image_response diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 1cfc6e06ee9..edc8d64d9c2 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -2,7 +2,6 @@ import time import types from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Iterator, Mapping from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast -from urllib.parse import urlparse import httpx @@ -55,6 +54,7 @@ from .common_utils import ( OpenAIError, build_output_token_limit_response, drop_params_from_unprocessable_entity_error, + is_openai_backed_api_base, is_output_token_limit_error, ) from .workload_identity import resolve_openai_workload_identity_config @@ -1190,10 +1190,8 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): """ if stream_options is not None: return {"stream_options": stream_options} - else: - # by default litellm will include usage for openai endpoints - if api_base is None or urlparse(api_base).hostname == "api.openai.com": - return {"stream_options": {"include_usage": True}} + if api_base is None or is_openai_backed_api_base(api_base): + return {"stream_options": {"include_usage": True}} return {} # Embedding diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index a0db7aadb9e..ecc89c5f135 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -33,8 +33,9 @@ import time import uuid from collections.abc import Mapping, Sequence from dataclasses import dataclass +from itertools import accumulate from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Union, cast +from typing import TYPE_CHECKING, Any, Final, NamedTuple, Union, cast from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall from pydantic import BaseModel, TypeAdapter @@ -42,6 +43,7 @@ from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, OpenAiResponsesToChatCompletionStreamIterator, ) from litellm.llms.base_llm.guardrail_translation.base_translation import ( @@ -74,6 +76,7 @@ from litellm.types.llms.openai import ( OutputTextDoneEvent, ResponseAPIUsage, ResponseCompletedEvent, + ResponsesAPIOptionalRequestParams, ResponsesAPIResponse, ResponsesAPIStreamEvents, ResponsesAPIStreamingResponse, @@ -115,6 +118,199 @@ class ResponsesStreamChunk(TypedDict, total=False): content_index: ReadOnly[int] +_PATCHABLE_ITEM_FIELDS: Final[Mapping[str, str]] = MappingProxyType( + {"function_call_output": "output", "message": "content"} +) + +_EMPTY_RESPONSES_REQUEST: Final[ResponsesAPIOptionalRequestParams] = {} + + +def _item_rewrite_field(item: Mapping[str, object]) -> str | None: + item_type: Final = item.get("type") + if item_type is None: + return "content" if "content" in item else None + if not isinstance(item_type, str): + return None + return _PATCHABLE_ITEM_FIELDS.get(item_type) + + +def _rewritten_input_item(item: Mapping[str, object], rewritten: object) -> Mapping[str, object] | None: + field: Final = _item_rewrite_field(item) + if field is None or not isinstance(rewritten, Mapping): + return None + rewritten_content: Final = rewritten.get("content") + if isinstance(item.get(field), str) and isinstance(rewritten_content, str): + return {**item, field: rewritten_content} # mutable-ok: request input items must stay JSON-plain dicts + rewritten_row: Final = cast("AllMessageValues", rewritten) # cast-ok: guardrails hand back chat-shaped rows + converted_items, _ = LiteLLMResponsesTransformationHandler().convert_chat_completion_messages_to_responses_api( + [rewritten_row] # mutable-ok: converter signature takes a list + ) + if len(converted_items) != 1 or not isinstance(converted_items[0], Mapping): + return None + first_converted: Final = cast("Mapping[str, object]", converted_items[0]) # cast-ok: isinstance-checked above + converted_value: Final = first_converted.get(field) + if converted_value is None: + return None + return {**item, field: converted_value} # mutable-ok: request input items must stay JSON-plain dicts + + +def _is_function_call_item(item: object) -> bool: + return isinstance(item, Mapping) and item.get("type") in ("function_call", "custom_tool_call") + + +def _last_message_role(messages: Sequence[object]) -> str | None: + if not messages: + return None + last: Final = messages[-1] + role: Final = last.get("role") if isinstance(last, Mapping) else getattr(last, "role", None) + return role if isinstance(role, str) else None + + +def _provenance_unit_bounds( + raw_input: Sequence[object], + solo_conversions: Sequence[Sequence[object]], +) -> tuple[tuple[int, int], ...]: + trailing_roles: Final = tuple( + accumulate( + (_last_message_role(messages) for messages in solo_conversions), + lambda previous, current: current if current is not None else previous, + ) + ) + start_indexes: Final = tuple( + index + for index in range(len(raw_input)) + if index == 0 or not (_is_function_call_item(raw_input[index]) and trailing_roles[index - 1] == "assistant") + ) + return tuple(zip(start_indexes, (*start_indexes[1:], len(raw_input)))) + + +def _input_item_provenance( + raw_input: Sequence[object], + expected_messages: Sequence[object], +) -> tuple[Mapping[int, int], frozenset[int]] | None: + if not all(isinstance(item, Mapping) for item in raw_input): + return None + solo_conversions: Final = tuple( + LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=cast("ResponseInputParam", [item]), # cast-ok: items checked as Mappings above + responses_api_request=_EMPTY_RESPONSES_REQUEST, + ) + for item in raw_input + ) + full_conversion: Final = tuple( + LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=cast("ResponseInputParam", list(raw_input)), # cast-ok: items checked as Mappings above + responses_api_request=_EMPTY_RESPONSES_REQUEST, + ) + ) + if full_conversion != tuple(expected_messages): + return None + units: Final = _provenance_unit_bounds(raw_input, solo_conversions) + unit_messages: Final = tuple( + tuple(solo_conversions[start]) + if end - start == 1 + else tuple( + LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=cast("ResponseInputParam", list(raw_input[start:end])), # cast-ok: checked as Mappings above + responses_api_request=_EMPTY_RESPONSES_REQUEST, + ) + ) + for start, end in units + ) + if tuple(message for messages in unit_messages for message in messages) != full_conversion: + return None + boundaries: Final = tuple(accumulate((len(messages) for messages in unit_messages), initial=0)) + item_for_message: Final = MappingProxyType( + { + message_index: start + for unit_index, (start, end) in enumerate(units) + if end - start == 1 + for message_index in range(boundaries[unit_index], boundaries[unit_index + 1]) + } + ) + tainted: Final = frozenset( + message_index + for unit_index, (start, end) in enumerate(units) + if end - start > 1 + for message_index in range(boundaries[unit_index], boundaries[unit_index + 1]) + ) + return item_for_message, tainted + + +class _RequestFields(NamedTuple): + input: tuple[object, ...] + instructions: str | None + + +class _ExtractedInputs(NamedTuple): + inputs: GenericGuardrailAPIInputs + task_mappings: tuple[tuple[int, int | None], ...] + + +def _patched_request_fields( + raw_input: object, + instructions: object, + original_messages: Sequence[object], + structured_messages: Sequence[object], +) -> _RequestFields | None: + if not isinstance(raw_input, list) or len(original_messages) != len(structured_messages): + return None + offset: Final = 1 if instructions else 0 + provenance: Final = _input_item_provenance(raw_input, tuple(original_messages)[offset:]) + if provenance is None: + return None + item_for_message, tainted = provenance + changed: Final = tuple( + (index, rewritten) + for index, (original, rewritten) in enumerate(zip(original_messages, structured_messages)) + if original != rewritten + ) + instruction_rewrites: Final = tuple(rewritten for index, rewritten in changed if index < offset) + rewritten_instructions: Final = ( + instruction_rewrites[0].get("content") + if instruction_rewrites and isinstance(instruction_rewrites[0], Mapping) + else instructions + ) + instructions_value: Final = rewritten_instructions if isinstance(rewritten_instructions, str) else None + if rewritten_instructions is not None and instructions_value is None: + return None + body_changes: Final = tuple((index - offset, rewritten) for index, rewritten in changed if index >= offset) + if any(message_index in tainted or message_index not in item_for_message for message_index, _ in body_changes): + return None + replacements: Final = MappingProxyType( + { + item_for_message[message_index]: _rewritten_input_item( + cast("Mapping[str, object]", raw_input[item_for_message[message_index]]), # cast-ok: checked Mappings + rewritten, + ) + for message_index, rewritten in body_changes + } + ) + if len(replacements) != len(body_changes) or any(item is None for item in replacements.values()): + return None + return _RequestFields( + input=tuple(replacements.get(index, item) for index, item in enumerate(raw_input)), + instructions=instructions_value, + ) + + +def _patch_or_convert_request_fields( + raw_input: object, + instructions: object, + original_messages: Sequence[object], + structured_messages: Sequence[AllMessageValues], +) -> _RequestFields | None: + if not isinstance(structured_messages, list): + return None + patched: Final = _patched_request_fields(raw_input, instructions, original_messages, structured_messages) + if patched is not None: + return patched + input_items, converted_instructions = ( + LiteLLMResponsesTransformationHandler().convert_chat_completion_messages_to_responses_api(structured_messages) + ) + return _RequestFields(input=tuple(input_items), instructions=converted_instructions) + + def _next_stream_sequence_number(responses_so_far: Sequence[Any] | None) -> int: sequence_numbers: Final = ( item.get("sequence_number") if isinstance(item, dict) else getattr(item, "sequence_number", None) @@ -162,9 +358,8 @@ class OpenAIResponsesHandler(BaseTranslation): Handles both string input and list of message objects. """ input_data: Final[str | ResponseInputParam | None] = data.get("input") - if input_data is None: + if not isinstance(input_data, (str, list)): return data - structured_messages: Final = self.get_structured_messages(data) raw_tools: Final = data.get("tools") original_tools: Final[tuple[Mapping[str, object], ...]] = ( @@ -173,94 +368,93 @@ class OpenAIResponsesHandler(BaseTranslation): flattened_tool_groups: Final = tuple( form.chat_tools for form in LiteLLMCompletionResponsesConfig.responses_tools_to_chat_forms(original_tools) ) - flattened_tools: Final = tuple( - cast(ChatCompletionToolParam, tool) # cast-ok: mcp tools ride along in the guardrail's tool list - for group in flattened_tool_groups - for tool in group - ) - tools_to_check: Final[list[ChatCompletionToolParam]] = list( # mutable-ok: guardrail inputs want a list - copy.deepcopy(flattened_tools) - ) - - # Handle simple string input - if isinstance(input_data, str): - inputs = GenericGuardrailAPIInputs(texts=[input_data]) - if tools_to_check: - inputs["tools"] = tools_to_check - if structured_messages: - inputs["structured_messages"] = structured_messages - # Include model information if available - model = data.get("model") - if model: - inputs["model"] = model - - guardrailed_inputs = await guardrail_to_apply.apply_guardrail( - inputs=inputs, - request_data=data, - input_type="request", - logging_obj=litellm_logging_obj, - ) - guardrailed_texts = guardrailed_inputs.get("texts", []) - data["input"] = guardrailed_texts[0] if guardrailed_texts else input_data - self._apply_guardrailed_tools_to_data( - data, original_tools, flattened_tool_groups, guardrailed_inputs.get("tools") - ) - verbose_proxy_logger.debug("OpenAI Responses API: Processed string input") - return data - - # Handle list input (ResponseInputParam) - if not isinstance(input_data, list): + extracted: Final = self._extract_guardrail_inputs(data, input_data, flattened_tool_groups) + if not extracted.inputs.get("texts"): return data + if structured_messages: + extracted.inputs["structured_messages"] = structured_messages + guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail( + inputs=extracted.inputs, + request_data=data, + input_type="request", + logging_obj=litellm_logging_obj, + ) + self._apply_guardrailed_tools_to_data( + data, original_tools, flattened_tool_groups, guardrailed_inputs.get("tools") + ) + written_back: Final = self._written_back_request_fields(data, structured_messages, guardrailed_inputs) + if written_back is not None: + data["input"] = list(written_back.input) # mutable-ok: JSON body + if written_back.instructions is None: + data.pop("instructions", None) + else: + data["instructions"] = written_back.instructions # rebind-ok: data is an out-param + elif isinstance(input_data, str): + guardrailed_texts: Final = guardrailed_inputs.get("texts") or () + data["input"] = guardrailed_texts[0] if guardrailed_texts else input_data # rebind-ok: data is an out-param + else: + await self._apply_guardrail_responses_to_input( + messages=input_data, + responses=guardrailed_inputs.get("texts") or (), + task_mappings=extracted.task_mappings, + ) + verbose_proxy_logger.debug("OpenAI Responses API: Processed input messages: %s", data.get("input")) + return data + def _extract_guardrail_inputs( + self, + data: Mapping[str, object], + input_data: "str | ResponseInputParam", + flattened_tool_groups: Sequence[Sequence[Mapping[str, object]]], + ) -> _ExtractedInputs: texts_to_check: Final[list[str]] = [] images_to_check: Final[list[str]] = [] task_mappings: Final[list[tuple[int, int | None]]] = [] - - # Step 1: Extract all text content, images, and tools - for msg_idx, message in enumerate(input_data): - self._extract_input_text_and_images( - message=message, - msg_idx=msg_idx, - texts_to_check=texts_to_check, - images_to_check=images_to_check, - task_mappings=task_mappings, + tools_to_check: Final[list[ChatCompletionToolParam]] = list( # mutable-ok: guardrail inputs want a list + copy.deepcopy( + tuple( + cast(ChatCompletionToolParam, tool) # cast-ok: mcp tools ride along in the guardrail's tool list + for group in flattened_tool_groups + for tool in group + ) ) + ) + if isinstance(input_data, str): + texts_to_check.append(input_data) + else: + for msg_idx, message in enumerate(input_data): + self._extract_input_text_and_images( + message=message, + msg_idx=msg_idx, + texts_to_check=texts_to_check, + images_to_check=images_to_check, + task_mappings=task_mappings, + ) + inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check) + if images_to_check: + inputs["images"] = images_to_check + if tools_to_check: + inputs["tools"] = tools_to_check + model: Final = data.get("model") + if isinstance(model, str): + inputs["model"] = model + return _ExtractedInputs(inputs=inputs, task_mappings=tuple(task_mappings)) - # Step 2: Apply guardrail to all texts in batch - if texts_to_check: - inputs = GenericGuardrailAPIInputs(texts=texts_to_check) - if images_to_check: - inputs["images"] = images_to_check - if tools_to_check: - inputs["tools"] = tools_to_check - if structured_messages: - inputs["structured_messages"] = structured_messages - # Include model information if available - model = data.get("model") - if model: - inputs["model"] = model - guardrailed_inputs = await guardrail_to_apply.apply_guardrail( - inputs=inputs, - request_data=data, - input_type="request", - logging_obj=litellm_logging_obj, - ) - - guardrailed_texts = guardrailed_inputs.get("texts", []) - self._apply_guardrailed_tools_to_data( - data, original_tools, flattened_tool_groups, guardrailed_inputs.get("tools") - ) - - # Step 3: Map guardrail responses back to original input structure - await self._apply_guardrail_responses_to_input( - messages=input_data, - responses=guardrailed_texts, - task_mappings=task_mappings, - ) - - verbose_proxy_logger.debug("OpenAI Responses API: Processed input messages: %s", input_data) - - return data + @staticmethod + def _written_back_request_fields( + data: Mapping[str, object], + structured_messages: Sequence[AllMessageValues] | None, + guardrailed_inputs: GenericGuardrailAPIInputs, + ) -> _RequestFields | None: + guardrailed: Final = guardrailed_inputs.get("structured_messages") + if guardrailed is None or guardrailed is structured_messages: + return None + return _patch_or_convert_request_fields( + data.get("input"), + data.get("instructions"), + structured_messages or (), + guardrailed, + ) def extract_request_tool_names(self, data: dict) -> list[str]: """Extract tool names from Responses API request (tools[].name for function @@ -331,8 +525,8 @@ class OpenAIResponsesHandler(BaseTranslation): async def _apply_guardrail_responses_to_input( self, messages: Any, # Can be List[Dict[str, Any]] or ResponseInputParam - responses: list[str], - task_mappings: list[tuple[int, int | None]], + responses: Sequence[str], + task_mappings: Sequence[tuple[int, int | None]], ) -> None: """ Apply guardrail responses back to input messages. diff --git a/litellm/main.py b/litellm/main.py index b460393c49d..71e69789a2c 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -26,6 +26,7 @@ from copy import deepcopy from functools import partial from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args +from urllib.parse import urlsplit from litellm._logging import _redact_string from litellm._uuid import uuid @@ -60,6 +61,7 @@ if TYPE_CHECKING: from litellm.types.utils import TokenCountResponse from litellm.constants import ( + AZURE_OPENAI_AUDIO_PROVIDERS, DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT, ) @@ -984,6 +986,12 @@ def mock_completion( _OPENAI_DEFAULT_API_BASE: Final = "https://api.openai.com/v1" +_OPENAI_API_HOST: Final = "api.openai.com" + + +def _is_openai_backed_api_base(api_base: str) -> bool: + hostname: Final = urlsplit(api_base).hostname + return hostname is not None and (hostname == _OPENAI_API_HOST or hostname.endswith(f".{_OPENAI_API_HOST}")) def _resolve_openai_api_base(api_base: str | None) -> str: @@ -1053,7 +1061,7 @@ def responses_api_bridge_check( # natively by Chat Completions with reasoning on, so custom-only requests stay on # chat and keep their native custom tool_call response shape. # - The UNSET-effort arm only fires against endpoints known to enforce that - # constraint (the default OpenAI endpoint, or Azure OpenAI where api_base is + # constraint (any api.openai.com host, or Azure OpenAI where api_base is # always set): chat-only OpenAI-compatible backends registered under the openai # provider with a custom api_base and gpt-5.4+ model names serve tools without # reasoning fine and have no /responses route, so they keep pre-existing @@ -1068,14 +1076,15 @@ def responses_api_bridge_check( reasoning_active = reasoning_effort.get("effort") != "none" or reasoning_effort.get("summary") is not None else: reasoning_active = reasoning_effort != "none" - # The reasoning+tools constraint is enforced only by the real OpenAI endpoint (and Azure OpenAI). - # Resolve the effective base arg>global>env>default exactly as the chat handler does, so a custom - # base set via litellm.api_base or OPENAI_BASE_URL/OPENAI_API_BASE isn't misread as the default and - # bridged to a /responses route it lacks. A whitespace-only base collapses to the default too. - resolved_api_base: Final = _resolve_openai_api_base(api_base) - on_constraint_enforcing_endpoint: Final = custom_llm_provider == "azure" or resolved_api_base.strip() in ( - "", - _OPENAI_DEFAULT_API_BASE, + # The reasoning+tools constraint is enforced by the real OpenAI backend behind any api.openai.com + # host (the default URL or a PrivateLink hostname such as .privatelink.api.openai.com) and + # by Azure OpenAI. Resolve the effective base arg>global>env>default exactly as the chat handler + # does, so a custom base set via litellm.api_base or OPENAI_BASE_URL/OPENAI_API_BASE isn't misread + # as the default and bridged to a /responses route it lacks. A whitespace-only base collapses to + # the default too. + resolved_api_base: Final = _resolve_openai_api_base(api_base).strip() + on_constraint_enforcing_endpoint: Final = ( + custom_llm_provider == "azure" or resolved_api_base == "" or _is_openai_backed_api_base(resolved_api_base) ) if ( custom_llm_provider in ("openai", "azure") @@ -7769,7 +7778,7 @@ def transcription( provider=LlmProviders(custom_llm_provider), ) - if custom_llm_provider == "azure" and provider_config is None: + if custom_llm_provider in AZURE_OPENAI_AUDIO_PROVIDERS and provider_config is None: # azure configs api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") @@ -8056,7 +8065,10 @@ def speech( custom_llm_provider=custom_llm_provider, ) response: HttpxBinaryResponseContent | Coroutine[object, object, HttpxBinaryResponseContent] | None = None - if custom_llm_provider == "openai" or custom_llm_provider in litellm.openai_compatible_providers: + if custom_llm_provider == "openai" or ( + custom_llm_provider in litellm.openai_compatible_providers + and custom_llm_provider not in AZURE_OPENAI_AUDIO_PROVIDERS + ): if voice is None or not (isinstance(voice, str)): raise litellm.BadRequestError( message="'voice' is required to be passed as a string for OpenAI TTS", @@ -8110,7 +8122,7 @@ def speech( aspeech=aspeech, shared_session=shared_session, ) - elif custom_llm_provider == "azure": + elif custom_llm_provider in AZURE_OPENAI_AUDIO_PROVIDERS: # Check if this is Azure Speech Service (Cognitive Services TTS) if model.startswith("speech/"): from litellm.llms.azure.text_to_speech.transformation import ( diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index cd187760fd2..9ed0b90c516 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -29277,6 +29277,75 @@ "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, + "gpt-6-astra": { + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05, + "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05, + "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-05, + "cache_creation_input_token_cost_flex": 6.25e-06, + "cache_creation_input_token_cost_priority": 2.5e-05, + "cache_read_input_token_cost": 1e-06, + "cache_read_input_token_cost_above_272k_tokens": 2e-06, + "cache_read_input_token_cost_above_272k_tokens_flex": 1e-06, + "cache_read_input_token_cost_above_272k_tokens_priority": 4e-06, + "cache_read_input_token_cost_flex": 5e-07, + "cache_read_input_token_cost_priority": 2e-06, + "input_cost_per_token": 1e-05, + "input_cost_per_token_above_272k_tokens": 2e-05, + "input_cost_per_token_above_272k_tokens_flex": 1e-05, + "input_cost_per_token_above_272k_tokens_priority": 4e-05, + "input_cost_per_token_batches": 5e-06, + "input_cost_per_token_flex": 5e-06, + "input_cost_per_token_priority": 2e-05, + "litellm_provider": "openai", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-05, + "output_cost_per_token_above_272k_tokens": 7.5e-05, + "output_cost_per_token_above_272k_tokens_flex": 3.75e-05, + "output_cost_per_token_above_272k_tokens_priority": 0.00015, + "output_cost_per_token_batches": 2.5e-05, + "output_cost_per_token_flex": 2.5e-05, + "output_cost_per_token_priority": 0.0001, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_native_streaming": true, + "supports_none_reasoning_effort": false, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "supports_xhigh_reasoning_effort": true + }, "gpt-5.6": { "cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, @@ -52911,6 +52980,11 @@ "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "output_cost_per_token": 3.3e-05, "output_cost_per_token_above_272k_tokens": 4.95e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.012, + "search_context_size_low": 0.012, + "search_context_size_medium": 0.012 + }, "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -52945,6 +53019,11 @@ "cache_read_input_token_cost_above_272k_tokens": 4.4e-07, "output_cost_per_token": 1.32e-05, "output_cost_per_token_above_272k_tokens": 1.98e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.012, + "search_context_size_low": 0.012, + "search_context_size_medium": 0.012 + }, "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -53007,6 +53086,11 @@ "cache_read_input_token_cost_above_272k_tokens": 4.4e-08, "output_cost_per_token": 1.32e-06, "output_cost_per_token_above_272k_tokens": 1.98e-06, + "search_context_cost_per_query": { + "search_context_size_high": 0.012, + "search_context_size_low": 0.012, + "search_context_size_medium": 0.012 + }, "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -53195,6 +53279,11 @@ "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "output_cost_per_token": 3.3e-05, "output_cost_per_token_above_272k_tokens": 4.95e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.012, + "search_context_size_low": 0.012, + "search_context_size_medium": 0.012 + }, "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -53226,6 +53315,11 @@ "cache_read_input_token_cost_above_272k_tokens": 5.5e-07, "output_cost_per_token": 1.65e-05, "output_cost_per_token_above_272k_tokens": 2.475e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.012, + "search_context_size_low": 0.012, + "search_context_size_medium": 0.012 + }, "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 97f5f59d2dc..5d5a25e7cd6 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1216,7 +1216,7 @@ class GenerateKeyRequest(KeyRequestBase): organization_id: str | None = None project_id: str | None = None - @field_validator("team_id", "organization_id", mode="before") + @field_validator("team_id", "organization_id", "project_id", mode="before") @classmethod def treat_cleared_id_as_unset(cls, v: object) -> object: if v == "": @@ -2608,9 +2608,9 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): None, description="When set to True, rejects requests that contain client-side 'metadata.tags' to prevent users from influencing budgets by sending different tags. Tags can only be inherited from the API key metadata.", ) - missing_session_id: Literal["generate", "reject"] | None = Field( + missing_session_id: Literal["generate", "reject", "omit"] | None = Field( None, - description="What to do with LLM API requests that carry no session id (x-litellm-session-id header, metadata.session_id, etc.). 'generate' stamps one id into litellm_session_id, litellm_trace_id and metadata.session_id so SpendLogs and logging callbacks agree; 'reject' returns 400. Unset keeps the legacy behavior where SpendLogs falls back to the trace id while callbacks get no session id.", + description="What to do with LLM API requests that carry no session id (x-litellm-session-id header, metadata.session_id, etc.). 'generate' stamps one id into litellm_session_id, litellm_trace_id and metadata.session_id so SpendLogs and logging callbacks agree; 'reject' returns 400; 'omit' leaves SpendLogs.session_id null, matching callbacks such as Langfuse that only record a client-established metadata.session_id. Unset keeps the legacy behavior where SpendLogs falls back to the trace id while callbacks get no session id.", ) enable_public_model_hub: bool = Field( default=False, @@ -4239,6 +4239,8 @@ class TeamAccessGroupModelGrant(LiteLLMPydanticObjectBase): access_group_id: str access_group_name: str models: tuple[str, ...] + mcp_server_ids: tuple[str, ...] = () + agent_ids: tuple[str, ...] = () class TeamInfoResponseObjectTeamTable(LiteLLM_TeamTable): diff --git a/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py b/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py index 1fc3c06e6bf..93d859066b0 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py +++ b/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py @@ -916,9 +916,10 @@ class CompresrGuardrail(CustomGuardrail): def _mirror_texts_channel(input_texts: object, applied: _CompressionResult) -> list[object] | None: """Compressed content mirrored into the Responses `texts` channel. - The chat/Anthropic handlers round-trip ``structured_messages``; the - Responses translation cannot rebuild its input from chat messages and - instead writes back through ``texts``. This matches by value, so a + The chat/Anthropic/Responses handlers round-trip + ``structured_messages``; translations without that round-trip write + back through ``texts``, so the compressed content is mirrored there + too. This matches by value, so a replacement is applied only when it is unambiguous: one compression per text, and every occurrence in ``texts`` accounted for by a compressed target. Anything else is left uncompressed rather than risk a wrong or diff --git a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py index fc881a60f43..9d993384461 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py +++ b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py @@ -50,6 +50,9 @@ if TYPE_CHECKING: from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel BYPASS_HEADER: Final = "x-headroom-bypass" +_STREAM_CONVERTIBLE_CALL_TYPES: Final = frozenset( + (CallTypes.completion, CallTypes.acompletion, CallTypes.responses, CallTypes.aresponses) +) HEADROOM_RETRIEVE_TOOL_NAME: Final = "headroom_retrieve" _HASH_PATTERN: Final = re.compile(r"hash=([a-f0-9]{24})") _HASH_CACHE_TTL_SECONDS: Final = 15 * 60 @@ -725,6 +728,10 @@ class HeadroomGuardrail(CustomGuardrail): verbose_proxy_logger.debug("Headroom: %s header set; skipping compression", BYPASS_HEADER) return inputs + if request_data.get("background"): + verbose_proxy_logger.debug("Headroom: background request; skipping compression") + return inputs + structured_messages: Final = inputs.get("structured_messages") if not _is_object_list(structured_messages) or not structured_messages: return inputs @@ -826,9 +833,9 @@ class HeadroomGuardrail(CustomGuardrail): ) -> dict[str, Any] | None: # mutable-ok: overrides CustomLogger hook whose contract is a plain dict base_result: Final = await super().async_pre_call_deployment_hook(kwargs, call_type) effective: Final = base_result if base_result is not None else kwargs - if call_type not in (CallTypes.completion, CallTypes.acompletion): + if call_type not in _STREAM_CONVERTIBLE_CALL_TYPES: return base_result - if not effective.get("stream"): + if not effective.get("stream") or effective.get("background"): return base_result if not has_headroom_retrieve_tool(effective.get("tools")): return base_result diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 47aafda2337..7254b05db2e 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -168,11 +168,8 @@ class _ProxyDBLogger(CustomLogger): "custom_llm_provider" ) or request_data.get("custom_llm_provider", "") - # Propagate standard_logging_object and litellm_trace_id from the - # Logging instance so that _get_session_id_for_spend_log uses the same - # trace_id that Langfuse received (via async_failure_handler). - # Without this, the DB session_id would be a random UUID that doesn't - # match the Langfuse trace_id, making failed requests unsearchable. + # Propagate standard_logging_object and litellm_trace_id from the Logging + # instance so the failure row carries the same trace_id Langfuse received. _litellm_logging_obj: Final = request_data.get("litellm_logging_obj") if _litellm_logging_obj is not None: if not request_data.get("standard_logging_object"): diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 1d440448c2f..f752d7cfa89 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -25,6 +25,7 @@ from litellm.constants import ( PRE_CALL_EXECUTED_GUARDRAILS_KEY, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY, + SESSION_ID_OMITTED_METADATA_KEY, ) from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( @@ -733,12 +734,18 @@ def apply_missing_session_id_policy( general_settings: Mapping[str, object] | None, request: Request, ) -> None: + for metadata_key in ("metadata", "litellm_metadata"): + if isinstance(client_metadata := data.get(metadata_key), dict): + client_metadata.pop(SESSION_ID_OMITTED_METADATA_KEY, None) + metadata: Final = data.get(_metadata_variable_name) policy: Final = general_settings.get("missing_session_id") if general_settings else None if policy is None or not _is_llm_inference_route(request): return - metadata: Final = data.get(_metadata_variable_name) if not isinstance(metadata, dict): return + if policy == "omit": + metadata[SESSION_ID_OMITTED_METADATA_KEY] = True + return if data.get("litellm_session_id") or metadata.get("session_id"): return match policy: @@ -760,7 +767,8 @@ def apply_missing_session_id_policy( ) case _: verbose_proxy_logger.warning( - "Ignoring unknown general_settings.missing_session_id=%r; expected 'generate' or 'reject'", policy + "Ignoring unknown general_settings.missing_session_id=%r; expected 'generate', 'reject' or 'omit'", + policy, ) diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index 5326074ad3c..93423ca5a1a 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -426,6 +426,11 @@ async def add_new_user_to_default_team( await asyncio.gather(*tasks, return_exceptions=True) +async def _fetch_user_team_ids(user_id: str, prisma_client: "PrismaClient") -> tuple[str, ...]: + user_row: Final = await _user_table(prisma_client).find_unique(where={"user_id": user_id}) + return tuple(user_row.teams) if user_row is not None else () + + @router.post( "/user/new", tags=["Internal User management"], @@ -580,6 +585,11 @@ async def new_user( ) user_id: Final = cast(str | None, response.get("user_id", None)) + attached_team_ids: Final = ( + await _fetch_user_team_ids(user_id=user_id, prisma_client=prisma_client) + if user_id is not None and (_team_id is not None or teams is not None) + else None + ) if organization_ids is not None and user_id is not None: await _add_user_to_organizations( @@ -596,6 +606,8 @@ async def new_user( response_dict[key] = value response_dict["key"] = response.get("token", "") + if attached_team_ids is not None: + response_dict["teams"] = list(attached_team_ids) new_user_response: Final = NewUserResponse.model_validate(response_dict) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 714cf252e69..72cdc75c29e 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -4318,6 +4318,8 @@ async def _resolve_team_access_group_resources( access_group_id=group.access_group_id, access_group_name=group.access_group_name, models=tuple(group.access_model_names or ()), + mcp_server_ids=tuple(group.access_mcp_server_ids or ()), + agent_ids=tuple(group.access_agent_ids or ()), ) for group in resolved_groups ), diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index b48b8d81494..29f216fd450 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -1730,6 +1730,16 @@ def get_vertex_ai_allowed_incoming_headers(request: Request) -> dict: return headers +def _is_vertex_anthropic_count_tokens_route(endpoint: str) -> bool: + return endpoint.rsplit("/", 1)[-1].split(":", 1)[0] == "count-tokens" + + +def _upstream_headers_for_vertex_route(endpoint: str, headers: Mapping[str, str]) -> Mapping[str, str]: + if not _is_vertex_anthropic_count_tokens_route(endpoint): + return headers + return MappingProxyType({name: value for name, value in headers.items() if name.lower() != "anthropic-beta"}) + + def get_vertex_pass_through_handler( call_type: Literal["discovery", "aiplatform"], # noqa: UP037 # ruff reports quoted Literal values here ) -> BaseVertexAIPassThroughHandler: @@ -2128,7 +2138,7 @@ async def _base_vertex_proxy_route( endpoint_func: Final = create_pass_through_route( endpoint=endpoint, target=target, - custom_headers=headers, + custom_headers=_upstream_headers_for_vertex_route(endpoint, headers), is_streaming_request=is_streaming_request, ) # dynamically construct pass-through endpoint based on incoming path diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 79d5d0a016f..323756bf204 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -40,6 +40,7 @@ from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid from litellm.constants import ( MAXIMUM_TRACEBACK_LINES_TO_LOG, + SESSION_ID_OMITTED_METADATA_KEY, WEBSOCKET_CLOSE_REASON_MAX_BYTES, ) from litellm.integrations.custom_guardrail import CustomGuardrail @@ -581,8 +582,9 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): ) # Set internal keys after merging client-supplied metadata so a request - # body that mirrors them cannot clobber the authenticated key or the - # real parent span. + # body that mirrors them cannot clobber the authenticated key, the real + # parent span, or the proxy's own session-id decision. + _metadata.pop(SESSION_ID_OMITTED_METADATA_KEY, None) _metadata["user_api_key"] = user_api_key_dict.api_key _metadata["litellm_parent_otel_span"] = user_api_key_dict.parent_otel_span _metadata["user_api_key_budget_reservation"] = user_api_key_dict.budget_reservation diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 295faaa980a..2a50d5170f0 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -173,6 +173,9 @@ class _SessionSpendRow(TypedDict): session_cache_hit_count: ReadOnly[int] session_llm_count: ReadOnly[int] session_agent_count: ReadOnly[int] + session_total_prompt_tokens: ReadOnly[int] + session_total_completion_tokens: ReadOnly[int] + session_total_tokens: ReadOnly[int] session_models: ReadOnly[Sequence[str]] @@ -188,6 +191,9 @@ class _SessionSpendStats(NamedTuple): session_cache_hit_count: int session_llm_count: int session_agent_count: int + session_total_prompt_tokens: int + session_total_completion_tokens: int + session_total_tokens: int session_models: Sequence[str] session_models_truncated: bool @@ -4287,8 +4293,8 @@ async def _build_ui_spend_logs_response( Build the paginated response for the UI spend-logs endpoint. When ``enrich_session_counts`` is ``True`` (the default for the v1/UI - endpoint), each row is enriched with ``session_total_count`` plus spend - and call-type aggregates so the frontend knows which sessions are + endpoint), each row is enriched with ``session_total_count`` plus spend, + token and call-type aggregates so the frontend knows which sessions are expandable (multi-call sessions). One ``GROUP BY (session_id, api_key)`` query serves every referenced session, keyed per api key so two callers reusing a session id never see each other's totals. Rows without a @@ -4356,7 +4362,10 @@ async def _build_ui_spend_logs_response( COUNT(*) FILTER ( WHERE call_type NOT IN {_MCP_CALL_TYPES_SQL} AND call_type != {_AGENT_CALL_TYPE_SQL} )::int AS session_llm_count, - COUNT(*) FILTER (WHERE call_type = {_AGENT_CALL_TYPE_SQL})::int AS session_agent_count + COUNT(*) FILTER (WHERE call_type = {_AGENT_CALL_TYPE_SQL})::int AS session_agent_count, + COALESCE(SUM(prompt_tokens), 0)::bigint AS session_total_prompt_tokens, + COALESCE(SUM(completion_tokens), 0)::bigint AS session_total_completion_tokens, + COALESCE(SUM(total_tokens), 0)::bigint AS session_total_tokens FROM "LiteLLM_SpendLogs" WHERE session_id = ANY($1::text[]) AND api_key = ANY($2::text[]) @@ -4389,6 +4398,9 @@ async def _build_ui_spend_logs_response( session_cache_hit_count=int(row.get("session_cache_hit_count") or 0), session_llm_count=int(row.get("session_llm_count") or 0), session_agent_count=int(row.get("session_agent_count") or 0), + session_total_prompt_tokens=int(row.get("session_total_prompt_tokens") or 0), + session_total_completion_tokens=int(row.get("session_total_completion_tokens") or 0), + session_total_tokens=int(row.get("session_total_tokens") or 0), session_models=models[:_SESSION_MODELS_LIMIT], session_models_truncated=len(models) > _SESSION_MODELS_LIMIT, ) @@ -4418,6 +4430,9 @@ async def _build_ui_spend_logs_response( row_dict["session_cache_hit_count"] = session_stats.session_cache_hit_count row_dict["session_llm_count"] = session_stats.session_llm_count row_dict["session_agent_count"] = session_stats.session_agent_count + row_dict["session_total_prompt_tokens"] = session_stats.session_total_prompt_tokens + row_dict["session_total_completion_tokens"] = session_stats.session_total_completion_tokens + row_dict["session_total_tokens"] = session_stats.session_total_tokens row_dict["session_models"] = session_stats.session_models row_dict["session_models_truncated"] = session_stats.session_models_truncated enriched.append(row_dict) diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 7442d71bd96..a37c3ba4405 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -15,6 +15,7 @@ from litellm.constants import ( LITELLM_TRUNCATED_PAYLOAD_FIELD, LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, REDACTED_BY_LITELM_STRING, + SESSION_ID_OMITTED_METADATA_KEY, ) from litellm.constants import ( MAX_STRING_LENGTH_PROMPT_IN_DB as DEFAULT_MAX_STRING_LENGTH_PROMPT_IN_DB, @@ -578,7 +579,9 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs ), session_id=_get_session_id_for_spend_log( kwargs=kwargs, + metadata=metadata, standard_logging_payload=standard_logging_payload, + omit_when_missing=_omits_session_id_when_missing(metadata), ), request_duration_ms=_get_request_duration_ms(start_time, end_time), status=_get_status_for_spend_log( @@ -602,26 +605,39 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs raise e +def _omits_session_id_when_missing(metadata: Mapping[str, object] | None) -> bool: + """The pre-call stamp pins `omit` on for the requests that carry it, so a config reload between pre-call and spend + logging cannot fabricate a session. `apply_missing_session_id_policy` drops any client-supplied copy of the key + from both metadata buckets before stamping, which the merge of `litellm_metadata` into `metadata` makes + necessary, so a caller cannot forge it. Requests that never reach the pre-call helper, router-model + passthrough among them, carry no stamp, so they fall back to the configured policy and `omit` still covers their + spend logs.""" + if metadata is not None and metadata.get(SESSION_ID_OMITTED_METADATA_KEY): + return True + + from litellm.proxy.proxy_server import general_settings + + return general_settings.get("missing_session_id") == "omit" + + def _get_session_id_for_spend_log( - kwargs: dict, + kwargs: Mapping[str, object], + metadata: Mapping[str, object] | None, standard_logging_payload: StandardLoggingPayload | None, -) -> str: - """ - Get the session id for the spend log. + omit_when_missing: bool, +) -> str | None: + """Under `omit` only `metadata.session_id`, the key Langfuse reads, counts as a session; `litellm_session_id` may + be a copied trace id.""" + if omit_when_missing: + session_id: Final = metadata.get("session_id") if metadata else None + return str(session_id) if session_id else None - This ensures each spend log is associated with a unique session id. - - """ from litellm._uuid import uuid if standard_logging_payload is not None and standard_logging_payload.get("trace_id") is not None: return str(standard_logging_payload.get("trace_id")) - - # Users can dynamically set the trace_id for each request by passing `litellm_trace_id` in kwargs if kwargs.get("litellm_trace_id") is not None: return str(kwargs.get("litellm_trace_id")) - - # Ensure we always have a session id, if none is provided return str(uuid.uuid4()) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index cab2bd6d9db..24a65457928 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -7531,6 +7531,9 @@ def create_model_info_response( max_input_tokens = configured_input if configured_output is not None: max_output_tokens = configured_output + configured_mode: Final = llm_router.get_configured_mode(model_id) + if isinstance(configured_mode, str): + base["mode"] = configured_mode if max_input_tokens is not None: base["max_input_tokens"] = max_input_tokens diff --git a/litellm/proxy/vector_store_endpoints/management_endpoints.py b/litellm/proxy/vector_store_endpoints/management_endpoints.py index c928398a87f..fe4732c6492 100644 --- a/litellm/proxy/vector_store_endpoints/management_endpoints.py +++ b/litellm/proxy/vector_store_endpoints/management_endpoints.py @@ -30,7 +30,10 @@ from litellm.proxy._types import ( ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.rbac_utils import check_feature_access_for_user -from litellm.proxy.vector_store_endpoints.utils import can_user_access_vector_store +from litellm.proxy.vector_store_endpoints.utils import ( + can_user_access_vector_store, + filter_listable_vector_stores, +) from litellm.repositories.prisma_protocols import TableActions from litellm.repositories.table_repositories import ManagedVectorStoresRepository from litellm.types.vector_stores import ( @@ -390,11 +393,10 @@ async def list_vector_stores( # Filter vector stores based on access control accessible_vector_stores: Final = [] - for vs in vector_store_map.values(): - if await _check_vector_store_access(vs, user_api_key_dict): - redacted = LiteLLM_ManagedVectorStore(**vs) - redacted["litellm_params"] = _redact_sensitive_litellm_params(vs.get("litellm_params")) - accessible_vector_stores.append(redacted) + for vs in await filter_listable_vector_stores(vector_store_map.values(), user_api_key_dict): + redacted = LiteLLM_ManagedVectorStore(**vs) + redacted["litellm_params"] = _redact_sensitive_litellm_params(vs.get("litellm_params")) + accessible_vector_stores.append(redacted) total_count: Final = len(accessible_vector_stores) total_pages: Final = (total_count + page_size - 1) // page_size diff --git a/litellm/proxy/vector_store_endpoints/utils.py b/litellm/proxy/vector_store_endpoints/utils.py index 93f1510bf22..6e94a5a88ac 100644 --- a/litellm/proxy/vector_store_endpoints/utils.py +++ b/litellm/proxy/vector_store_endpoints/utils.py @@ -1,11 +1,17 @@ import json import re +from collections.abc import Iterable +from types import MappingProxyType from typing import Any, Final, Literal from fastapi import HTTPException, Request import litellm from litellm._logging import verbose_proxy_logger +from litellm.proxy._experimental.mcp_server.ui_session_utils import ( + is_ui_session_credential, + resolve_ui_session_team_ids, +) from litellm.proxy._types import ( LiteLLM_ObjectPermissionTable, LitellmUserRoles, @@ -160,10 +166,16 @@ async def can_user_access_vector_store( if _is_proxy_admin(user_api_key_dict): return True - vector_store_team_id: Final = vector_store.get("team_id") - if vector_store_team_id is None: + if vector_store.get("team_id") is None: return True + return await _is_vector_store_granted(vector_store, user_api_key_dict) + + +async def _is_vector_store_granted( + vector_store: LiteLLM_ManagedVectorStore, + user_api_key_dict: UserAPIKeyAuth, +) -> bool: vector_store_id: Final = vector_store.get("vector_store_id") or "" key_object_permission = user_api_key_dict.object_permission @@ -178,12 +190,70 @@ async def can_user_access_vector_store( if _object_permission_allows_vector_store(team_object_permission, vector_store_id): return True - if user_api_key_dict.team_id is not None and user_api_key_dict.team_id == vector_store_team_id: - return True + return user_api_key_dict.team_id is not None and user_api_key_dict.team_id == vector_store.get("team_id") + +async def _team_auth_context(team_id: str, user_api_key_dict: UserAPIKeyAuth) -> UserAPIKeyAuth: + from litellm.proxy.auth.auth_checks import get_team_object + from litellm.proxy.proxy_server import ( + prisma_client, + proxy_logging_obj, + user_api_key_cache, + ) + + team: Final = await get_team_object( + team_id=team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=user_api_key_dict.parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) + return user_api_key_dict.model_copy( + update=MappingProxyType( + { + "team_id": team_id, + "team_object_permission": team.object_permission, + "team_object_permission_id": team.object_permission_id, + } + ) + ) + + +async def _vector_store_listing_auth_contexts( + user_api_key_dict: UserAPIKeyAuth, +) -> tuple[UserAPIKeyAuth, ...]: + if not is_ui_session_credential(user_api_key_dict): + return (user_api_key_dict,) + session_key_context: Final = user_api_key_dict.model_copy( + update=MappingProxyType({"team_id": None, "team_object_permission": None, "team_object_permission_id": None}) + ) + team_ids: Final = await resolve_ui_session_team_ids(user_api_key_dict) + team_contexts: Final = tuple([await _team_auth_context(team_id, user_api_key_dict) for team_id in team_ids]) + return (session_key_context, *team_contexts) + + +async def _is_vector_store_granted_to_any( + vector_store: LiteLLM_ManagedVectorStore, + auth_contexts: tuple[UserAPIKeyAuth, ...], +) -> bool: + for auth_context in auth_contexts: + if await _is_vector_store_granted(vector_store, auth_context): + return True return False +async def filter_listable_vector_stores( + vector_stores: Iterable[LiteLLM_ManagedVectorStore], + user_api_key_dict: UserAPIKeyAuth, +) -> tuple[LiteLLM_ManagedVectorStore, ...]: + """Non-admins only see stores their key, one of their teams' object_permission, or team ownership grants.""" + if _is_proxy_admin(user_api_key_dict): + return tuple(vector_stores) + + auth_contexts: Final = await _vector_store_listing_auth_contexts(user_api_key_dict) + return tuple([vs for vs in vector_stores if await _is_vector_store_granted_to_any(vs, auth_contexts)]) + + async def get_litellm_managed_vector_store( vector_store_id: str, ) -> LiteLLM_ManagedVectorStore | None: diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index d4b9f4e8cce..3862aec445f 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -8,6 +8,7 @@ from typing import Any, Final, Literal, cast import litellm from litellm.constants import ( + AZURE_OPENAI_AUDIO_PROVIDERS, REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, request_timeout, @@ -400,7 +401,7 @@ async def _arealtime( litellm_metadata=_build_litellm_metadata(kwargs), query_params=query_params, ) - elif _custom_llm_provider == "azure": + elif _custom_llm_provider in AZURE_OPENAI_AUDIO_PROVIDERS: api_base = dynamic_api_base or litellm_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") # set API KEY api_key = dynamic_api_key or litellm.api_key or litellm.openai_key or get_secret_str("AZURE_API_KEY") diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index bc25f4fffb1..d7f8cd8f8bd 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -562,6 +562,12 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): hidden_params: Final = getattr(chunk, "_hidden_params", None) if hidden_params is not None: chunk_dict["_hidden_params"] = dict(hidden_params) if isinstance(hidden_params, dict) else hidden_params + if ( + chunk_dict.get("usage") is None + and isinstance(hidden_params, dict) + and hidden_params.get("usage") is not None + ): + chunk_dict["usage"] = hidden_params["usage"] return chunk_dict def create_reasoning_summary_text_done_event( diff --git a/litellm/router.py b/litellm/router.py index 4302dc4a283..b0f82086daf 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -9981,6 +9981,17 @@ class Router: coerce_token_limit(model_info.get("max_output_tokens")), ) + def get_configured_mode(self, model_name: str) -> "str | None": + """Return the mode explicitly configured for a concrete deployment.""" + deployment: Final = self.get_deployment_by_model_group_name(model_group_name=model_name) + if deployment is None: + return None + + mode: Final = deployment.model_info.get("mode") + if isinstance(mode, str) and mode.strip(): + return mode + return None + def get_configured_display_name(self, model_name: str) -> "str | None": """ Return the display_name explicitly configured in a concrete deployment's diff --git a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py index 39d3e25aacb..6f3ea8eb78a 100644 --- a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py @@ -82,6 +82,7 @@ class DeploymentAffinityCheck(CustomLogger): """ CACHE_KEY_PREFIX = "deployment_affinity:v1" + USER_ID_AFFINITY_PREFIX: Final = "user_id:" def __init__( self, @@ -253,15 +254,6 @@ class DeploymentAffinityCheck(CustomLogger): hashed_user_key: Final = cls._hash_user_key(user_key) if user_key is not None else "unscoped" return f"{cls.CACHE_KEY_PREFIX}:session:{model_group}:{hashed_user_key}:{session_id}" - @staticmethod - def _get_user_key_from_metadata_dict(metadata: dict) -> str | None: - # NOTE: affinity is keyed on the *API key hash* provided by the proxy (not the - # OpenAI `user` parameter, which is an end-user identifier). - user_key: Final = metadata.get("user_api_key_hash") - if user_key is None: - return None - return str(user_key) - @staticmethod def _get_session_id_from_metadata_dict(metadata: dict) -> str | None: session_id: Final = metadata.get("session_id") @@ -285,22 +277,30 @@ class DeploymentAffinityCheck(CustomLogger): return metadata_dicts @staticmethod - def _get_user_key_from_request_kwargs(request_kwargs: dict) -> str | None: + def _first_metadata_value(metadata_dicts: Sequence[dict], key: str) -> str | None: + value: Final = next((metadata[key] for metadata in metadata_dicts if metadata.get(key) is not None), None) + return None if value is None else str(value) + + @classmethod + def _get_user_key_from_request_kwargs(cls, request_kwargs: dict) -> str | None: """ Extract a stable affinity key from request kwargs. - Source (proxy): `metadata.user_api_key_hash` + Source (proxy): `metadata.user_api_key_hash` for virtual-key callers. JWT-authenticated + callers carry no key hash, so their `metadata.user_api_key_user_id` stands in for it, + namespaced under `USER_ID_AFFINITY_PREFIX` so a user id can never alias a key hash. Note: the OpenAI `user` parameter is an end-user identifier and is intentionally not used for deployment affinity. """ - # Check metadata dicts (Proxy usage) - for metadata in DeploymentAffinityCheck._iter_metadata_dicts(request_kwargs): - user_key = DeploymentAffinityCheck._get_user_key_from_metadata_dict(metadata=metadata) - if user_key is not None: - return user_key - - return None + metadata_dicts: Final = cls._iter_metadata_dicts(request_kwargs) + user_api_key_hash: Final = cls._first_metadata_value(metadata_dicts, "user_api_key_hash") + if user_api_key_hash is not None: + return user_api_key_hash + user_id: Final = cls._first_metadata_value(metadata_dicts, "user_api_key_user_id") + if user_id is None: + return None + return f"{cls.USER_ID_AFFINITY_PREFIX}{user_id}" @staticmethod def _get_session_id_from_request_kwargs(request_kwargs: dict) -> str | None: @@ -533,9 +533,9 @@ class DeploymentAffinityCheck(CustomLogger): return typed_healthy_deployments verbose_router_logger.debug( - "DeploymentAffinityCheck: api-key affinity hit -> deployment=%s user_key=%s", + "DeploymentAffinityCheck: caller affinity hit -> deployment=%s user_key=%s", model_id, - self._shorten_for_logs(user_key), + self._shorten_for_logs(self._hash_user_key(user_key)), ) return [deployment] @@ -626,7 +626,7 @@ class DeploymentAffinityCheck(CustomLogger): deployment_model_name, model_id, self.ttl_seconds, - self._shorten_for_logs(user_key), + self._shorten_for_logs(self._hash_user_key(user_key)), ) else: verbose_router_logger.debug( diff --git a/litellm/types/integrations/custom_logger.py b/litellm/types/integrations/custom_logger.py index 9a714e1724e..5de58a20242 100644 --- a/litellm/types/integrations/custom_logger.py +++ b/litellm/types/integrations/custom_logger.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from typing import Any, Final from pydantic import BaseModel, Field @@ -29,6 +30,13 @@ def is_interception_internal_key( return any(key.startswith(prefix) for prefix in prefixes) +CONVERTED_STREAM_KEYS: Final = frozenset(f"{prefix}_converted_stream" for prefix in INTERCEPTION_INTERNAL_PREFIXES) + + +def converted_stream_requested(params: Mapping[str, object]) -> bool: + return any(bool(params.get(key)) for key in CONVERTED_STREAM_KEYS) + + class AgenticLoopSafetyError(ValueError): """ Raised when an agentic-loop safety rail refuses a rerun. diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 32d88da0085..b33ff954c35 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -66,6 +66,7 @@ from pydantic import ( ConfigDict, Discriminator, Field, + NonNegativeInt, PrivateAttr, SerializerFunctionWrapHandler, field_serializer, @@ -1321,6 +1322,18 @@ class ResponseAPIUsage(BaseLiteLLMOpenAIResponseObject): model_config = {"extra": "allow"} +class WebSearchToolUsage(BaseModel): + model_config = ConfigDict(frozen=True) + + num_requests: NonNegativeInt + + +class ResponsesToolUsage(BaseModel): + model_config = ConfigDict(frozen=True) + + web_search: WebSearchToolUsage | None = None + + ResponsesAPIStatus = Literal["completed", "failed", "in_progress", "cancelled", "queued", "incomplete"] """ The status of the response generation. diff --git a/litellm/types/proxy/model_listing.py b/litellm/types/proxy/model_listing.py index b59c0f2cf19..24cfa85eee4 100644 --- a/litellm/types/proxy/model_listing.py +++ b/litellm/types/proxy/model_listing.py @@ -11,8 +11,8 @@ class ModelInfoMetadata(TypedDict): class ModelInfoResponse(TypedDict): """OpenAI-compatible model object. `mode`, `max_input_tokens`, and - `max_output_tokens` are attached when the cost map knows them; `metadata` - is present only when the endpoint is called with include_metadata=true. + `max_output_tokens` are attached when the cost map or deployment config + knows them; `metadata` is present only with include_metadata=true. """ id: str diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 569fce4f7b8..0bdbb83fbb4 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2543,6 +2543,7 @@ class ImageResponse(OpenAIImageResponse, BaseLiteLLMOpenAIResponseObject): ) super().__init__(created=created, data=_data, usage=_usage) + self.background = kwargs.get("background", None) self.quality = kwargs.get("quality", None) self.output_format = kwargs.get("output_format", None) self.size = kwargs.get("size", None) diff --git a/litellm/utils.py b/litellm/utils.py index 7db6a56d76b..7d081ce7652 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3338,6 +3338,9 @@ def get_optional_params_image_gen( continue passed_params[k] = v + provider_supported_params: Final[tuple[str, ...]] = ( + tuple(provider_config.get_supported_openai_params(model=model or "")) if provider_config is not None else () + ) default_params: Final = { "n": None, "quality": None, @@ -3348,6 +3351,7 @@ def get_optional_params_image_gen( "imageConfig": None, "tools": None, "web_search_options": None, + **{k: None for k in provider_supported_params}, } non_default_params: Final = _get_non_default_params( @@ -3407,10 +3411,9 @@ def get_optional_params_image_gen( if size is not None: optional_params["aspectRatio"] = _map_openai_size_to_vertex_ai_aspect_ratio(size) - openai_params: list[str] = list(default_params.keys()) - if provider_config is not None: - supported_params = provider_config.get_supported_openai_params(model=model or "") - openai_params = list(supported_params) + openai_params: Final[list[str]] = ( + list(provider_supported_params) if provider_config is not None else list(default_params.keys()) + ) optional_params = add_provider_specific_params_to_optional_params( optional_params=optional_params, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index cd187760fd2..9ed0b90c516 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -29277,6 +29277,75 @@ "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, + "gpt-6-astra": { + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05, + "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05, + "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-05, + "cache_creation_input_token_cost_flex": 6.25e-06, + "cache_creation_input_token_cost_priority": 2.5e-05, + "cache_read_input_token_cost": 1e-06, + "cache_read_input_token_cost_above_272k_tokens": 2e-06, + "cache_read_input_token_cost_above_272k_tokens_flex": 1e-06, + "cache_read_input_token_cost_above_272k_tokens_priority": 4e-06, + "cache_read_input_token_cost_flex": 5e-07, + "cache_read_input_token_cost_priority": 2e-06, + "input_cost_per_token": 1e-05, + "input_cost_per_token_above_272k_tokens": 2e-05, + "input_cost_per_token_above_272k_tokens_flex": 1e-05, + "input_cost_per_token_above_272k_tokens_priority": 4e-05, + "input_cost_per_token_batches": 5e-06, + "input_cost_per_token_flex": 5e-06, + "input_cost_per_token_priority": 2e-05, + "litellm_provider": "openai", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-05, + "output_cost_per_token_above_272k_tokens": 7.5e-05, + "output_cost_per_token_above_272k_tokens_flex": 3.75e-05, + "output_cost_per_token_above_272k_tokens_priority": 0.00015, + "output_cost_per_token_batches": 2.5e-05, + "output_cost_per_token_flex": 2.5e-05, + "output_cost_per_token_priority": 0.0001, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_native_streaming": true, + "supports_none_reasoning_effort": false, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "supports_xhigh_reasoning_effort": true + }, "gpt-5.6": { "cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, @@ -52911,6 +52980,11 @@ "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "output_cost_per_token": 3.3e-05, "output_cost_per_token_above_272k_tokens": 4.95e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.012, + "search_context_size_low": 0.012, + "search_context_size_medium": 0.012 + }, "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -52945,6 +53019,11 @@ "cache_read_input_token_cost_above_272k_tokens": 4.4e-07, "output_cost_per_token": 1.32e-05, "output_cost_per_token_above_272k_tokens": 1.98e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.012, + "search_context_size_low": 0.012, + "search_context_size_medium": 0.012 + }, "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -53007,6 +53086,11 @@ "cache_read_input_token_cost_above_272k_tokens": 4.4e-08, "output_cost_per_token": 1.32e-06, "output_cost_per_token_above_272k_tokens": 1.98e-06, + "search_context_cost_per_query": { + "search_context_size_high": 0.012, + "search_context_size_low": 0.012, + "search_context_size_medium": 0.012 + }, "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -53195,6 +53279,11 @@ "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "output_cost_per_token": 3.3e-05, "output_cost_per_token_above_272k_tokens": 4.95e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.012, + "search_context_size_low": 0.012, + "search_context_size_medium": 0.012 + }, "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -53226,6 +53315,11 @@ "cache_read_input_token_cost_above_272k_tokens": 5.5e-07, "output_cost_per_token": 1.65e-05, "output_cost_per_token_above_272k_tokens": 2.475e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.012, + "search_context_size_low": 0.012, + "search_context_size_medium": 0.012 + }, "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, diff --git a/pyproject.toml b/pyproject.toml index 161994635b8..d3038a60c42 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,8 +67,8 @@ proxy = [ "azure-identity>=1.25.2,<2.0", "azure-storage-blob>=12.28.0,<13.0", "mcp>=1.28.1,<2.0", - "litellm-proxy-extras==0.4.92", - "litellm-enterprise==0.1.63", + "litellm-proxy-extras==0.4.93", + "litellm-enterprise==0.1.64", "RestrictedPython>=8.5,<9.0", "rich>=13.9.4,<14.0", "InquirerPy>=0.3.4,<1.0", diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py index ef8d6c55148..f8f23ea015a 100644 --- a/tests/local_testing/test_completion.py +++ b/tests/local_testing/test_completion.py @@ -2879,7 +2879,6 @@ def response_format_tests(response: litellm.ModelResponse): "model", [ "bedrock/mistral.mistral-large-2407-v1:0", - "bedrock/cohere.command-r-plus-v1:0", "us.anthropic.claude-sonnet-4-5-20250929-v1:0", "mistral.mistral-7b-instruct-v0:2", "meta.llama3-8b-instruct-v1:0", diff --git a/tests/local_testing/test_streaming.py b/tests/local_testing/test_streaming.py index 07d693af447..bf39d3155b7 100644 --- a/tests/local_testing/test_streaming.py +++ b/tests/local_testing/test_streaming.py @@ -1168,7 +1168,6 @@ async def test_completion_replicate_llama3_streaming(sync_mode): "model, region", [ # ["bedrock/ai21.jamba-instruct-v1:0", "us-east-1"], - # ["bedrock/cohere.command-r-plus-v1:0", None], ["us.anthropic.claude-sonnet-4-5-20250929-v1:0", None], # ["mistral.mistral-7b-instruct-v0:2", None], # ["meta.llama3-8b-instruct-v1:0", None], @@ -1271,7 +1270,7 @@ def test_bedrock_claude_3_streaming(): "model", [ "claude-haiku-4-5-20251001", - "cohere.command-r-plus-v1:0", # bedrock + "bedrock/mistral.mistral-7b-instruct-v0:2", "gpt-3.5-turbo", ], ) diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 21b60d7a216..6590718878d 100644 --- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -1295,6 +1295,19 @@ def test_text_plus_tool_calls_sequence(): # ============================================================================= +def test_developer_message_content_uses_input_text(): + handler = LiteLLMResponsesTransformationHandler() + + input_items, instructions = handler.convert_chat_completion_messages_to_responses_api( + [{"role": "developer", "content": "Always answer in French."}] + ) + + assert instructions is None + assert input_items == [ + {"type": "message", "role": "developer", "content": [{"type": "input_text", "text": "Always answer in French."}]} + ] + + def test_tool_message_output_uses_input_text_not_output_text(): """ Test that tool message content uses input_text type, not output_text. diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index b7f0ca1efe1..0b6832d4bef 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -1662,6 +1662,54 @@ def test_generic_cost_per_token_gpt56_cyber( assert completion_cost == pytest.approx(completion_tokens * output_rate) +@pytest.mark.parametrize( + "service_tier,tier_multiplier", + [(None, 1.0), ("flex", 0.5), ("priority", 2.0), ("fast", 2.0)], +) +@pytest.mark.parametrize( + "prompt_tokens,input_side_multiplier,output_multiplier", + [(100000, 1.0, 1.0), (300000, 2.0, 1.5)], +) +def test_generic_cost_per_token_gpt_6_astra_price_sheet( + _local_model_cost_map, + service_tier, + tier_multiplier, + prompt_tokens, + input_side_multiplier, + output_multiplier, +): + """gpt-6-astra launch price sheet: $10 input, $1 cache read, $12.50 cache write, $50 output per 1M tokens. + + Above 272K prompt tokens the input-side rates double and the output rate is 1.5x on the whole + request. Flex is half the applicable rate and fast mode, billed as priority, is double it. + """ + cached_tokens = 50000 + cache_write_tokens = 40000 + text_tokens = prompt_tokens - cached_tokens - cache_write_tokens + completion_tokens = 1000 + usage = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=cached_tokens, cache_write_tokens=cache_write_tokens + ), + ) + + prompt_cost, completion_cost = generic_cost_per_token( + model="gpt-6-astra", + usage=usage, + custom_llm_provider="openai", + service_tier=service_tier, + ) + + input_side = tier_multiplier * input_side_multiplier + assert prompt_cost == pytest.approx( + input_side * (text_tokens * 1e-5 + cached_tokens * 1e-6 + cache_write_tokens * 1.25e-5) + ) + assert completion_cost == pytest.approx(tier_multiplier * output_multiplier * completion_tokens * 5e-5) + + @pytest.mark.parametrize( "model,input_cost,output_cost,cache_read_cost", [ diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py index fd795ffcc96..6cc3dcceebc 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py @@ -1,4 +1,5 @@ import os +from collections.abc import Mapping, Sequence import pytest @@ -6,7 +7,7 @@ import litellm from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, ) -from litellm.types.llms.openai import FileSearchTool, WebSearchOptions +from litellm.types.llms.openai import FileSearchTool, ResponsesAPIResponse, WebSearchOptions from litellm.types.utils import ModelResponse, StandardBuiltInToolsParams @@ -928,3 +929,125 @@ def test_web_search_gate_reads_server_side_tool_usage_details_without_citations( standard_built_in_tools_params=None, ) assert cost == 3 * _DEFAULT_WEB_SEARCH_COST_PER_CALL + + +_BEDROCK_MANTLE_WEB_SEARCH_MODELS = ( + "bedrock_mantle/openai.gpt-5.6-sol", + "bedrock_mantle/openai.gpt-5.6-terra", + "bedrock_mantle/openai.gpt-5.6-luna", + "bedrock_mantle/openai.gpt-5.5", + "bedrock_mantle/openai.gpt-5.4", +) + +_BEDROCK_MANTLE_WEB_SEARCH_RATE = 0.012 + + +def _responses_with_web_search( + model: str, actions: Sequence[Mapping[str, str]], tool_usage: Mapping[str, object] | None = None +) -> ResponsesAPIResponse: + payload = { + "id": "resp_1", + "created_at": 1756900000, + "model": model.split("/", 1)[-1], + "object": "response", + "status": "completed", + "output": [ + {"type": "web_search_call", "id": f"ws_{i}", "status": "completed", "action": action} + for i, action in enumerate(actions) + ], + } + return ResponsesAPIResponse.model_validate( + payload if tool_usage is None else {**payload, "tool_usage": tool_usage} + ) + + +def _web_search_cost(model: str, response: ResponsesAPIResponse, custom_llm_provider: str) -> float: + from litellm.types.utils import Usage + + return StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model=model, + response_object=response, + usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), + custom_llm_provider=custom_llm_provider, + standard_built_in_tools_params=None, + ) + + +@pytest.mark.parametrize("model", _BEDROCK_MANTLE_WEB_SEARCH_MODELS) +def test_bedrock_mantle_web_search_billed_per_query(local_model_cost_map, model): + """Two Bedrock-reported web searches bill 2 x $0.012 under the prefixed and the bare model id alike.""" + pricing = litellm.get_model_info(model)["search_context_cost_per_query"] + assert pricing == { + "search_context_size_low": _BEDROCK_MANTLE_WEB_SEARCH_RATE, + "search_context_size_medium": _BEDROCK_MANTLE_WEB_SEARCH_RATE, + "search_context_size_high": _BEDROCK_MANTLE_WEB_SEARCH_RATE, + } + + response = _responses_with_web_search( + model, + actions=[{"type": "search", "query": "litellm"}, {"type": "search", "query": "bedrock web search"}], + tool_usage={"web_search": {"num_requests": 2}}, + ) + for cost_model in (model, model.split("/", 1)[1]): + cost = _web_search_cost(cost_model, response, "bedrock_mantle") + assert cost == pytest.approx(2 * _BEDROCK_MANTLE_WEB_SEARCH_RATE), ( + f"{cost_model} must bill 2 x ${_BEDROCK_MANTLE_WEB_SEARCH_RATE} for 2 web searches, got ${cost}" + ) + + +@pytest.mark.parametrize("num_requests", [1, 0]) +def test_web_search_call_count_prefers_provider_reported_num_requests(local_model_cost_map, num_requests): + """A search plus an open_page fetch bills tool_usage.web_search.num_requests, never the two items.""" + model = "bedrock_mantle/openai.gpt-5.6-sol" + response = _responses_with_web_search( + model, + actions=[ + {"type": "search", "query": "litellm"}, + {"type": "open_page", "url": "https://docs.litellm.ai/"}, + ], + tool_usage={"web_search": {"num_requests": num_requests}}, + ) + + cost = _web_search_cost(model, response, "bedrock_mantle") + + assert cost == pytest.approx(num_requests * _BEDROCK_MANTLE_WEB_SEARCH_RATE), ( + f"{num_requests} reported web search requests must bill {num_requests} x " + f"${_BEDROCK_MANTLE_WEB_SEARCH_RATE}, got ${cost}" + ) + + +@pytest.mark.parametrize( + "tool_usage", + [None, {}, {"web_search": None}, {"web_search": {"num_requests": "many"}}, {"web_search": {"num_requests": -1}}], +) +def test_web_search_call_count_falls_back_to_items_without_reported_count(local_model_cost_map, tool_usage): + """Without a usable reported count the per-call path keeps counting web_search_call items.""" + model = "bedrock_mantle/openai.gpt-5.6-sol" + response = _responses_with_web_search( + model, + actions=[{"type": "search", "query": "litellm"}, {"type": "search", "query": "bedrock web search"}], + tool_usage=tool_usage, + ) + + cost = _web_search_cost(model, response, "bedrock_mantle") + + assert cost == pytest.approx(2 * _BEDROCK_MANTLE_WEB_SEARCH_RATE), ( + f"2 web_search_call items with tool_usage={tool_usage!r} must bill 2 x " + f"${_BEDROCK_MANTLE_WEB_SEARCH_RATE}, got ${cost}" + ) + + +def test_web_search_call_count_reads_reported_count_beside_other_tool_usage_entries(local_model_cost_map): + """OpenAI reports web_search.num_requests next to other tool entries, which must not disable the reported count.""" + response = _responses_with_web_search( + "gpt-5.6", + actions=[{"type": "search", "query": "S&P 500 close"}, {"type": "open_page", "url": "https://example.com/"}], + tool_usage={ + "image_gen": {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}, + "web_search": {"num_requests": 1}, + }, + ) + + cost = _web_search_cost("gpt-5.6", response, "openai") + + assert cost == pytest.approx(0.01), f"1 reported OpenAI web search must bill 1 x $0.01, not the 2 items, got ${cost}" diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index 4b2a3f9247c..0aa73833677 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -4761,6 +4761,43 @@ async def test_async_stream_assembled_response_keeps_vertex_traffic_type(logging assert assembled._hidden_params["provider_specific_fields"]["traffic_type"] == "ON_DEMAND_FLEX" +@pytest.mark.asyncio +async def test_async_fake_stream_final_chunk_carries_hidden_usage(logging_obj: Logging): + from litellm.llms.base_llm.base_model_iterator import MockResponseIterator + from litellm.types.utils import ModelResponse + + model_response = ModelResponse( + id="chatcmpl-fake-stream", + model="my-random-model", + choices=[ + { + "index": 0, + "message": {"role": "assistant", "content": "hello world"}, + "finish_reason": "stop", + } + ], + ) + model_response.usage = Usage(prompt_tokens=1234, completion_tokens=7, total_tokens=1241) + + wrapper = CustomStreamWrapper( + completion_stream=MockResponseIterator(model_response=model_response), + model="my-random-model", + custom_llm_provider="anthropic", + logging_obj=logging_obj, + ) + + final_chunk = None + async for chunk in wrapper: + final_chunk = chunk + + assert final_chunk is not None + hidden_usage = final_chunk._hidden_params.get("usage") + assert hidden_usage is not None + assert hidden_usage.prompt_tokens == 1234 + assert hidden_usage.completion_tokens == 7 + assert hidden_usage.total_tokens == 1241 + + class TestStableStreamingResponseId: """ All chunks of one streamed response must share the same top-level id diff --git a/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py b/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py index 11a727c9635..33fbb4e8fc7 100644 --- a/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py +++ b/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py @@ -31,6 +31,46 @@ async def test_get_openai_compatible_provider_info(): assert custom_llm_provider == "azure" +@pytest.mark.parametrize( + "model, api_base, expected_provider", + [ + ("azure_ai/gpt-4o", "https://my-resource.services.ai.azure.com", "azure_ai"), + ("azure_ai/gpt-4o", "https://my-resource.services.ai.azure.com/models", "azure_ai"), + ("azure_ai/gpt-5.4-nano", "https://my-resource.services.ai.azure.com", "azure_ai"), + ("azure_ai/gpt-4o", "https://my-resource.openai.azure.com", "azure"), + ( + "azure_ai/gpt-4o", + "https://my-resource.services.ai.azure.com/openai/deployments/gpt-4o/chat/completions" + "?api-version=2024-08-01-preview", + "azure", + ), + ("azure_ai/mistral-large-latest", "https://my-resource.services.ai.azure.com", "azure_ai"), + ("azure_ai/mistral-large-latest", "https://my-resource.openai.azure.com", "azure_ai"), + ], +) +def test_foundry_base_keeps_azure_ai_provider(model: str, api_base: str, expected_provider: str): + """Regression for #38276: a Foundry .services.ai.azure.com base must not be reclassified as azure.""" + config = AzureAIStudioConfig() + ( + _, + _, + custom_llm_provider, + ) = config._get_openai_compatible_provider_info( + model=model, + api_base=api_base, + api_key="my-key", + custom_llm_provider="azure_ai", + ) + assert custom_llm_provider == expected_provider + + +def test_is_azure_openai_model_without_api_base_keeps_azure_ai(): + """Metadata lookups (get_model_info, supports_* checks) carry no api_base and must not flip the provider.""" + config = AzureAIStudioConfig() + assert config._is_azure_openai_model(model="azure_ai/gpt-4o", api_base=None) is False + assert config._is_azure_openai_model(model="azure_ai/gpt-4o", api_base="https://my-res.openai.azure.com") is True + + def test_azure_ai_validate_environment(): config = AzureAIStudioConfig() headers = config.validate_environment( diff --git a/tests/test_litellm/llms/azure_ai/embed/test_azure_ai_embed_handler.py b/tests/test_litellm/llms/azure_ai/embed/test_azure_ai_embed_handler.py new file mode 100644 index 00000000000..0401629171d --- /dev/null +++ b/tests/test_litellm/llms/azure_ai/embed/test_azure_ai_embed_handler.py @@ -0,0 +1,69 @@ +import httpx +import pytest +import respx + +from litellm import embedding +from litellm.llms.azure_ai.embed.handler import _foundry_models_route_base + +EMBEDDING_PAYLOAD = { + "object": "list", + "data": [{"object": "embedding", "embedding": [0.1, 0.2], "index": 0}], + "model": "text-embedding-3-small", + "usage": {"prompt_tokens": 2, "total_tokens": 2}, +} + + +@pytest.mark.parametrize( + ("api_base", "expected"), + [ + ( + "https://my-foundry.services.ai.azure.com", + "https://my-foundry.services.ai.azure.com/models", + ), + ( + "https://my-foundry.services.ai.azure.com/", + "https://my-foundry.services.ai.azure.com/models", + ), + ( + "https://my-foundry.services.ai.azure.com?api-version=2024-05-01-preview", + "https://my-foundry.services.ai.azure.com/models?api-version=2024-05-01-preview", + ), + ( + "https://my-foundry.services.ai.azure.com/models", + "https://my-foundry.services.ai.azure.com/models", + ), + ( + "https://my-foundry.services.ai.azure.com/openai/deployments/text-embedding-3-small", + "https://my-foundry.services.ai.azure.com/openai/deployments/text-embedding-3-small", + ), + ( + "https://my-resource.openai.azure.com", + "https://my-resource.openai.azure.com", + ), + ( + "https://Mistral-serverless.eastus2.models.ai.azure.com", + "https://Mistral-serverless.eastus2.models.ai.azure.com", + ), + (None, None), + ], +) +def test_foundry_models_route_base(api_base, expected): + assert _foundry_models_route_base(api_base) == expected + + +@respx.mock +def test_azure_ai_embedding_calls_foundry_models_route(): + route = respx.post("https://my-foundry.services.ai.azure.com/models/embeddings").mock( + return_value=httpx.Response(200, json=EMBEDDING_PAYLOAD) + ) + + response = embedding( + model="azure_ai/text-embedding-3-small", + input=["hello world"], + api_base="https://my-foundry.services.ai.azure.com", + api_key="fake-key", + ) + + assert route.called + assert response.data is not None + assert len(response.data) == 1 diff --git a/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py b/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py index 8dc4620dd1b..b6281834f24 100644 --- a/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py +++ b/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py @@ -10,11 +10,11 @@ Tests the cost calculation for Dashscope models including: import math import os +from datetime import datetime, timezone import pytest # Add the project root to Python path - import litellm from litellm.llms.dashscope.cost_calculator import ( cost_per_token as dashscope_cost_per_token, @@ -526,3 +526,139 @@ class TestDashscopeCostCalculator: assert prompt_cost == 0.0 assert math.isclose(completion_cost, 500 * 1.6e-06, rel_tol=1e-10) + + OFF_PEAK_WINDOW = "14:00-00:00" + INSIDE_WINDOW = datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc) + OUTSIDE_WINDOW = datetime(2026, 9, 3, 9, 0, tzinfo=timezone.utc) + + def _register_off_peak_flat_model(self, model_key: str, off_peak_pricing: dict) -> None: + litellm.model_cost[model_key] = { + "litellm_provider": "dashscope", + "mode": "chat", + "input_cost_per_token": 2.4e-06, + "output_cost_per_token": 4.8e-06, + "cache_read_input_token_cost": 2e-07, + "cache_creation_input_token_cost": 3e-06, + "off_peak_pricing": off_peak_pricing, + } + + def test_dashscope_off_peak_window_swaps_in_the_off_peak_rates(self): + """ + Regression (LIT-6782): a deployment configured with off_peak_pricing kept billing the + standard dashscope rates inside its window, while the same block on a deepseek + deployment billed the off-peak rates. + """ + self._register_off_peak_flat_model( + "dashscope/deepseek-off-peak-test", + { + "hours_utc": self.OFF_PEAK_WINDOW, + "input_cost_per_token": 1.2e-06, + "output_cost_per_token": 2.4e-06, + "cache_read_input_token_cost": 1e-07, + }, + ) + usage = Usage( + prompt_tokens=1000, + completion_tokens=200, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=300, cache_creation_tokens=100), + ) + + prompt_cost, completion_cost = dashscope_cost_per_token( + model="deepseek-off-peak-test", usage=usage, current_time=self.INSIDE_WINDOW + ) + + assert math.isclose(prompt_cost, (600 * 1.2e-06) + (300 * 1e-07) + (100 * 3e-06), rel_tol=1e-10) + assert math.isclose(completion_cost, 200 * 2.4e-06, rel_tol=1e-10) + + peak_prompt_cost, peak_completion_cost = dashscope_cost_per_token( + model="deepseek-off-peak-test", usage=usage, current_time=self.OUTSIDE_WINDOW + ) + + assert math.isclose(peak_prompt_cost, (600 * 2.4e-06) + (300 * 2e-07) + (100 * 3e-06), rel_tol=1e-10) + assert math.isclose(peak_completion_cost, 200 * 4.8e-06, rel_tol=1e-10) + + def test_dashscope_off_peak_window_overrides_the_selected_tier(self): + """An open off-peak window bills the whole request at the flat off-peak rates, whichever tier + the input volume selected.""" + self._register_tiered_model( + "dashscope/qwen-tiered-off-peak-test", + [ + {"range": [0, 1000], "input_cost_per_token": 4e-07, "output_cost_per_token": 1.6e-06}, + {"range": [1000, 2000], "input_cost_per_token": 8e-07, "output_cost_per_token": 3.2e-06}, + ], + ) + litellm.model_cost["dashscope/qwen-tiered-off-peak-test"]["off_peak_pricing"] = { + "hours_utc": self.OFF_PEAK_WINDOW, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 4e-07, + } + usage = Usage(prompt_tokens=1500, completion_tokens=300) + + prompt_cost, completion_cost = dashscope_cost_per_token( + model="qwen-tiered-off-peak-test", usage=usage, current_time=self.INSIDE_WINDOW + ) + + assert math.isclose(prompt_cost, 1500 * 1e-07, rel_tol=1e-10) + assert math.isclose(completion_cost, 300 * 4e-07, rel_tol=1e-10) + + peak_prompt_cost, peak_completion_cost = dashscope_cost_per_token( + model="qwen-tiered-off-peak-test", usage=usage, current_time=self.OUTSIDE_WINDOW + ) + + assert math.isclose(peak_prompt_cost, 1500 * 8e-07, rel_tol=1e-10) + assert math.isclose(peak_completion_cost, 300 * 3.2e-06, rel_tol=1e-10) + + def test_dashscope_off_peak_rates_left_unset_keep_the_standard_rates(self): + """A block that only overrides the input rate leaves output and cache reads on the standard + rates, and an explicit reasoning rate is never swapped out.""" + self._register_off_peak_flat_model( + "dashscope/qwen-partial-off-peak-test", + {"hours_utc": self.OFF_PEAK_WINDOW, "input_cost_per_token": 1.2e-06}, + ) + litellm.model_cost["dashscope/qwen-partial-off-peak-test"]["output_cost_per_reasoning_token"] = 9e-06 + usage = Usage( + prompt_tokens=1000, + completion_tokens=200, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=300), + completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=50), + ) + + prompt_cost, completion_cost = dashscope_cost_per_token( + model="qwen-partial-off-peak-test", usage=usage, current_time=self.INSIDE_WINDOW + ) + + assert math.isclose(prompt_cost, (700 * 1.2e-06) + (300 * 2e-07), rel_tol=1e-10) + assert math.isclose(completion_cost, (150 * 4.8e-06) + (50 * 9e-06), rel_tol=1e-10) + + def test_dashscope_off_peak_output_rate_covers_reasoning_without_a_dedicated_rate(self): + """Reasoning tokens on a model with no dedicated reasoning rate follow the off-peak output + rate, the same way they follow the standard output rate outside the window.""" + self._register_off_peak_flat_model( + "dashscope/qwen-reasoning-off-peak-test", + {"hours_utc": self.OFF_PEAK_WINDOW, "output_cost_per_token": 2.4e-06}, + ) + usage = Usage( + prompt_tokens=100, + completion_tokens=200, + completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=50), + ) + + _, completion_cost = dashscope_cost_per_token( + model="qwen-reasoning-off-peak-test", usage=usage, current_time=self.INSIDE_WINDOW + ) + + assert math.isclose(completion_cost, 200 * 2.4e-06, rel_tol=1e-10) + + def test_dashscope_off_peak_defaults_to_the_current_time(self): + """The proxy's cost dispatch passes no clock, so an all-day window has to apply on the + default current time.""" + self._register_off_peak_flat_model( + "dashscope/qwen-all-day-off-peak-test", + {"hours_utc": "00:00-00:00", "input_cost_per_token": 1.2e-06, "output_cost_per_token": 2.4e-06}, + ) + usage = Usage(prompt_tokens=1000, completion_tokens=200) + + prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-all-day-off-peak-test", usage=usage) + + assert math.isclose(prompt_cost, 1000 * 1.2e-06, rel_tol=1e-10) + assert math.isclose(completion_cost, 200 * 2.4e-06, rel_tol=1e-10) diff --git a/tests/test_litellm/llms/openai/image_generation/test_gpt_transformation.py b/tests/test_litellm/llms/openai/image_generation/test_gpt_transformation.py new file mode 100644 index 00000000000..d9b87627b58 --- /dev/null +++ b/tests/test_litellm/llms/openai/image_generation/test_gpt_transformation.py @@ -0,0 +1,39 @@ +from unittest.mock import MagicMock + +import httpx +import pytest + +from litellm.llms.azure.image_generation.gpt_transformation import AzureGPTImageGenerationConfig +from litellm.llms.openai.image_generation.gpt_transformation import GPTImageGenerationConfig +from litellm.types.utils import ImageResponse + + +@pytest.mark.parametrize("config", [GPTImageGenerationConfig(), AzureGPTImageGenerationConfig()]) +def test_transform_image_generation_response_keeps_provider_echo(config): + raw_response = httpx.Response( + status_code=200, + json={ + "created": 1788457009, + "data": [{"b64_json": "/9j/4AAQSkZJRg=="}], + "output_format": "jpeg", + "background": "opaque", + "quality": "low", + "size": "1024x1024", + }, + request=httpx.Request("POST", "https://api.openai.com/v1/images/generations"), + ) + + image_response = config.transform_image_generation_response( + model="gpt-image-2", + raw_response=raw_response, + model_response=ImageResponse(), + logging_obj=MagicMock(), + request_data={"prompt": "a red apple", "output_format": "jpeg"}, + optional_params={"output_format": "jpeg"}, + litellm_params={}, + encoding=None, + ) + + assert image_response.output_format == "jpeg" + assert image_response.quality == "low" + assert image_response.background == "opaque" diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py index 295121167d6..a453708040e 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -1329,6 +1329,523 @@ class TestOpenAIResponsesHandlerToolInjection: assert "injected_tool" in names +COMPRESSED_MARKER = "[compressed document; retrieve the full text with hash=b573993006976af767214fac]" + + +class StructuredRewriteGuardrail(CustomGuardrail): + """Guardrail that rewrites whole messages via structured_messages and leaves + texts untouched, the way message-compressing guardrails do.""" + + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional[Any] = None, + ) -> GenericGuardrailAPIInputs: + messages = list(inputs.get("structured_messages") or []) + first_user = next(i for i, m in enumerate(messages) if m.get("role") == "user") + rewritten = [ + {**m, "content": COMPRESSED_MARKER} if i == first_user else m for i, m in enumerate(messages) + ] + return {**inputs, "structured_messages": rewritten} + + +class ToolOutputRewriteGuardrail(CustomGuardrail): + """Guardrail that compresses the first tool-result row, the way Headroom does.""" + + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional[Any] = None, + ) -> GenericGuardrailAPIInputs: + messages = list(inputs.get("structured_messages") or []) + first_tool = next(i for i, m in enumerate(messages) if isinstance(m, dict) and m.get("role") == "tool") + rewritten = [ + {**m, "content": COMPRESSED_MARKER} if i == first_tool else m for i, m in enumerate(messages) + ] + return {**inputs, "structured_messages": rewritten} + + +class DroppingRewriteGuardrail(CustomGuardrail): + """Guardrail that rewrites the first user row and drops the last row, so the + rewrite can only land through the full-conversion fallback.""" + + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional[Any] = None, + ) -> GenericGuardrailAPIInputs: + messages = list(inputs.get("structured_messages") or []) + first_user = next(i for i, m in enumerate(messages) if isinstance(m, dict) and m.get("role") == "user") + rewritten = [ + {**m, "content": COMPRESSED_MARKER} if i == first_user else m for i, m in enumerate(messages) + ] + return {**inputs, "structured_messages": rewritten[:-1]} + + +def _texts(item: dict) -> list[str]: + content = item.get("content") + if isinstance(content, str): + return [content] + return [part["text"] for part in content] + + +class TestStructuredMessagesWriteBack: + """A guardrail's structured_messages rewrite must land in the Responses request, + not only the per-text mapping the chat handler shares with it.""" + + @pytest.mark.asyncio + async def test_list_input_gets_rewritten_messages_and_keeps_instructions(self): + handler = OpenAIResponsesHandler() + data = { + "model": "gpt-5.6", + "instructions": "Answer from the memo only.", + "input": [ + {"role": "user", "content": "memo " * 400}, + {"role": "assistant", "content": "Understood."}, + {"role": "user", "content": "What is the codename?"}, + ], + } + + result = await handler.process_input_messages(data, StructuredRewriteGuardrail()) + + assert result["instructions"] == "Answer from the memo only." + user_items = [item for item in result["input"] if item.get("role") == "user"] + assert [_texts(item) for item in user_items] == [[COMPRESSED_MARKER], ["What is the codename?"]] + assert not any(item.get("role") == "system" for item in result["input"]) + assert _texts(next(item for item in result["input"] if item.get("role") == "assistant")) == ["Understood."] + + @pytest.mark.asyncio + async def test_string_input_becomes_rewritten_message_list(self): + handler = OpenAIResponsesHandler() + data = {"model": "gpt-5.6", "input": "memo " * 400} + + result = await handler.process_input_messages(data, StructuredRewriteGuardrail()) + + assert [_texts(item) for item in result["input"]] == [[COMPRESSED_MARKER]] + assert "instructions" not in result + + @pytest.mark.asyncio + async def test_developer_item_preserved_verbatim_by_row_patch(self): + handler = OpenAIResponsesHandler() + developer_item = {"role": "developer", "content": "Always answer in French."} + data = { + "model": "gpt-5.6", + "input": [ + developer_item, + {"role": "user", "content": "memo " * 400}, + {"role": "user", "content": "What is the codename?"}, + ], + } + + result = await handler.process_input_messages(data, StructuredRewriteGuardrail()) + + assert result["input"][0] is developer_item + assert developer_item["content"] == "Always answer in French." + assert _texts(result["input"][1]) == [COMPRESSED_MARKER] + assert _texts(result["input"][2]) == ["What is the codename?"] + + @pytest.mark.asyncio + async def test_reasoning_and_function_call_items_survive_tool_output_compression(self): + handler = OpenAIResponsesHandler() + reasoning_item = { + "id": "rs_123", + "type": "reasoning", + "summary": [], + "encrypted_content": "gAAAAA-signed-reasoning", + } + function_call_item = { + "id": "fc_123", + "type": "function_call", + "call_id": "call_abc", + "name": "read_document", + "arguments": '{"path": "memo.txt"}', + "status": "completed", + } + data = { + "model": "gpt-5.6", + "instructions": "Answer from the memo only.", + "input": [ + reasoning_item, + function_call_item, + {"type": "function_call_output", "call_id": "call_abc", "output": "memo " * 400}, + {"role": "user", "content": "What is the codename?"}, + ], + } + + result = await handler.process_input_messages(data, ToolOutputRewriteGuardrail()) + + assert result["instructions"] == "Answer from the memo only." + assert result["input"][0] is reasoning_item + assert reasoning_item["encrypted_content"] == "gAAAAA-signed-reasoning" + assert result["input"][1] is function_call_item + assert function_call_item["id"] == "fc_123" + assert result["input"][2] == { + "type": "function_call_output", + "call_id": "call_abc", + "output": COMPRESSED_MARKER, + } + assert result["input"][3] == {"role": "user", "content": "What is the codename?"} + + @pytest.mark.asyncio + async def test_web_search_call_item_preserved_verbatim(self): + handler = OpenAIResponsesHandler() + web_search_item = { + "id": "ws_123", + "type": "web_search_call", + "status": "completed", + "action": {"type": "search", "query": "codename memo"}, + } + data = { + "model": "gpt-5.6", + "input": [ + web_search_item, + {"role": "user", "content": "memo " * 400}, + {"role": "user", "content": "What is the codename?"}, + ], + } + + result = await handler.process_input_messages(data, StructuredRewriteGuardrail()) + + assert result["input"][0] is web_search_item + assert _texts(result["input"][1]) == [COMPRESSED_MARKER] + assert _texts(result["input"][2]) == ["What is the codename?"] + + @pytest.mark.asyncio + async def test_row_count_change_falls_back_to_full_conversion(self): + handler = OpenAIResponsesHandler() + data = { + "model": "gpt-5.6", + "input": [ + {"role": "developer", "content": "Always answer in French."}, + {"role": "user", "content": "memo " * 400}, + {"role": "user", "content": "What is the codename?"}, + ], + } + + result = await handler.process_input_messages(data, DroppingRewriteGuardrail()) + + assert len(result["input"]) == 2 + developer = next(item for item in result["input"] if item.get("role") == "developer") + assert developer["content"] == [{"type": "input_text", "text": "Always answer in French."}] + assert _texts(next(item for item in result["input"] if item.get("role") == "user")) == [COMPRESSED_MARKER] + + @pytest.mark.asyncio + async def test_same_inputs_object_back_keeps_the_text_mapping(self): + handler = OpenAIResponsesHandler() + original_input = [ + {"role": "user", "content": "Hello"}, + {"role": "user", "content": [{"type": "input_text", "text": "Again"}]}, + ] + data = {"model": "gpt-5.6", "input": original_input} + + result = await handler.process_input_messages(data, MockGuardrail()) + + assert result["input"] is original_input + assert [_texts(item) for item in result["input"]] == [["Hello [GUARDRAILED]"], ["Again [GUARDRAILED]"]] + + +class AllToolOutputsRewriteGuardrail(CustomGuardrail): + """Guardrail that compresses every tool-result row.""" + + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional[Any] = None, + ) -> GenericGuardrailAPIInputs: + messages = list(inputs.get("structured_messages") or []) + rewritten = [ + {**m, "content": COMPRESSED_MARKER} if isinstance(m, dict) and m.get("role") == "tool" else m + for m in messages + ] + return {**inputs, "structured_messages": rewritten} + + +class AssistantRewriteGuardrail(CustomGuardrail): + """Guardrail that rewrites the first assistant row's content.""" + + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional[Any] = None, + ) -> GenericGuardrailAPIInputs: + messages = list(inputs.get("structured_messages") or []) + first = next(i for i, m in enumerate(messages) if isinstance(m, dict) and m.get("role") == "assistant") + rewritten = [{**m, "content": COMPRESSED_MARKER} if i == first else m for i, m in enumerate(messages)] + return {**inputs, "structured_messages": rewritten} + + +class DictStructuredMessagesGuardrail(CustomGuardrail): + """Guardrail that hands back a raw evaluation dict instead of a message list, + the way HiddenLayer v2 does.""" + + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional[Any] = None, + ) -> GenericGuardrailAPIInputs: + return {**inputs, "structured_messages": {"evaluation": "allowed", "messages": []}} + + +def _parallel_tool_call_input() -> list: + return [ + {"id": "fc_1", "type": "function_call", "call_id": "call_1", "name": "read_a", "arguments": "{}"}, + {"id": "fc_2", "type": "function_call", "call_id": "call_2", "name": "read_b", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "call_1", "output": "memo " * 400}, + {"type": "function_call_output", "call_id": "call_2", "output": "note " * 400}, + {"role": "user", "content": "What is the codename?"}, + ] + + +class TestProvenancePatching: + """The O(n) provenance pass must keep patching rewritten rows in place for the + shapes real agent loops produce, and fall back safely everywhere else.""" + + @pytest.mark.asyncio + async def test_parallel_tool_call_outputs_both_patched(self): + handler = OpenAIResponsesHandler() + raw_input = _parallel_tool_call_input() + fc_1, fc_2 = raw_input[0], raw_input[1] + data = {"model": "gpt-5.6", "input": raw_input} + + result = await handler.process_input_messages(data, AllToolOutputsRewriteGuardrail()) + + assert result["input"][0] is fc_1 + assert result["input"][1] is fc_2 + assert result["input"][2] == {"type": "function_call_output", "call_id": "call_1", "output": COMPRESSED_MARKER} + assert result["input"][3] == {"type": "function_call_output", "call_id": "call_2", "output": COMPRESSED_MARKER} + assert result["input"][4] == {"role": "user", "content": "What is the codename?"} + + @pytest.mark.asyncio + async def test_assistant_turn_with_tool_call_keeps_items_verbatim(self): + handler = OpenAIResponsesHandler() + assistant_item = {"role": "assistant", "content": "Let me read the memo."} + function_call_item = { + "id": "fc_9", + "type": "function_call", + "call_id": "call_9", + "name": "read_document", + "arguments": '{"path": "memo.txt"}', + } + data = { + "model": "gpt-5.6", + "input": [ + assistant_item, + function_call_item, + {"type": "function_call_output", "call_id": "call_9", "output": "memo " * 400}, + {"role": "user", "content": "What is the codename?"}, + ], + } + + result = await handler.process_input_messages(data, ToolOutputRewriteGuardrail()) + + assert result["input"][0] is assistant_item + assert result["input"][1] is function_call_item + assert result["input"][2] == {"type": "function_call_output", "call_id": "call_9", "output": COMPRESSED_MARKER} + + @pytest.mark.asyncio + async def test_rewrite_of_merged_tool_call_message_falls_back(self): + handler = OpenAIResponsesHandler() + raw_input = _parallel_tool_call_input() + data = {"model": "gpt-5.6", "input": raw_input} + + result = await handler.process_input_messages(data, AssistantRewriteGuardrail()) + + assert not any(item is original for item in result["input"] for original in raw_input) + assistant_items = [item for item in result["input"] if item.get("role") == "assistant"] + assert [_texts(item) for item in assistant_items] == [[COMPRESSED_MARKER]] + + @pytest.mark.asyncio + async def test_rewrite_of_lone_function_call_message_falls_back(self): + handler = OpenAIResponsesHandler() + data = { + "model": "gpt-5.6", + "input": [ + {"id": "fc_1", "type": "function_call", "call_id": "call_1", "name": "read_a", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "call_1", "output": "memo memo"}, + {"role": "user", "content": "What is the codename?"}, + ], + } + + raw_input = data["input"] + result = await handler.process_input_messages(data, AssistantRewriteGuardrail()) + + assert not any(item is original for item in result["input"] for original in raw_input) + assistant_items = [item for item in result["input"] if item.get("role") == "assistant"] + assert [_texts(item) for item in assistant_items] == [[COMPRESSED_MARKER]] + + def test_provenance_bails_on_non_mapping_item(self): + from litellm.llms.openai.responses.guardrail_translation.handler import _input_item_provenance + + assert _input_item_provenance(["not a mapping"], []) is None + + def test_provenance_bails_when_expected_messages_disagree(self): + from litellm.llms.openai.responses.guardrail_translation.handler import _input_item_provenance + + assert _input_item_provenance([{"role": "user", "content": "hi"}], [{"role": "user", "content": "bye"}]) is None + + def test_provenance_bails_on_unpredicted_merge(self): + from litellm.llms.openai.responses.guardrail_translation.handler import _input_item_provenance + from litellm.responses.litellm_completion_transformation.transformation import ( + LiteLLMCompletionResponsesConfig, + ) + + raw_input = [ + {"id": "fc_1", "type": "function_call", "call_id": "call_1", "name": "read_a", "arguments": "{}"}, + {"role": "assistant", "content": "Reading the memo now."}, + ] + expected = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=raw_input, responses_api_request={} + ) + assert len(expected) == 1 + assert _input_item_provenance(raw_input, expected) is None + + def test_provenance_maps_and_taints_parallel_tool_calls(self): + from litellm.llms.openai.responses.guardrail_translation.handler import _input_item_provenance + from litellm.responses.litellm_completion_transformation.transformation import ( + LiteLLMCompletionResponsesConfig, + ) + + raw_input = _parallel_tool_call_input() + expected = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=raw_input, responses_api_request={} + ) + provenance = _input_item_provenance(raw_input, expected) + assert provenance is not None + item_for_message, tainted = provenance + assert tainted == {0} + assert dict(item_for_message) == {1: 2, 2: 3, 3: 4} + + +class TestDictStructuredMessagesGuard: + """A guardrail handing back a non-list structured_messages payload must not + blow up the request; the write-back is skipped instead.""" + + @pytest.mark.asyncio + async def test_list_input_survives_dict_structured_messages(self): + handler = OpenAIResponsesHandler() + original_input = [{"role": "user", "content": "Hello"}] + data = {"model": "gpt-5.6", "input": original_input} + + result = await handler.process_input_messages(data, DictStructuredMessagesGuardrail()) + + assert result["input"] is original_input + assert result["input"] == [{"role": "user", "content": "Hello"}] + + @pytest.mark.asyncio + async def test_string_input_survives_dict_structured_messages(self): + handler = OpenAIResponsesHandler() + data = {"model": "gpt-5.6", "input": "Hello there"} + + result = await handler.process_input_messages(data, DictStructuredMessagesGuardrail()) + + assert result["input"] == "Hello there" + + +class SystemRewriteGuardrail(CustomGuardrail): + """Guardrail that rewrites the system row, the way prompt-hardening guardrails do.""" + + def __init__(self, rewritten_content: Any = COMPRESSED_MARKER): + super().__init__() + self.rewritten_content = rewritten_content + + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional[Any] = None, + ) -> GenericGuardrailAPIInputs: + messages = list(inputs.get("structured_messages") or []) + first = next(i for i, m in enumerate(messages) if isinstance(m, dict) and m.get("role") == "system") + rewritten = [ + {**m, "content": self.rewritten_content} if i == first else m for i, m in enumerate(messages) + ] + return {**inputs, "structured_messages": rewritten} + + +class TestPatchEdgeBranches: + @pytest.mark.asyncio + async def test_multimodal_user_item_rewritten_through_conversion(self): + handler = OpenAIResponsesHandler() + data = { + "model": "gpt-5.6", + "input": [ + {"role": "user", "content": [{"type": "input_text", "text": "memo " * 400}]}, + {"role": "user", "content": "What is the codename?"}, + ], + } + + result = await handler.process_input_messages(data, StructuredRewriteGuardrail()) + + assert _texts(result["input"][0]) == [COMPRESSED_MARKER] + assert result["input"][1] == {"role": "user", "content": "What is the codename?"} + + @pytest.mark.asyncio + async def test_instructions_rewrite_lands_in_instructions_field(self): + handler = OpenAIResponsesHandler() + user_item = {"role": "user", "content": "What is the codename?"} + data = { + "model": "gpt-5.6", + "instructions": "Answer from the memo only.", + "input": [user_item], + } + + result = await handler.process_input_messages(data, SystemRewriteGuardrail()) + + assert result["instructions"] == COMPRESSED_MARKER + assert result["input"][0] is user_item + + @pytest.mark.asyncio + async def test_non_string_instructions_rewrite_falls_back(self): + handler = OpenAIResponsesHandler() + user_item = {"role": "user", "content": "What is the codename?"} + data = { + "model": "gpt-5.6", + "instructions": "Answer from the memo only.", + "input": [user_item], + } + + result = await handler.process_input_messages( + data, SystemRewriteGuardrail(rewritten_content=[{"type": "text", "text": COMPRESSED_MARKER}]) + ) + + assert result["input"][0] is not user_item + + @pytest.mark.asyncio + async def test_unpredicted_merge_falls_back_through_patch(self): + handler = OpenAIResponsesHandler() + raw_input = [ + {"id": "fc_1", "type": "function_call", "call_id": "call_1", "name": "read_a", "arguments": "{}"}, + {"role": "assistant", "content": "Reading the memo now."}, + {"type": "function_call_output", "call_id": "call_1", "output": "memo memo"}, + {"role": "user", "content": "memo " * 400}, + ] + data = {"model": "gpt-5.6", "input": raw_input} + + result = await handler.process_input_messages(data, StructuredRewriteGuardrail()) + + assert not any(item is original for item in result["input"] for original in raw_input) + user_items = [item for item in result["input"] if item.get("role") == "user"] + assert _texts(user_items[0]) == [COMPRESSED_MARKER] + + def test_item_rewrite_field_ignores_non_string_type(self): + from litellm.llms.openai.responses.guardrail_translation.handler import _item_rewrite_field + + assert _item_rewrite_field({"type": 123, "content": "hello"}) is None + + class ToolEditingGuardrail(CustomGuardrail): """Guardrail that rewrites the flattened chat tools it was handed through ``edit``""" diff --git a/tests/test_litellm/llms/openai/test_openai.py b/tests/test_litellm/llms/openai/test_openai.py new file mode 100644 index 00000000000..136b837f191 --- /dev/null +++ b/tests/test_litellm/llms/openai/test_openai.py @@ -0,0 +1,52 @@ +import pytest + +from litellm.llms.openai.openai import OpenAIChatCompletion + + +@pytest.mark.parametrize( + "api_base", + [ + None, + "https://api.openai.com/v1", + "https://api.openai.com:443/v1", + "https://southcentralus.privatelink.api.openai.com/v1", + "https://eu.api.openai.com/v1", + "https://us.api.openai.com/v1", + "HTTPS://API.OPENAI.COM/v1/", + ], +) +def test_get_stream_options_defaults_include_usage_on_every_openai_backed_host(api_base): + """ + PrivateLink and regional hostnames reach the real OpenAI backend, so a stream with no caller + stream_options must ask for the usage chunk exactly as the default base does. Regression guard + for LIT-6875: spend for those deployments fell back to local token counting. + """ + assert OpenAIChatCompletion().get_stream_options(stream_options=None, api_base=api_base) == { + "stream_options": {"include_usage": True} + } + + +@pytest.mark.parametrize( + "api_base", + [ + "https://my-gateway.example/v1", + "https://api.openai.com.evil.example/v1", + "https://notapi.openai.com/v1", + "https://gateway.example/v1?upstream=api.openai.com", + "https://openai.internal.example/api.openai.com/v1", + ], +) +def test_get_stream_options_leaves_foreign_hosts_without_a_usage_default(api_base): + """Only the host decides: an OpenAI-compatible backend elsewhere may not support stream_options at all.""" + assert OpenAIChatCompletion().get_stream_options(stream_options=None, api_base=api_base) == {} + + +@pytest.mark.parametrize( + "api_base", + ["https://southcentralus.privatelink.api.openai.com/v1", "https://my-gateway.example/v1"], +) +def test_get_stream_options_passes_caller_stream_options_through_on_any_host(api_base): + caller_options = {"include_usage": False} + assert OpenAIChatCompletion().get_stream_options(stream_options=caller_options, api_base=api_base) == { + "stream_options": caller_options + } diff --git a/tests/test_litellm/llms/openai/test_openai_common_utils.py b/tests/test_litellm/llms/openai/test_openai_common_utils.py index 3ae29e411e8..d3c21c5bd5a 100644 --- a/tests/test_litellm/llms/openai/test_openai_common_utils.py +++ b/tests/test_litellm/llms/openai/test_openai_common_utils.py @@ -7,7 +7,7 @@ import pytest import litellm from litellm.litellm_core_utils.token_counter import token_counter -from litellm.llms.openai.common_utils import BaseOpenAILLM +from litellm.llms.openai.common_utils import BaseOpenAILLM, is_openai_backed_api_base # Test parameters for different API functions API_FUNCTION_PARAMS = [ @@ -392,3 +392,22 @@ async def test_async_genuine_bad_request_still_raises(provider, stream): with pytest.raises(litellm.BadRequestError): await _call_and_drain() + + +@pytest.mark.parametrize( + ("api_base", "expected"), + [ + ("https://api.openai.com/v1", True), + ("https://api.openai.com:443/v1/", True), + ("https://southcentralus.privatelink.api.openai.com/v1", True), + ("https://eu.api.openai.com/v1", True), + ("HTTPS://API.OPENAI.COM/v1", True), + ("https://my-gateway.example/v1", False), + ("https://api.openai.com.evil.example/v1", False), + ("https://notapi.openai.com/v1", False), + ("https://gateway.example/v1?upstream=api.openai.com", False), + ("not a url", False), + ], +) +def test_is_openai_backed_api_base_decides_by_hostname_only(api_base, expected): + assert is_openai_backed_api_base(api_base) is expected diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py index c04fb7b30ec..400eaf8ab3f 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py @@ -193,6 +193,33 @@ async def test_apply_guardrail_compresses_and_returns_structured_messages( assert "headroom" in _applied_guardrails(request_data) +@pytest.mark.asyncio +async def test_apply_guardrail_leaves_background_requests_uncompressed( + guardrail: HeadroomGuardrail, +): + inputs = GenericGuardrailAPIInputs( + texts=["A" * 5000], + structured_messages=ORIGINAL_MESSAGES, + ) + request_data = {"model": "gpt-4o", "background": True} + + with patch.object( + guardrail.async_handler, + "post", + new_callable=AsyncMock, + return_value=_make_compress_response(COMPRESSED_MESSAGES), + ) as post: + result = await guardrail.apply_guardrail( + inputs=inputs, + request_data=request_data, + input_type="request", + ) + + assert result is inputs + post.assert_not_awaited() + assert _recorded_guardrail_entries(request_data) == [] + + def _recorded_guardrail_response(request_data: dict) -> dict: entries = request_data["metadata"]["standard_logging_guardrail_information"] assert len(entries) == 1 @@ -954,6 +981,40 @@ async def test_passthrough_handler_does_not_log_headroom_as_run( assert "headroom" not in _applied_guardrails(data) +@pytest.mark.asyncio +async def test_responses_request_sends_compressed_input_and_retrieve_tool_upstream( + guardrail: HeadroomGuardrail, +): + """Regression for LIT-6494: on /v1/responses the compressed messages must be + written back into `input`, not only the retrieve tool into `tools`, or the + model keeps reading the full document and never calls headroom_retrieve.""" + from litellm.llms.openai.responses.guardrail_translation.handler import OpenAIResponsesHandler + + data = { + "model": "gpt-5.6", + "instructions": ORIGINAL_MESSAGES[0]["content"], + "input": [{"role": m["role"], "content": m["content"]} for m in ORIGINAL_MESSAGES[1:]], + "tools": [{"type": "function", "name": "get_weather", "parameters": {"type": "object", "properties": {}}}], + } + + with patch.object( + guardrail.async_handler, + "post", + new_callable=AsyncMock, + return_value=_make_compress_response(COMPRESSED_MESSAGES_WITH_HASH), + ): + result = await OpenAIResponsesHandler().process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert result["instructions"] == ORIGINAL_MESSAGES[0]["content"] + assert [item["content"] for item in result["input"]] == [ + COMPRESSED_MESSAGES_WITH_HASH[0]["content"], + ORIGINAL_MESSAGES[2]["content"], + ORIGINAL_MESSAGES[3]["content"], + ] + assert "A" * 5000 not in json.dumps(result["input"]) + assert [tool["name"] for tool in result["tools"]] == ["get_weather", HEADROOM_RETRIEVE_TOOL_NAME] + + @pytest.mark.asyncio async def test_apply_guardrail_http_error_raises(): guardrail = _make_guardrail() @@ -1950,6 +2011,58 @@ def _openai_text_payload(content: str) -> dict: return _openai_completion_payload({"role": "assistant", "content": content}, "stop") +def _responses_retrieve_tool_definition() -> dict: + return {"type": "function", **_retrieve_tool_definition()["function"]} + + +def _openai_responses_payload(output_item: dict) -> dict: + return { + "id": "resp_ccr", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": "gpt-4o", + "output": [output_item], + "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15}, + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "error": None, + "incomplete_details": None, + "instructions": None, + "metadata": {}, + "temperature": 1.0, + "top_p": 1.0, + "text": {"format": {"type": "text"}}, + "truncation": "disabled", + } + + +def _openai_responses_retrieve_call_payload() -> dict: + return _openai_responses_payload( + { + "type": "function_call", + "id": "fc_ccr", + "call_id": "call_ccr", + "name": HEADROOM_RETRIEVE_TOOL_NAME, + "arguments": json.dumps({"hash": CCR_HASH}), + "status": "completed", + } + ) + + +def _openai_responses_text_payload(text: str) -> dict: + return _openai_responses_payload( + { + "type": "message", + "id": "msg_ccr", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": text, "annotations": []}], + } + ) + + @pytest.mark.parametrize( "call_type, stream, tools, expect_conversion", [ @@ -1958,12 +2071,14 @@ def _openai_text_payload(content: str) -> dict: (CallTypes.acompletion, False, [_retrieve_tool_definition()], False), (CallTypes.acompletion, True, [{"type": "function", "function": {"name": "get_weather"}}], False), (CallTypes.acompletion, True, None, False), - (CallTypes.aresponses, True, [_retrieve_tool_definition()], False), + (CallTypes.aresponses, True, [_retrieve_tool_definition()], True), + (CallTypes.responses, True, [_responses_retrieve_tool_definition()], True), + (CallTypes.aresponses, False, [_retrieve_tool_definition()], False), (CallTypes.anthropic_messages, True, [_retrieve_tool_definition()], False), ], ) @pytest.mark.asyncio -async def test_pre_call_deployment_hook_converts_stream_only_for_ccr_chat_completions( +async def test_pre_call_deployment_hook_converts_stream_only_for_ccr_chat_completions_and_responses( guardrail: HeadroomGuardrail, call_type: CallTypes, stream: bool, @@ -1986,6 +2101,22 @@ async def test_pre_call_deployment_hook_converts_stream_only_for_ccr_chat_comple assert kwargs["stream"] is True +@pytest.mark.asyncio +async def test_pre_call_deployment_hook_leaves_background_streams_alone(guardrail: HeadroomGuardrail): + kwargs = { + "model": "gpt-4o", + "stream": True, + "background": True, + "tools": [_responses_retrieve_tool_definition()], + } + + result = await guardrail.async_pre_call_deployment_hook(kwargs=kwargs, call_type=CallTypes.aresponses) + + assert result is kwargs + assert HEADROOM_CONVERTED_STREAM_KEY not in kwargs + assert kwargs["stream"] is True + + @pytest.mark.asyncio async def test_pre_call_deployment_hook_still_compresses_for_deployment_level_configs( guardrail: HeadroomGuardrail, @@ -2094,6 +2225,117 @@ async def test_streaming_chat_completion_resolves_ccr_retrieval_end_to_end( assert not any(key.startswith("_headroom_interception") for key in followup_body) +@pytest.mark.asyncio +async def test_streaming_responses_resolves_ccr_retrieval_end_to_end( + guardrail: HeadroomGuardrail, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +): + """Regression test for LIT-6481: streaming /v1/responses must resolve the + retrieve tool call server-side exactly like streaming /chat/completions does, + instead of streaming a headroom_retrieve function_call to the client.""" + original_content = "the full uncompressed document" + final_answer = "the document says hello" + guardrail._issued_hashes_by_call_id["ccr-call-id"] = ( + frozenset({CCR_HASH}), + time.monotonic() + 999, + ) + + monkeypatch.setenv("OPENAI_API_KEY", "sk-test") + monkeypatch.setattr(litellm, "callbacks", [guardrail]) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + upstream = respx_mock.post("https://api.openai.com/v1/responses").mock( + side_effect=[ + httpx.Response(200, json=_openai_responses_retrieve_call_payload()), + httpx.Response(200, json=_openai_responses_text_payload(final_answer)), + ] + ) + + with patch.object( + guardrail.async_handler, + "get", + new_callable=AsyncMock, + return_value=_make_retrieve_response(original_content), + ) as mock_get: + response = await litellm.aresponses( + model="openai/gpt-4o", + input=[{"role": "user", "content": f"summarize hash={CCR_HASH}"}], + tools=[_responses_retrieve_tool_definition()], + stream=True, + litellm_call_id="ccr-call-id", + ) + events = [event async for event in response] + + streamed_text = "".join( + getattr(event, "delta", "") for event in events if getattr(event, "type", None) == "response.output_text.delta" + ) + assert streamed_text == final_answer + assert not any("function_call" in str(getattr(event, "type", "")) for event in events) + assert not any( + getattr(getattr(event, "item", None), "type", None) == "function_call" for event in events + ) + mock_get.assert_called_once() + assert CCR_HASH in (mock_get.call_args.kwargs.get("url") or mock_get.call_args.args[0]) + + assert len(upstream.calls) == 2 + followup_body = json.loads(upstream.calls[1].request.content) + assert not followup_body.get("stream") + assert original_content in json.dumps(followup_body["input"]) + assert not any(key.startswith("_headroom_interception") for key in followup_body) + + +def test_sync_streaming_responses_resolves_ccr_retrieval_end_to_end( + guardrail: HeadroomGuardrail, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +): + """The synchronous responses() path converts the stream the same way, so it + must hand back a stream iterator with the resolved answer rather than the + completed response object.""" + original_content = "the full uncompressed document" + final_answer = "the document says hello" + guardrail._issued_hashes_by_call_id["ccr-call-id"] = ( + frozenset({CCR_HASH}), + time.monotonic() + 999, + ) + + monkeypatch.setenv("OPENAI_API_KEY", "sk-test") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "callbacks", [guardrail]) + upstream = respx_mock.post("https://api.openai.com/v1/responses").mock( + side_effect=[ + httpx.Response(200, json=_openai_responses_retrieve_call_payload()), + httpx.Response(200, json=_openai_responses_text_payload(final_answer)), + ] + ) + + with patch.object( + guardrail.async_handler, + "get", + new_callable=AsyncMock, + return_value=_make_retrieve_response(original_content), + ) as mock_get: + response = litellm.responses( + model="openai/gpt-4o", + input=[{"role": "user", "content": f"summarize hash={CCR_HASH}"}], + tools=[_responses_retrieve_tool_definition()], + stream=True, + litellm_call_id="ccr-call-id", + ) + events = list(response) + + streamed_text = "".join( + getattr(event, "delta", "") for event in events if getattr(event, "type", None) == "response.output_text.delta" + ) + assert streamed_text == final_answer + assert not any( + getattr(getattr(event, "item", None), "type", None) == "function_call" for event in events + ) + mock_get.assert_called_once() + assert len(upstream.calls) == 2 + assert not json.loads(upstream.calls[1].request.content).get("stream") + + # --------------------------------------------------------------------------- # LIT-5018: the turn the model is being asked to act on is never compressed. # diff --git a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py index f231eb66a50..022aeff4e20 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py @@ -1449,6 +1449,11 @@ async def test_new_user_default_teams_flow(mocker): return 5 # Low user count, under limit mock_prisma_client.db.litellm_usertable.count = mock_count + persisted_user_row = mocker.MagicMock() + persisted_user_row.teams = ["96fed65b-0182-4ff4-8429-2721cd7d42af"] + mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock( + return_value=persisted_user_row + ) # Mock duplicate checks to pass async def mock_check_duplicate_user_email(*args, **kwargs): @@ -1477,6 +1482,7 @@ async def test_new_user_default_teams_flow(mocker): "token": "sk-test-token-123", "expires": None, "max_budget": 100, + "teams": [], } # Mock _add_user_to_team @@ -1551,6 +1557,7 @@ async def test_new_user_default_teams_flow(mocker): # Verify response structure assert response.user_id == "test-user-123" assert response.key == "sk-test-token-123" + assert response.teams == ["96fed65b-0182-4ff4-8429-2721cd7d42af"] finally: # Restore original default params (always assign, never delattr — the attribute diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 68704f476b5..7954a4693cc 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -17505,6 +17505,18 @@ def test_generate_key_request_blank_team_id_is_personal(): assert GenerateKeyRequest(team_id="team-1").team_id == "team-1" +def test_generate_key_request_blank_organization_and_project_id_are_unset(): + from litellm.proxy._types import RegenerateKeyRequest + + cleared = GenerateKeyRequest(organization_id="", project_id="") + assert cleared.organization_id is None + assert cleared.project_id is None + assert "organization_id" not in cleared.model_dump(exclude_none=True) + assert RegenerateKeyRequest(organization_id="").organization_id is None + assert GenerateKeyRequest(organization_id="org-1", project_id="proj-1").organization_id == "org-1" + assert GenerateKeyRequest(organization_id="org-1", project_id="proj-1").project_id == "proj-1" + + def test_key_request_blank_organization_id_is_unset(): from litellm.proxy._types import RegenerateKeyRequest, UpdateKeyRequest diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index 088e82b370f..166fcb2863c 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -9897,11 +9897,11 @@ class TestResolveTeamAccessGroupResources: assert resolved.access_group_mcp_server_ids == ["mcp-1"] assert resolved.access_group_agent_ids == ["agent-1"] assert [ - (d.access_group_id, d.access_group_name, d.models) + (d.access_group_id, d.access_group_name, d.models, d.mcp_server_ids, d.agent_ids) for d in (resolved.access_group_details or []) ] == [ - ("ag-1", "shared-models", ("gpt-4", "claude-3")), - ("ag-2", "extra-models", ("claude-3", "gemini")), + ("ag-1", "shared-models", ("gpt-4", "claude-3"), ("mcp-1",), ()), + ("ag-2", "extra-models", ("claude-3", "gemini"), (), ("agent-1",)), ] @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index d3f17c73499..91367507247 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -5730,3 +5730,38 @@ async def test_pass_through_request_leaves_cost_router_logger_working(): verbose_logger.removeHandler(recorder) assert not raised, f"cost router logger raised on the passthrough logging params: {raised[0].exc_info}" + + +@pytest.mark.parametrize("client_metadata_key", ["litellm_metadata", "metadata"]) +def test_passthrough_client_cannot_forge_session_id_omission(client_metadata_key: str): + """The omit marker is proxy-owned: only the pre-call policy may set it. A pass-through body that carries + it in its own metadata must not null out SpendLogs.session_id on a request the proxy never omitted.""" + from litellm.constants import SESSION_ID_OMITTED_METADATA_KEY + from litellm.proxy.spend_tracking.spend_tracking_utils import _get_session_id_for_spend_log + + mock_request = MagicMock(spec=Request) + mock_request.method = "POST" + mock_request.url = "http://0.0.0.0:4000/gemini/v1beta/models/gemini-2.5-flash:generateContent" + mock_request.headers = Headers({}) + mock_request.scope = {} + + kwargs = HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint( + request=mock_request, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + passthrough_logging_payload=MagicMock(), + logging_obj=MagicMock(), + _parsed_body={client_metadata_key: {SESSION_ID_OMITTED_METADATA_KEY: True}}, + litellm_call_id="lit-6694-call-id", + ) + + metadata = kwargs["litellm_params"]["metadata"] + assert SESSION_ID_OMITTED_METADATA_KEY not in metadata + assert ( + _get_session_id_for_spend_log( + kwargs={}, + metadata=metadata, + standard_logging_payload={"trace_id": "per-call-random-trace-id"}, + omit_when_missing=bool(metadata.get(SESSION_ID_OMITTED_METADATA_KEY)), + ) + == "per-call-random-trace-id" + ) diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py b/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py index 961479c0393..6735f2a3780 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py @@ -5,6 +5,7 @@ import pytest from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( _base_vertex_proxy_route, + _upstream_headers_for_vertex_route, ) from litellm.types.router import DeploymentTypedDict @@ -348,6 +349,93 @@ async def test_vertex_passthrough_forwards_anthropic_beta_header(): assert headers_passed_through is False +VERTEX_ANTHROPIC_MODELS_PREFIX = "v1/projects/test-project/locations/global/publishers/anthropic/models/" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model_segment", "expects_anthropic_beta"), + [ + ("count-tokens:rawPredict", False), + ("claude-sonnet-4-6:streamRawPredict", True), + ], +) +async def test_vertex_passthrough_drops_anthropic_beta_only_on_count_tokens( + model_segment: str, expects_anthropic_beta: bool +): + with ( + patch( # test-quality-ok: the route reads this proxy global at call time, nothing injects it + "litellm.proxy.proxy_server.llm_router", None + ), + patch( # test-quality-ok: the route reads this proxy global at call time, nothing injects it + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.passthrough_endpoint_router" + ) as mock_pt_router, + patch( # test-quality-ok: the route offers no injection point for its header preparation + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints._prepare_vertex_auth_headers", + new_callable=AsyncMock, + ) as mock_prep_headers, + patch( # test-quality-ok: the upstream call is captured here, the route offers no injection point + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route" + ) as mock_create_route, + patch( # test-quality-ok: the route calls auth directly rather than through Depends + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.user_api_key_auth", + new_callable=AsyncMock, + ) as mock_auth, + patch( # test-quality-ok: the route reads the request body for this, a MagicMock request has none + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.is_streaming_request_fn", + new_callable=AsyncMock, + return_value=False, + ), + ): + mock_pt_router.get_vertex_credentials.return_value = MagicMock() + mock_prep_headers.return_value = ( + { + "anthropic-beta": "tool-search-tool-2025-10-19,web-search-2025-03-05", + "content-type": "application/json", + "Authorization": "Bearer vertex-access-token", + }, + "https://aiplatform.googleapis.com", + False, + "test-project", + "global", + ) + mock_create_route.return_value = AsyncMock() + mock_auth.return_value = UserAPIKeyAuth(api_key="sk-litellm-secret-key") + + await _base_vertex_proxy_route( + endpoint=f"{VERTEX_ANTHROPIC_MODELS_PREFIX}{model_segment}", + request=MagicMock(), + fastapi_response=MagicMock(), + get_vertex_pass_through_handler=MagicMock(), + ) + + upstream_headers = mock_create_route.call_args.kwargs["custom_headers"] + assert ("anthropic-beta" in upstream_headers) is expects_anthropic_beta + assert upstream_headers["Authorization"] == "Bearer vertex-access-token" + assert upstream_headers["content-type"] == "application/json" + + +def test_upstream_headers_for_vertex_route_filters_anthropic_beta_by_route(): + headers = { + "Anthropic-Beta": "effort-2025-11-24", + "content-type": "application/json", + "Authorization": "Bearer vertex-access-token", + } + + count_tokens_headers = _upstream_headers_for_vertex_route( + f"{VERTEX_ANTHROPIC_MODELS_PREFIX}count-tokens:rawPredict", headers + ) + model_headers = _upstream_headers_for_vertex_route( + f"{VERTEX_ANTHROPIC_MODELS_PREFIX}claude-sonnet-4-6:rawPredict", headers + ) + + assert dict(count_tokens_headers) == { + "content-type": "application/json", + "Authorization": "Bearer vertex-access-token", + } + assert dict(model_headers) == headers + + @pytest.mark.asyncio async def test_vertex_passthrough_does_not_forward_litellm_auth_token(): """ diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 60a32102946..f4cd8814bc1 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -4234,6 +4234,91 @@ async def test_build_ui_spend_logs_response_sums_multi_round_session_spend(): assert call_args[2] == [api_key] +@pytest.mark.asyncio +async def test_build_ui_spend_logs_response_sums_multi_round_session_tokens(): + """ + Regression test for LIT-4929: the logs table showed the summed session cost but + only the last call's token usage. Every row of a multi-round session must carry + the session-wide prompt, completion and total token sums from the aggregate + query, while rows outside a session carry none of them. + """ + from litellm.proxy.spend_tracking.spend_management_endpoints import ( + _build_ui_spend_logs_response, + ) + + session_id = "sess-multi-round-tokens" + api_key = "hashed-key-xyz" + dict_rows = [ + { + "request_id": "req-1", + "session_id": session_id, + "call_type": "completion", + "api_key": api_key, + "total_tokens": 10, + "prompt_tokens": 7, + "completion_tokens": 3, + }, + { + "request_id": "req-2", + "session_id": session_id, + "call_type": "completion", + "api_key": api_key, + "total_tokens": 50, + "prompt_tokens": 35, + "completion_tokens": 15, + }, + { + "request_id": "req-3", + "session_id": None, + "call_type": "completion", + "api_key": api_key, + "total_tokens": 5, + "prompt_tokens": 4, + "completion_tokens": 1, + }, + ] + + mock_prisma = MagicMock() + mock_prisma.db.query_raw = AsyncMock( + return_value=[ + { + "session_id": session_id, + "api_key": api_key, + "session_total_count": 2, + "session_total_spend": 0.06, + "mcp_tool_call_count": 0, + "mcp_tool_call_spend": 0.0, + "session_total_prompt_tokens": 42, + "session_total_completion_tokens": 18, + "session_total_tokens": 60, + } + ] + ) + + result = await _build_ui_spend_logs_response( + prisma_client=mock_prisma, + data=dict_rows, + total_records=3, + page=1, + page_size=50, + total_pages=1, + enrich_session_counts=True, + ) + + rows = result["data"] + session_rows = rows[:2] + assert [row["session_total_tokens"] for row in session_rows] == [60, 60] + assert [row["session_total_prompt_tokens"] for row in session_rows] == [42, 42] + assert [row["session_total_completion_tokens"] for row in session_rows] == [18, 18] + assert [(row["total_tokens"], row["prompt_tokens"], row["completion_tokens"]) for row in session_rows] == [ + (10, 7, 3), + (50, 35, 15), + ] + + token_keys = ("session_total_tokens", "session_total_prompt_tokens", "session_total_completion_tokens") + assert all(key not in rows[2] for key in token_keys) + + @pytest.mark.asyncio async def test_build_ui_spend_logs_response_session_cache_hit_count(): """ diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 9e5917637a8..323930eee60 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -14,6 +14,7 @@ from litellm.constants import ( LITELLM_TRUNCATED_PAYLOAD_FIELD, LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, REDACTED_BY_LITELM_STRING, + SESSION_ID_OMITTED_METADATA_KEY, ) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy.spend_tracking.spend_tracking_utils import ( @@ -21,6 +22,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( _get_proxy_server_request_for_spend_logs_payload, _get_request_duration_ms, _get_response_for_spend_logs_payload, + _get_session_id_for_spend_log, _get_spend_logs_metadata, _get_vector_store_request_for_spend_logs_payload, _is_master_key, @@ -33,6 +35,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( get_logging_payload, get_spend_logs_id, ) +from litellm.proxy._types import SpendLogsPayload from litellm.proxy.utils import hash_token from litellm.types.utils import ( StandardLoggingHiddenParams, @@ -74,6 +77,110 @@ def test_get_logging_payload_maps_openai_cached_tokens_to_cache_read_input_token assert additional_usage_values["prompt_tokens_details"]["cached_tokens"] == 123 +_TRACE_ONLY_STANDARD_LOGGING: Final = cast( + StandardLoggingPayload, + { + "trace_id": "trace-abc", + "session_id": "trace-abc", + "metadata": {}, + "model_map_information": None, + "request_tags": [], + }, +) + + +def _trace_only_session_id(omit_when_missing: bool) -> str | None: + """get_litellm_params copies metadata.trace_id into litellm_session_id, so every field echoes the trace id.""" + return _get_session_id_for_spend_log( + kwargs={"litellm_trace_id": "trace-abc", "litellm_session_id": "trace-abc"}, + metadata={"trace_id": "trace-abc"}, + standard_logging_payload=_TRACE_ONLY_STANDARD_LOGGING, + omit_when_missing=omit_when_missing, + ) + + +def test_omit_leaves_session_id_none_when_only_a_trace_id_exists(): + assert _trace_only_session_id(omit_when_missing=True) is None + + +def test_omit_leaves_session_id_none_without_any_ids(): + assert ( + _get_session_id_for_spend_log(kwargs={}, metadata=None, standard_logging_payload=None, omit_when_missing=True) + is None + ) + + +def test_omit_records_metadata_session_id(): + session_id: Final = _get_session_id_for_spend_log( + kwargs={"litellm_session_id": "chain-1"}, + metadata={"trace_id": "chain-1", "session_id": "chain-1"}, + standard_logging_payload=_TRACE_ONLY_STANDARD_LOGGING, + omit_when_missing=True, + ) + assert session_id == "chain-1" + + +def test_legacy_policy_keeps_trace_id_fallback(): + assert _trace_only_session_id(omit_when_missing=False) == "trace-abc" + generated: Final = _get_session_id_for_spend_log( + kwargs={}, metadata=None, standard_logging_payload=None, omit_when_missing=False + ) + assert len(str(generated)) == 36 + + +@pytest.mark.parametrize( + ("request_metadata", "expected"), + [ + ({"trace_id": "trace-abc"}, "trace-abc"), + ({"trace_id": "trace-abc", SESSION_ID_OMITTED_METADATA_KEY: True}, None), + ({"trace_id": "trace-abc", "session_id": "chain-1", SESSION_ID_OMITTED_METADATA_KEY: True}, "chain-1"), + ], +) +def test_get_logging_payload_reads_omit_decision_stamped_on_request( + request_metadata: dict[str, object], expected: str | None +): + """The pre-call stamp, not the live general_settings, decides the policy, so a config reload between + pre-call and spend logging cannot fabricate a session for a request accepted under `omit`.""" + with patch( # test-quality-ok: proves log time ignores proxy config; general_settings is yaml, not an HTTP boundary + "litellm.proxy.proxy_server.general_settings", {"missing_session_id": "generate"} + ): + payload: SpendLogsPayload = get_logging_payload( + kwargs={ + "model": "gpt-4o-mini", + "litellm_trace_id": "trace-abc", + "litellm_params": {"litellm_session_id": "trace-abc", "metadata": request_metadata}, + "standard_logging_object": _TRACE_ONLY_STANDARD_LOGGING, + }, + response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + assert payload["session_id"] == expected + + +@pytest.mark.parametrize("policy", ["omit", "generate", None]) +def test_get_logging_payload_applies_omit_to_requests_that_carry_no_stamp(policy: str | None): + """Router-model passthrough calls `allm_passthrough_route` directly and never reaches the pre-call helper that + stamps the omit decision, so an unstamped request falls back to the configured policy. Without that fallback + `missing_session_id: omit` would fabricate a uuid session id on every passthrough spend log while its Langfuse + trace has none, which is the divergence the policy exists to remove.""" + with patch( # test-quality-ok: general_settings is proxy config, loaded from yaml, not an HTTP boundary + "litellm.proxy.proxy_server.general_settings", {} if policy is None else {"missing_session_id": policy} + ): + payload: SpendLogsPayload = get_logging_payload( + kwargs={ + "model": "claude-opus-4", + "litellm_trace_id": "trace-abc", + "litellm_params": {"litellm_session_id": "trace-abc", "metadata": {"trace_id": "trace-abc"}}, + "standard_logging_object": _TRACE_ONLY_STANDARD_LOGGING, + }, + response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + assert payload["session_id"] == (None if policy == "omit" else "trace-abc") + + def test_get_logging_payload_preserves_anthropic_cache_read_input_tokens(): additional_usage_values = _get_additional_usage_values_for_usage( litellm.Usage( @@ -277,9 +384,7 @@ def test_sanitize_request_body_for_spend_logs_payload_long_string(): from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB # Create a string longer than MAX_STRING_LENGTH_PROMPT_IN_DB (2048) - long_string = ( - "a" * 3000 - ) # Create a string longer than MAX_STRING_LENGTH_PROMPT_IN_DB + long_string = "a" * 3000 # Create a string longer than MAX_STRING_LENGTH_PROMPT_IN_DB request_body = {"text": long_string, "normal_text": "short text"} sanitized = _sanitize_request_body_for_spend_logs_payload(request_body) @@ -329,9 +434,7 @@ def test_sanitize_request_body_for_spend_logs_payload_nested_list(): # Create a string longer than MAX_STRING_LENGTH_PROMPT_IN_DB long_string = "a" * (MAX_STRING_LENGTH_PROMPT_IN_DB + 500) - request_body = { - "items": [{"text": long_string}, {"text": "short"}, [{"text": long_string}]] - } + request_body = {"items": [{"text": long_string}, {"text": "short"}, [{"text": long_string}]]} sanitized = _sanitize_request_body_for_spend_logs_payload(request_body) # Calculate expected lengths based on actual MAX_STRING_LENGTH_PROMPT_IN_DB @@ -415,14 +518,10 @@ def test_sanitize_request_body_for_spend_logs_payload_circular_reference(): # Test that it handles circular reference without infinite recursion sanitized = _sanitize_request_body_for_spend_logs_payload(a) - assert sanitized == { - "b": {"a": {}} - } # Should return empty dict for circular reference + assert sanitized == {"b": {"a": {}}} # Should return empty dict for circular reference -@patch( - "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs" -) +@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") def test_get_vector_store_request_for_spend_logs_payload_store_prompts_true( mock_should_store, ): @@ -431,27 +530,16 @@ def test_get_vector_store_request_for_spend_logs_payload_store_prompts_true( # Sample vector store request metadata vector_store_request = [ - { - "vector_store_search_response": { - "data": [ - {"content": [{"text": "sensitive information", "type": "text"}]} - ] - } - } + {"vector_store_search_response": {"data": [{"content": [{"text": "sensitive information", "type": "text"}]}]}} ] # When store_prompts is True, the original data should be returned unchanged result = _get_vector_store_request_for_spend_logs_payload(vector_store_request) assert result == vector_store_request - assert ( - result[0]["vector_store_search_response"]["data"][0]["content"][0]["text"] - == "sensitive information" - ) + assert result[0]["vector_store_search_response"]["data"][0]["content"][0]["text"] == "sensitive information" -@patch( - "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs" -) +@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") def test_get_vector_store_request_for_spend_logs_payload_store_prompts_false( mock_should_store, ): @@ -460,32 +548,18 @@ def test_get_vector_store_request_for_spend_logs_payload_store_prompts_false( # Sample vector store request metadata vector_store_request = [ - { - "vector_store_search_response": { - "data": [ - {"content": [{"text": "sensitive information", "type": "text"}]} - ] - } - } + {"vector_store_search_response": {"data": [{"content": [{"text": "sensitive information", "type": "text"}]}]}} ] # When store_prompts is False, text should be redacted result = _get_vector_store_request_for_spend_logs_payload(vector_store_request) assert result is not None - assert ( - result[0]["vector_store_search_response"]["data"][0]["content"][0]["text"] - == REDACTED_BY_LITELM_STRING - ) + assert result[0]["vector_store_search_response"]["data"][0]["content"][0]["text"] == REDACTED_BY_LITELM_STRING # Ensure other fields are unchanged - assert ( - result[0]["vector_store_search_response"]["data"][0]["content"][0]["type"] - == "text" - ) + assert result[0]["vector_store_search_response"]["data"][0]["content"][0]["type"] == "text" -@patch( - "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs" -) +@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") def test_get_vector_store_request_for_spend_logs_payload_null_input(mock_should_store): # When input is None mock_should_store.return_value = False @@ -493,9 +567,7 @@ def test_get_vector_store_request_for_spend_logs_payload_null_input(mock_should_ assert result is None -@patch( - "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs" -) +@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") def test_get_messages_for_spend_logs_realtime_returns_messages(mock_should_store): """ Test that _get_messages_for_spend_logs_payload returns messages @@ -522,9 +594,7 @@ def test_get_messages_for_spend_logs_realtime_returns_messages(mock_should_store assert parsed[1]["content"] == "What is the weather today?" -@patch( - "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs" -) +@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") def test_get_messages_for_spend_logs_strips_null_bytes(mock_should_store): """Regression for PostgreSQL 22P05: NUL bytes must be stripped from messages.""" mock_should_store.return_value = True @@ -541,9 +611,7 @@ def test_get_messages_for_spend_logs_strips_null_bytes(mock_should_store): assert parsed[0]["content"] == "helloworld" -@patch( - "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs" -) +@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") def test_get_messages_for_spend_logs_realtime_empty_when_disabled(mock_should_store): """ Test that _get_messages_for_spend_logs_payload returns '{}' for realtime calls @@ -561,9 +629,7 @@ def test_get_messages_for_spend_logs_realtime_empty_when_disabled(mock_should_st assert result == "{}" -@patch( - "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs" -) +@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") def test_get_messages_for_spend_logs_non_realtime_returns_empty(mock_should_store): """ Test that _get_messages_for_spend_logs_payload returns '{}' for non-realtime @@ -581,9 +647,7 @@ def test_get_messages_for_spend_logs_non_realtime_returns_empty(mock_should_stor assert result == "{}" -@patch( - "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs" -) +@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") def test_get_response_for_spend_logs_payload_truncates_large_base64(mock_should_store): from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB @@ -611,9 +675,7 @@ def test_get_response_for_spend_logs_payload_truncates_large_base64(mock_should_ assert parsed["data"][0]["other_field"] == "value" -@patch( - "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs" -) +@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") def test_get_response_for_spend_logs_payload_strips_null_bytes(mock_should_store): """Regression for PostgreSQL 22P05: NUL bytes must be stripped from response.""" mock_should_store.return_value = True @@ -626,18 +688,14 @@ def test_get_response_for_spend_logs_payload_strips_null_bytes(mock_should_store assert json.loads(response_json)["content"] == "answerhere" -@patch( - "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs" -) +@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") def test_get_response_for_spend_logs_payload_truncates_large_embedding( mock_should_store, ): from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB mock_should_store.return_value = True - embedding_values = [ - round(i * 0.0001, 6) for i in range(MAX_STRING_LENGTH_PROMPT_IN_DB + 500) - ] + embedding_values = [round(i * 0.0001, 6) for i in range(MAX_STRING_LENGTH_PROMPT_IN_DB + 500)] large_embedding = json.dumps(embedding_values) payload = cast( StandardLoggingPayload, @@ -685,9 +743,7 @@ def test_truncation_includes_db_safeguard_note(): ) -@patch( - "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs" -) +@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") def test_response_truncation_logs_info_message(mock_should_store): """ Test that when response is truncated before DB storage, an info log is emitted @@ -702,18 +758,14 @@ def test_response_truncation_logs_info_message(mock_should_store): {"response": {"data": [{"content": large_text}]}}, ) - with patch( - "litellm.proxy.spend_tracking.spend_tracking_utils.verbose_proxy_logger" - ) as mock_logger: + with patch("litellm.proxy.spend_tracking.spend_tracking_utils.verbose_proxy_logger") as mock_logger: _get_response_for_spend_logs_payload(payload) mock_logger.info.assert_called_once() log_msg = mock_logger.info.call_args[0][0] assert "response was truncated" in log_msg -@patch( - "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs" -) +@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") def test_request_body_truncation_logs_info_message(mock_should_store): """ Test that when request body is truncated before DB storage, an info log is emitted. @@ -722,18 +774,10 @@ def test_request_body_truncation_logs_info_message(mock_should_store): mock_should_store.return_value = True large_prompt = "C" * (MAX_STRING_LENGTH_PROMPT_IN_DB + 500) - litellm_params = { - "proxy_server_request": { - "body": {"messages": [{"role": "user", "content": large_prompt}]} - } - } + litellm_params = {"proxy_server_request": {"body": {"messages": [{"role": "user", "content": large_prompt}]}}} - with patch( - "litellm.proxy.spend_tracking.spend_tracking_utils.verbose_proxy_logger" - ) as mock_logger: - _get_proxy_server_request_for_spend_logs_payload( - metadata={}, litellm_params=litellm_params, kwargs={} - ) + with patch("litellm.proxy.spend_tracking.spend_tracking_utils.verbose_proxy_logger") as mock_logger: + _get_proxy_server_request_for_spend_logs_payload(metadata={}, litellm_params=litellm_params, kwargs={}) mock_logger.info.assert_called_once() log_msg = mock_logger.info.call_args[0][0] assert "request body was truncated" in log_msg @@ -870,14 +914,10 @@ def test_get_logging_payload_api_key_preserved_when_standard_logging_payload_is_ ) # The api_key should be hashed (not the raw key) - assert ( - payload["api_key"] != test_api_key - ), "api_key should be hashed, not the raw key" + assert payload["api_key"] != test_api_key, "api_key should be hashed, not the raw key" # The api_key should be a valid hash (64 character hex string for SHA256) - assert ( - len(payload["api_key"]) == 64 - ), f"Expected 64 character hash, got {len(payload['api_key'])} characters" + assert len(payload["api_key"]) == 64, f"Expected 64 character hash, got {len(payload['api_key'])} characters" # Verify other fields are set correctly assert payload["model"] == "openai/gpt-4.1" @@ -1019,9 +1059,7 @@ async def test_api_key_preserved_through_failure_hook_to_database(): assert payload_api_key is not None, "🚨 CRITICAL: payload['api_key'] is None!" - assert ( - payload_api_key == hashed_key - ), f"🚨 CRITICAL: Expected api_key={hashed_key}, got {payload_api_key}" + assert payload_api_key == hashed_key, f"🚨 CRITICAL: Expected api_key={hashed_key}, got {payload_api_key}" # Verify token parameter matches assert data["token"] == hashed_key, f"Token parameter should be {hashed_key}" @@ -1066,9 +1104,7 @@ def test_get_logging_payload_includes_agent_id_from_kwargs(): end_time=end_time, ) - assert ( - payload["agent_id"] == test_agent_id - ), f"Expected agent_id '{test_agent_id}', got '{payload.get('agent_id')}'" + assert payload["agent_id"] == test_agent_id, f"Expected agent_id '{test_agent_id}', got '{payload.get('agent_id')}'" @patch("litellm.proxy.proxy_server.master_key", None) @@ -1093,9 +1129,7 @@ def test_get_logging_payload_includes_overhead_in_spend_logs_metadata(): startTime=1234567890.0, endTime=1234567891.0, completionStartTime=None, - model_map_information=StandardLoggingModelInformation( - model_map_key="gpt-3.5-turbo", model_map_value=None - ), + model_map_information=StandardLoggingModelInformation(model_map_key="gpt-3.5-turbo", model_map_value=None), model="gpt-3.5-turbo", model_id="model-123", model_group="openai", @@ -1173,9 +1207,9 @@ def test_get_logging_payload_includes_overhead_in_spend_logs_metadata(): metadata = json.loads(metadata_json) # Verify overhead is stored directly in metadata - assert ( - metadata.get("litellm_overhead_time_ms") == test_overhead_ms - ), f"Expected overhead '{test_overhead_ms}', got '{metadata.get('litellm_overhead_time_ms')}'" + assert metadata.get("litellm_overhead_time_ms") == test_overhead_ms, ( + f"Expected overhead '{test_overhead_ms}', got '{metadata.get('litellm_overhead_time_ms')}'" + ) @patch("litellm.proxy.proxy_server.master_key", None) @@ -1228,9 +1262,7 @@ def test_get_logging_payload_handles_missing_overhead_gracefully(): startTime=1234567890.0, endTime=1234567891.0, completionStartTime=None, - model_map_information=StandardLoggingModelInformation( - model_map_key="gpt-3.5-turbo", model_map_value=None - ), + model_map_information=StandardLoggingModelInformation(model_map_key="gpt-3.5-turbo", model_map_value=None), model="gpt-3.5-turbo", model_id="model-123", model_group="openai", @@ -1309,14 +1341,12 @@ def test_get_logging_payload_handles_missing_overhead_gracefully(): metadata = json.loads(metadata_json) # When overhead is None, litellm_overhead_time_ms should be None or not present - assert ( - metadata.get("litellm_overhead_time_ms") is None - ), "litellm_overhead_time_ms should be None when overhead is not provided" + assert metadata.get("litellm_overhead_time_ms") is None, ( + "litellm_overhead_time_ms should be None when overhead is not provided" + ) -@patch( - "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs" -) +@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") def test_spend_logs_redacts_request_and_response_when_turn_off_message_logging_enabled( mock_should_store, ): @@ -1347,9 +1377,7 @@ def test_spend_logs_redacts_request_and_response_when_turn_off_message_logging_e ) parsed_request = json.loads(request_result) - assert parsed_request["messages"] == [ - {"role": "user", "content": "redacted-by-litellm"} - ] + assert parsed_request["messages"] == [{"role": "user", "content": "redacted-by-litellm"}] assert parsed_request["model"] == "gpt-4" # Test response redaction - use dict response to verify redaction @@ -1368,9 +1396,7 @@ def test_spend_logs_redacts_request_and_response_when_turn_off_message_logging_e {"response": response_dict}, ) - response_result = _get_response_for_spend_logs_payload( - payload=payload, kwargs=kwargs - ) + response_result = _get_response_for_spend_logs_payload(payload=payload, kwargs=kwargs) # When redaction is enabled and response is a dict (not ModelResponse), # perform_redaction redacts content in-place within the choices structure @@ -1415,30 +1441,22 @@ def test_should_store_prompts_and_responses_in_spend_logs_case_insensitive_strin # When env var is True, should return True mock_get_secret_bool.return_value = True result = _should_store_prompts_and_responses_in_spend_logs() - assert ( - result is True - ), f"Expected True (from env var) for '{false_value}', got {result}" + assert result is True, f"Expected True (from env var) for '{false_value}', got {result}" # When env var is False, should return False mock_get_secret_bool.return_value = False result = _should_store_prompts_and_responses_in_spend_logs() - assert ( - result is False - ), f"Expected False (from env var) for '{false_value}', got {result}" + assert result is False, f"Expected False (from env var) for '{false_value}', got {result}" # Test when general_settings doesn't have the key at all with patch("litellm.proxy.proxy_server.general_settings", {}): mock_get_secret_bool.return_value = True result = _should_store_prompts_and_responses_in_spend_logs() - assert ( - result is True - ), "Expected True (from env var) when key missing, got False" + assert result is True, "Expected True (from env var) when key missing, got False" mock_get_secret_bool.return_value = False result = _should_store_prompts_and_responses_in_spend_logs() - assert ( - result is False - ), "Expected False (from env var) when key missing, got True" + assert result is False, "Expected False (from env var) when key missing, got True" def test_get_spend_logs_metadata_guardrail_info_fallback_from_metadata(): @@ -1831,9 +1849,7 @@ def test_get_logging_payload_includes_retry_info_in_spend_logs_metadata(): startTime=1234567890.0, endTime=1234567891.0, completionStartTime=None, - model_map_information=StandardLoggingModelInformation( - model_map_key="gpt-3.5-turbo", model_map_value=None - ), + model_map_information=StandardLoggingModelInformation(model_map_key="gpt-3.5-turbo", model_map_value=None), model="gpt-3.5-turbo", model_id="model-123", model_group="openai", @@ -1897,12 +1913,10 @@ def test_get_logging_payload_includes_retry_info_in_spend_logs_metadata(): metadata = json.loads(payload["metadata"]) - assert ( - metadata.get("attempted_retries") == 2 - ), f"Expected attempted_retries=2, got {metadata.get('attempted_retries')}" - assert ( - metadata.get("max_retries") == 3 - ), f"Expected max_retries=3, got {metadata.get('max_retries')}" + assert metadata.get("attempted_retries") == 2, ( + f"Expected attempted_retries=2, got {metadata.get('attempted_retries')}" + ) + assert metadata.get("max_retries") == 3, f"Expected max_retries=3, got {metadata.get('max_retries')}" @patch("litellm.proxy.proxy_server.master_key", None) @@ -1930,9 +1944,7 @@ def test_get_logging_payload_handles_missing_retry_info_gracefully(): startTime=1234567890.0, endTime=1234567891.0, completionStartTime=None, - model_map_information=StandardLoggingModelInformation( - model_map_key="gpt-3.5-turbo", model_map_value=None - ), + model_map_information=StandardLoggingModelInformation(model_map_key="gpt-3.5-turbo", model_map_value=None), model="gpt-3.5-turbo", model_id="model-123", model_group="openai", @@ -1996,20 +2008,14 @@ def test_get_logging_payload_handles_missing_retry_info_gracefully(): metadata = json.loads(payload["metadata"]) - assert ( - metadata.get("attempted_retries") is None - ), "attempted_retries should be None when not provided" - assert ( - metadata.get("max_retries") is None - ), "max_retries should be None when not provided" + assert metadata.get("attempted_retries") is None, "attempted_retries should be None when not provided" + assert metadata.get("max_retries") is None, "max_retries should be None when not provided" def test_get_request_duration_ms_normal(): """Test that request duration is correctly computed in milliseconds.""" start = datetime.datetime(2025, 1, 1, 0, 0, 0, tzinfo=timezone.utc) - end = datetime.datetime( - 2025, 1, 1, 0, 0, 2, 500000, tzinfo=timezone.utc - ) # 2.5s later + end = datetime.datetime(2025, 1, 1, 0, 0, 2, 500000, tzinfo=timezone.utc) # 2.5s later result = _get_request_duration_ms(start, end) assert result == 2500 @@ -2039,9 +2045,7 @@ def test_get_logging_payload_includes_request_duration_ms(): "litellm_params": {"api_base": "https://api.openai.com"}, "standard_logging_object": None, } - response_obj = { - "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15} - } + response_obj = {"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}} with ( patch("litellm.proxy.proxy_server.master_key", None), @@ -2107,16 +2111,12 @@ def test_sanitize_request_body_strips_secret_fields(): } sanitized = _sanitize_request_body_for_spend_logs_payload(request_body) - assert ( - "secret_fields" not in sanitized - ), "secret_fields must be stripped from the sanitized request body" + assert "secret_fields" not in sanitized, "secret_fields must be stripped from the sanitized request body" assert sanitized["model"] == "gpt-4" assert sanitized["messages"] == [{"role": "user", "content": "hi"}] -@patch( - "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs" -) +@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") def test_proxy_server_request_payload_excludes_secret_fields(mock_should_store): """ End-to-end test: when the proxy_server_request body contains @@ -2140,14 +2140,10 @@ def test_proxy_server_request_payload_excludes_secret_fields(mock_should_store): } } - result = _get_proxy_server_request_for_spend_logs_payload( - metadata={}, litellm_params=litellm_params, kwargs={} - ) + result = _get_proxy_server_request_for_spend_logs_payload(metadata={}, litellm_params=litellm_params, kwargs={}) parsed = json.loads(result) - assert ( - "secret_fields" not in parsed - ), "secret_fields must never appear in the spend-log proxy_server_request column" + assert "secret_fields" not in parsed, "secret_fields must never appear in the spend-log proxy_server_request column" assert parsed["model"] == "gpt-4" assert parsed["messages"] == [{"role": "user", "content": "hello"}] @@ -2176,10 +2172,7 @@ def test_redact_prompt_leaks_strips_input_value_python_repr(): def test_redact_prompt_leaks_strips_input_value_json(): - error_text = ( - '{"error":{"message":"validation failed",' - '"input":[{"role":"user","content":"top-secret-content"}]}}' - ) + error_text = '{"error":{"message":"validation failed","input":[{"role":"user","content":"top-secret-content"}]}}' redacted = _redact_prompt_leaks_in_error_string(error_text) assert "top-secret-content" not in redacted assert REDACTED_BY_LITELM_STRING in redacted @@ -2203,9 +2196,7 @@ def test_redact_prompt_leaks_empty_string(): assert _redact_prompt_leaks_in_error_string("") == "" -@patch( - "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs" -) +@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") def test_sanitize_error_information_redacts_when_not_storing_prompts( mock_should_store, ): @@ -2233,9 +2224,7 @@ def test_sanitize_error_information_redacts_when_not_storing_prompts( assert sanitized["llm_provider"] == "openai" -@patch( - "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs" -) +@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") def test_sanitize_error_information_skips_redaction_when_storing_prompts( mock_should_store, ): @@ -2246,9 +2235,7 @@ def test_sanitize_error_information_skips_redaction_when_storing_prompts( "error_class": "RateLimitError", "llm_provider": "openai", "traceback": "", - "error_message": ( - 'OpenAIException - {"error":{"input":[{"role":"user","content":"kept"}]}}' - ), + "error_message": ('OpenAIException - {"error":{"input":[{"role":"user","content":"kept"}]}}'), } sanitized = _sanitize_error_information_for_spend_logs(error_info) @@ -2259,9 +2246,7 @@ def test_sanitize_error_information_skips_redaction_when_storing_prompts( assert REDACTED_BY_LITELM_STRING not in sanitized["error_message"] -@patch( - "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs" -) +@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") def test_sanitize_error_information_caps_size_regardless_of_prompt_flag( mock_should_store, ): @@ -2292,9 +2277,7 @@ def test_sanitize_error_information_none_passthrough(): assert _sanitize_error_information_for_spend_logs(None) is None -@patch( - "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs" -) +@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") def test_sanitize_error_information_reproduces_lit_2992(mock_should_store): # Mirrors the reproduced row body from LIT-2992 — a RateLimitError whose # message embeds 178 pydantic validation errors, each carrying a full @@ -2335,10 +2318,7 @@ def test_redact_prompt_leaks_handles_nested_multimodal_content(): # Multi-modal payload: 'content' is itself a list. The depth-1 regex # would stop at the inner '['; the parser-based scanner must walk # through balanced nested brackets. - error_text = ( - '{"error":{"messages":[{"role":"user",' - '"content":[{"type":"text","text":"top-secret-multimodal"}]}]}}' - ) + error_text = '{"error":{"messages":[{"role":"user","content":[{"type":"text","text":"top-secret-multimodal"}]}]}}' redacted = _redact_prompt_leaks_in_error_string(error_text) assert "top-secret-multimodal" not in redacted assert REDACTED_BY_LITELM_STRING in redacted @@ -2347,9 +2327,7 @@ def test_redact_prompt_leaks_handles_nested_multimodal_content(): def test_redact_prompt_leaks_handles_bracket_in_prompt_text(): # Prompt text contains a literal '[' — the depth-1 regex would close # the outer ']' prematurely. The parser must respect string quoting. - error_text = ( - '{"error":{"input":[{"role":"user","content":"secret[123 still secret"}]}}' - ) + error_text = '{"error":{"input":[{"role":"user","content":"secret[123 still secret"}]}}' redacted = _redact_prompt_leaks_in_error_string(error_text) assert "secret[123" not in redacted assert "still secret" not in redacted @@ -2368,8 +2346,7 @@ def test_redact_prompt_leaks_handles_escaped_quote_in_prompt_text(): def test_redact_prompt_leaks_handles_nested_input_python_repr(): # Python dict-repr with nested list inside 'input' — single quotes. error_text = ( - "validation error: {'input': [{'role': 'user', " - "'content': [{'type': 'text', 'text': 'leaked-nested-text'}]}]}" + "validation error: {'input': [{'role': 'user', 'content': [{'type': 'text', 'text': 'leaked-nested-text'}]}]}" ) redacted = _redact_prompt_leaks_in_error_string(error_text) assert "leaked-nested-text" not in redacted @@ -2385,9 +2362,7 @@ def test_redact_prompt_leaks_handles_unterminated_value(): assert REDACTED_BY_LITELM_STRING in redacted -@patch( - "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs" -) +@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") def test_sanitize_error_information_redacts_traceback_when_not_storing_prompts( mock_should_store, ): @@ -2419,9 +2394,7 @@ def test_sanitize_error_information_redacts_traceback_when_not_storing_prompts( assert "ValueError: invalid request" in sanitized["traceback"] -@patch( - "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs" -) +@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") def test_sanitize_error_information_skips_traceback_redaction_when_storing_prompts( mock_should_store, ): @@ -2431,9 +2404,7 @@ def test_sanitize_error_information_skips_traceback_redaction_when_storing_promp "error_code": "500", "error_class": "ValueError", "llm_provider": "", - "traceback": ( - 'raise ValueError({"input":[{"role":"user","content":"tb-kept"}]})' - ), + "traceback": ('raise ValueError({"input":[{"role":"user","content":"tb-kept"}]})'), "error_message": "invalid request", } @@ -2448,20 +2419,14 @@ def test_redact_prompt_leaks_strips_prompt_key_completions_payload(): # /v1/completions echoes the user input under the top-level 'prompt' key # rather than 'messages'. Without 'prompt' coverage the body would survive # the redactor when store_prompts_in_spend_logs is False. - error_text = ( - '{"error":{"message":"validation failed",' - '"prompt":"super-secret-completion-text"}}' - ) + error_text = '{"error":{"message":"validation failed","prompt":"super-secret-completion-text"}}' redacted = _redact_prompt_leaks_in_error_string(error_text) assert "super-secret-completion-text" not in redacted assert REDACTED_BY_LITELM_STRING in redacted def test_redact_prompt_leaks_strips_prompt_key_python_repr(): - error_text = ( - "{'model': 'gpt-3.5-turbo-instruct', " - "'prompt': 'leaked-completion-prompt-body'}" - ) + error_text = "{'model': 'gpt-3.5-turbo-instruct', 'prompt': 'leaked-completion-prompt-body'}" redacted = _redact_prompt_leaks_in_error_string(error_text) assert "leaked-completion-prompt-body" not in redacted assert REDACTED_BY_LITELM_STRING in redacted @@ -2495,11 +2460,7 @@ def test_redact_prompt_leaks_strips_pydantic_input_value_list(): def test_redact_prompt_leaks_strips_pydantic_input_value_dict(): - error_text = ( - "[type=dict_type, " - "input_value={'role': 'user', 'content': 'leaked-dict-content'}, " - "input_type=dict]" - ) + error_text = "[type=dict_type, input_value={'role': 'user', 'content': 'leaked-dict-content'}, input_type=dict]" redacted = _redact_prompt_leaks_in_error_string(error_text) assert "leaked-dict-content" not in redacted assert REDACTED_BY_LITELM_STRING in redacted @@ -2541,9 +2502,7 @@ def test_redact_prompt_leaks_combined_quoted_key_and_pydantic_assignment(): assert redacted.count(REDACTED_BY_LITELM_STRING) >= 2 -@patch( - "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs" -) +@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") def test_sanitize_error_information_redacts_pydantic_assignment_form( mock_should_store, ): @@ -2741,9 +2700,7 @@ def test_get_spend_logs_metadata_non_sk_raw_key_hashed(): def test_get_spend_logs_metadata_already_hashed_unchanged_with_provenance(): already_hashed = hash_token("sk-some-key") - meta = _get_spend_logs_metadata( - {"user_api_key": already_hashed, "user_api_key_hash": already_hashed} - ) + meta = _get_spend_logs_metadata({"user_api_key": already_hashed, "user_api_key_hash": already_hashed}) assert meta["user_api_key"] == already_hashed assert hash_token(already_hashed) != meta["user_api_key"] # no double-hash @@ -2758,9 +2715,7 @@ def test_get_spend_logs_metadata_already_hashed_no_provenance_is_rehashed(): def test_get_spend_logs_metadata_provenance_bypass_requires_hash_match(): already_hashed = hash_token("sk-some-key") different_hash = hash_token("sk-other-key") - meta = _get_spend_logs_metadata( - {"user_api_key": already_hashed, "user_api_key_hash": different_hash} - ) + meta = _get_spend_logs_metadata({"user_api_key": already_hashed, "user_api_key_hash": different_hash}) assert meta["user_api_key"] == hash_token(already_hashed) @@ -2797,16 +2752,12 @@ def test_get_logging_payload_uses_recovered_combined_usage_on_failure(): "model": "anthropic/claude-haiku-4-5", "call_type": "acompletion", "litellm_params": {"metadata": {"user_api_key": "sk-test"}}, - "combined_usage_object": Usage( - prompt_tokens=30, completion_tokens=1, total_tokens=31 - ), + "combined_usage_object": Usage(prompt_tokens=30, completion_tokens=1, total_tokens=31), } response_obj = Exception("MidStreamFallbackError: read timeout") now = datetime.datetime.now(timezone.utc) - payload = get_logging_payload( - kwargs=kwargs, response_obj=response_obj, start_time=now, end_time=now - ) + payload = get_logging_payload(kwargs=kwargs, response_obj=response_obj, start_time=now, end_time=now) assert payload["prompt_tokens"] == 30 assert payload["completion_tokens"] == 1 @@ -2825,9 +2776,7 @@ def test_get_logging_payload_failure_without_recovered_usage_is_zero(): response_obj = Exception("BadRequestError") now = datetime.datetime.now(timezone.utc) - payload = get_logging_payload( - kwargs=kwargs, response_obj=response_obj, start_time=now, end_time=now - ) + payload = get_logging_payload(kwargs=kwargs, response_obj=response_obj, start_time=now, end_time=now) assert payload["total_tokens"] == 0 @@ -2853,9 +2802,7 @@ def test_get_logging_payload_sets_litellm_call_id_for_correlation(): } now = datetime.datetime.now(timezone.utc) - payload = get_logging_payload( - kwargs=kwargs, response_obj=response_obj, start_time=now, end_time=now - ) + payload = get_logging_payload(kwargs=kwargs, response_obj=response_obj, start_time=now, end_time=now) metadata = json.loads(payload["metadata"]) assert payload["request_id"] == provider_response_id @@ -2882,9 +2829,7 @@ def test_get_logging_payload_litellm_call_id_falls_back_to_litellm_params(): } now = datetime.datetime.now(timezone.utc) - payload = get_logging_payload( - kwargs=kwargs, response_obj=response_obj, start_time=now, end_time=now - ) + payload = get_logging_payload(kwargs=kwargs, response_obj=response_obj, start_time=now, end_time=now) assert json.loads(payload["metadata"])["litellm_call_id"] == trace_call_id @@ -2901,14 +2846,10 @@ def test_get_logging_payload_litellm_call_id_when_response_has_no_id(): "litellm_call_id": trace_call_id, "litellm_params": {"metadata": {"user_api_key": "sk-test"}}, } - response_obj = { - "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2} - } + response_obj = {"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}} now = datetime.datetime.now(timezone.utc) - payload = get_logging_payload( - kwargs=kwargs, response_obj=response_obj, start_time=now, end_time=now - ) + payload = get_logging_payload(kwargs=kwargs, response_obj=response_obj, start_time=now, end_time=now) assert json.loads(payload["metadata"])["litellm_call_id"] == trace_call_id assert payload["request_id"] == trace_call_id @@ -2932,9 +2873,7 @@ def test_get_logging_payload_cache_hit_keeps_raw_litellm_call_id(): } now = datetime.datetime.now(timezone.utc) - payload = get_logging_payload( - kwargs=kwargs, response_obj=response_obj, start_time=now, end_time=now - ) + payload = get_logging_payload(kwargs=kwargs, response_obj=response_obj, start_time=now, end_time=now) assert json.loads(payload["metadata"])["litellm_call_id"] == trace_call_id assert "_cache_hit" in payload["request_id"] @@ -3074,9 +3013,7 @@ def test_get_logging_payload_hashes_bearer_prefixed_api_key(): assert not payload["api_key"].startswith("Bearer"), ( f"api_key column contains plaintext Bearer key: {payload['api_key']}" ) - assert not payload["api_key"].startswith("sk-"), ( - f"api_key column contains unhashed key: {payload['api_key']}" - ) + assert not payload["api_key"].startswith("sk-"), f"api_key column contains unhashed key: {payload['api_key']}" metadata_dict = json.loads(payload["metadata"]) assert not metadata_dict["user_api_key"].startswith("Bearer"), ( @@ -3747,9 +3684,7 @@ def test_get_logging_payload_includes_fallback_info_in_spend_logs_metadata(): startTime=1234567890.0, endTime=1234567891.0, completionStartTime=None, - model_map_information=StandardLoggingModelInformation( - model_map_key="gpt-3.5-turbo", model_map_value=None - ), + model_map_information=StandardLoggingModelInformation(model_map_key="gpt-3.5-turbo", model_map_value=None), model="gpt-3.5-turbo", model_id="model-123", model_group="openai", @@ -3813,12 +3748,12 @@ def test_get_logging_payload_includes_fallback_info_in_spend_logs_metadata(): metadata = json.loads(payload["metadata"]) - assert ( - metadata.get("attempted_fallbacks") == 2 - ), f"Expected attempted_fallbacks=2, got {metadata.get('attempted_fallbacks')}" - assert ( - metadata.get("original_model_group") == "azure-gpt-fallback" - ), f"Expected original_model_group=azure-gpt-fallback, got {metadata.get('original_model_group')}" + assert metadata.get("attempted_fallbacks") == 2, ( + f"Expected attempted_fallbacks=2, got {metadata.get('attempted_fallbacks')}" + ) + assert metadata.get("original_model_group") == "azure-gpt-fallback", ( + f"Expected original_model_group=azure-gpt-fallback, got {metadata.get('original_model_group')}" + ) def test_get_logging_payload_handles_missing_fallback_info_gracefully(): @@ -3844,9 +3779,7 @@ def test_get_logging_payload_handles_missing_fallback_info_gracefully(): startTime=1234567890.0, endTime=1234567891.0, completionStartTime=None, - model_map_information=StandardLoggingModelInformation( - model_map_key="gpt-3.5-turbo", model_map_value=None - ), + model_map_information=StandardLoggingModelInformation(model_map_key="gpt-3.5-turbo", model_map_value=None), model="gpt-3.5-turbo", model_id="model-123", model_group="openai", @@ -3910,12 +3843,10 @@ def test_get_logging_payload_handles_missing_fallback_info_gracefully(): metadata = json.loads(payload["metadata"]) - assert ( - metadata.get("attempted_fallbacks") is None - ), "attempted_fallbacks should be None when not provided" - assert ( - metadata.get("original_model_group") is None - ), "original_model_group should be None when not provided" + assert metadata.get("attempted_fallbacks") is None, "attempted_fallbacks should be None when not provided" + assert metadata.get("original_model_group") is None, "original_model_group should be None when not provided" + + @pytest.mark.parametrize("bucket", ["metadata", "litellm_metadata"]) def test_injected_cache_breakpoints_survive_into_spend_log_metadata(bucket): """The injection marker only gates savings if it reaches the spend-log row. diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index 8366e5546a9..72d37650963 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -41,21 +41,18 @@ from litellm.litellm_core_utils.get_provider_specific_headers import ( from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( TRUSTED_CALLBACK_VARS_FIELD, ) -from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY +from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY, SESSION_ID_OMITTED_METADATA_KEY from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.fireworks_ai.common_utils import get_fireworks_session_id from litellm.types.utils import CredentialItem - def test_check_if_token_is_service_account(): """ Test that only keys with `service_account_id` in metadata are considered service accounts """ # Test case 1: Service account token - service_account_token = UserAPIKeyAuth( - api_key="test-key", metadata={"service_account_id": "test-service-account"} - ) + service_account_token = UserAPIKeyAuth(api_key="test-key", metadata={"service_account_id": "test-service-account"}) assert check_if_token_is_service_account(service_account_token) == True # Test case 2: Regular user token @@ -63,9 +60,7 @@ def test_check_if_token_is_service_account(): assert check_if_token_is_service_account(regular_token) == False # Test case 3: Token with other metadata - other_metadata_token = UserAPIKeyAuth( - api_key="test-key", metadata={"user_id": "test-user"} - ) + other_metadata_token = UserAPIKeyAuth(api_key="test-key", metadata={"user_id": "test-user"}) assert check_if_token_is_service_account(other_metadata_token) == False @@ -112,15 +107,11 @@ class TestGetMetadataVariableName: def test_returns_litellm_metadata_for_bedrock_invoke(self): # GH#30629: bedrock passthrough must use litellm_metadata # to prevent key-level tags from leaking into provider body - request = self._make_request( - "/bedrock/model/us.anthropic.claude-sonnet-4-6/invoke" - ) + request = self._make_request("/bedrock/model/us.anthropic.claude-sonnet-4-6/invoke") assert _get_metadata_variable_name(request) == "litellm_metadata" def test_returns_litellm_metadata_for_bedrock_converse(self): - request = self._make_request( - "/bedrock/model/us.anthropic.claude-sonnet-4-6/converse" - ) + request = self._make_request("/bedrock/model/us.anthropic.claude-sonnet-4-6/converse") assert _get_metadata_variable_name(request) == "litellm_metadata" @@ -128,9 +119,7 @@ def test_get_enforced_params_for_service_account_settings(): """ Test that service account enforced params are only added to service account keys """ - service_account_token = UserAPIKeyAuth( - api_key="test-key", metadata={"service_account_id": "test-service-account"} - ) + service_account_token = UserAPIKeyAuth(api_key="test-key", metadata={"service_account_id": "test-service-account"}) general_settings_with_service_account_settings = { "service_account_settings": {"enforced_params": ["metadata.service"]}, } @@ -140,9 +129,7 @@ def test_get_enforced_params_for_service_account_settings(): ) assert result == ["metadata.service"] - regular_token = UserAPIKeyAuth( - api_key="test-key", metadata={"enforced_params": ["user"]} - ) + regular_token = UserAPIKeyAuth(api_key="test-key", metadata={"enforced_params": ["user"]}) result = _get_enforced_params( general_settings=general_settings_with_service_account_settings, user_api_key_dict=regular_token, @@ -155,9 +142,7 @@ def test_get_enforced_params_for_service_account_settings(): [ ( {"enforced_params": ["param1", "param2"]}, - UserAPIKeyAuth( - api_key="test_api_key", user_id="test_user_id", org_id="test_org_id" - ), + UserAPIKeyAuth(api_key="test_api_key", user_id="test_user_id", org_id="test_org_id"), ["param1", "param2"], ), ( @@ -183,9 +168,7 @@ def test_get_enforced_params_for_service_account_settings(): ), ], ) -def test_get_enforced_params( - general_settings, user_api_key_dict, expected_enforced_params -): +def test_get_enforced_params(general_settings, user_api_key_dict, expected_enforced_params): from litellm.proxy.litellm_pre_call_utils import _get_enforced_params enforced_params = _get_enforced_params(general_settings, user_api_key_dict) @@ -441,9 +424,7 @@ async def test_add_litellm_data_to_request_strips_admin_injection_slots(): populated = updated["metadata"] assert populated["user_api_key_metadata"] == real_admin_metadata assert populated["user_api_key_team_metadata"] == real_admin_metadata - assert "_pipeline_managed_guardrails" not in populated or populated[ - "_pipeline_managed_guardrails" - ] != ["evaded"] + assert "_pipeline_managed_guardrails" not in populated or populated["_pipeline_managed_guardrails"] != ["evaded"] other = updated.get("litellm_metadata") or {} assert other.get("user_api_key_metadata") in (None, {}, real_admin_metadata) @@ -697,9 +678,7 @@ async def test_add_litellm_data_to_request_proxy_server_request_body_is_post_str snapshot_body = updated["proxy_server_request"]["body"] assert snapshot_body is not None snapshot_metadata = snapshot_body.get("metadata") or {} - assert "user_api_key_user_id" not in snapshot_metadata or ( - snapshot_metadata["user_api_key_user_id"] != "victim" - ) + assert "user_api_key_user_id" not in snapshot_metadata or (snapshot_metadata["user_api_key_user_id"] != "victim") @pytest.mark.asyncio @@ -754,9 +733,7 @@ async def test_add_litellm_data_to_request_body_snapshot_excludes_secret_fields( ) # secret_fields must exist on the live data dict - assert ( - "secret_fields" in updated - ), "secret_fields must still be present on the live data dict" + assert "secret_fields" in updated, "secret_fields must still be present on the live data dict" assert "raw_headers" in updated["secret_fields"] # But the body snapshot must NOT contain secret_fields @@ -815,8 +792,7 @@ async def test_add_litellm_data_to_request_body_snapshot_excludes_proxy_server_r snapshot_body = updated["proxy_server_request"]["body"] assert "proxy_server_request" not in snapshot_body, ( - "proxy_server_request must be excluded from its own body snapshot " - "to prevent the body from self-referencing" + "proxy_server_request must be excluded from its own body snapshot to prevent the body from self-referencing" ) @@ -1344,23 +1320,18 @@ async def test_add_litellm_data_to_request_strips_client_redaction_bypass_contro assert "turn_off_message_logging" not in (updated.get("litellm_params") or {}).get("metadata", {}) assert "turn_off_message_logging" not in updated["metadata"] assert "turn_off_message_logging" not in (updated.get("litellm_metadata") or {}) + assert "litellm-disable-message-redaction" not in {header.lower() for header in updated["metadata"]["headers"]} assert "litellm-disable-message-redaction" not in { - header.lower() for header in updated["metadata"]["headers"] - } - assert "litellm-disable-message-redaction" not in { - header.lower() - for header in updated["metadata"]["requester_metadata"].get("headers", {}) + header.lower() for header in updated["metadata"]["requester_metadata"].get("headers", {}) } assert "litellm-disable-message-redaction" not in { header.lower() for header in updated["proxy_server_request"]["headers"] } assert "litellm-disable-message-redaction" not in { - header.lower() - for header in updated["proxy_server_request"]["body"]["metadata"]["headers"] + header.lower() for header in updated["proxy_server_request"]["body"]["metadata"]["headers"] } assert "litellm-disable-message-redaction" not in { - header.lower() - for header in (updated.get("litellm_metadata") or {}).get("headers", {}) + header.lower() for header in (updated.get("litellm_metadata") or {}).get("headers", {}) } @@ -1430,12 +1401,7 @@ async def test_add_litellm_data_to_request_admin_callback_vars_turn_off_message_ dynamic_params = initialize_standard_callback_dynamic_params(updated) assert dynamic_params.get("turn_off_message_logging") == "False" - assert ( - should_redact_message_logging( - {"standard_callback_dynamic_params": dynamic_params} - ) - is False - ) + assert should_redact_message_logging({"standard_callback_dynamic_params": dynamic_params}) is False finally: litellm.turn_off_message_logging = original_turn_off_message_logging @@ -1506,12 +1472,7 @@ async def test_add_litellm_data_to_request_admin_callback_vars_turn_off_message_ dynamic_params = initialize_standard_callback_dynamic_params(updated) assert dynamic_params.get("turn_off_message_logging") == "True" - assert ( - should_redact_message_logging( - {"standard_callback_dynamic_params": dynamic_params} - ) - is True - ) + assert should_redact_message_logging({"standard_callback_dynamic_params": dynamic_params}) is True finally: litellm.turn_off_message_logging = original_turn_off_message_logging @@ -1552,9 +1513,7 @@ async def test_add_litellm_data_to_request_allows_redaction_opt_out_with_admin_o "headers": {"litellm-disable-message-redaction": "true"}, "turn_off_message_logging": False, }, - "litellm_metadata": json.dumps( - {"headers": {"LiteLLM-Disable-Message-Redaction": "true"}} - ), + "litellm_metadata": json.dumps({"headers": {"LiteLLM-Disable-Message-Redaction": "true"}}), }, request=request_mock, user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key", **auth_kwargs), @@ -1567,19 +1526,15 @@ async def test_add_litellm_data_to_request_allows_redaction_opt_out_with_admin_o assert updated["turn_off_message_logging"] is False assert updated["metadata"]["turn_off_message_logging"] is False + assert "litellm-disable-message-redaction" in {header.lower() for header in updated["metadata"]["headers"]} assert "litellm-disable-message-redaction" in { - header.lower() for header in updated["metadata"]["headers"] - } - assert "litellm-disable-message-redaction" in { - header.lower() - for header in updated["metadata"]["requester_metadata"].get("headers", {}) + header.lower() for header in updated["metadata"]["requester_metadata"].get("headers", {}) } assert "litellm-disable-message-redaction" in { header.lower() for header in updated["proxy_server_request"]["headers"] } assert "litellm-disable-message-redaction" in { - header.lower() - for header in updated["proxy_server_request"]["body"]["metadata"]["headers"] + header.lower() for header in updated["proxy_server_request"]["body"]["metadata"]["headers"] } assert "litellm_metadata" not in updated @@ -1870,9 +1825,7 @@ async def test_add_litellm_data_to_request_audio_transcription_multipart(): request_mock.client.host = "127.0.0.1" # Simulate multipart data (metadata as string) - metadata_dict = { - "tags": ["jobID:214590dsff09fds", "taskName:run_page_classification"] - } + metadata_dict = {"tags": ["jobID:214590dsff09fds", "taskName:run_page_classification"]} stringified_metadata = json.dumps(metadata_dict) data = { @@ -2200,23 +2153,15 @@ def test_key_dynamic_logging_settings(): # Test with langfuse logging key_with_langfuse = UserAPIKeyAuth( api_key="test-key", - metadata={ - "logging": [{"callback_name": "langfuse", "callback_type": "success"}] - }, + metadata={"logging": [{"callback_name": "langfuse", "callback_type": "success"}]}, team_metadata={}, ) - result = KeyAndTeamLoggingSettings.get_key_dynamic_logging_settings( - key_with_langfuse - ) + result = KeyAndTeamLoggingSettings.get_key_dynamic_logging_settings(key_with_langfuse) assert result == [{"callback_name": "langfuse", "callback_type": "success"}] # Test with no logging metadata - key_without_logging = UserAPIKeyAuth( - api_key="test-key", metadata={}, team_metadata={} - ) - result = KeyAndTeamLoggingSettings.get_key_dynamic_logging_settings( - key_without_logging - ) + key_without_logging = UserAPIKeyAuth(api_key="test-key", metadata={}, team_metadata={}) + result = KeyAndTeamLoggingSettings.get_key_dynamic_logging_settings(key_without_logging) assert result is None @@ -2228,35 +2173,23 @@ def test_team_dynamic_logging_settings(): key_with_team_arize = UserAPIKeyAuth( api_key="test-key", metadata={}, - team_metadata={ - "logging": [{"callback_name": "arize", "callback_type": "failure"}] - }, - ) - result = KeyAndTeamLoggingSettings.get_team_dynamic_logging_settings( - key_with_team_arize + team_metadata={"logging": [{"callback_name": "arize", "callback_type": "failure"}]}, ) + result = KeyAndTeamLoggingSettings.get_team_dynamic_logging_settings(key_with_team_arize) assert result == [{"callback_name": "arize", "callback_type": "failure"}] # Test with langfuse team logging key_with_team_langfuse = UserAPIKeyAuth( api_key="test-key", metadata={}, - team_metadata={ - "logging": [{"callback_name": "langfuse", "callback_type": "success"}] - }, - ) - result = KeyAndTeamLoggingSettings.get_team_dynamic_logging_settings( - key_with_team_langfuse + team_metadata={"logging": [{"callback_name": "langfuse", "callback_type": "success"}]}, ) + result = KeyAndTeamLoggingSettings.get_team_dynamic_logging_settings(key_with_team_langfuse) assert result == [{"callback_name": "langfuse", "callback_type": "success"}] # Test with no team logging metadata - key_without_team_logging = UserAPIKeyAuth( - api_key="test-key", metadata={}, team_metadata={} - ) - result = KeyAndTeamLoggingSettings.get_team_dynamic_logging_settings( - key_without_team_logging - ) + key_without_team_logging = UserAPIKeyAuth(api_key="test-key", metadata={}, team_metadata={}) + result = KeyAndTeamLoggingSettings.get_team_dynamic_logging_settings(key_without_team_logging) assert result is None @@ -2337,9 +2270,7 @@ def test_get_dynamic_logging_metadata_with_arize_team_logging(): mock_proxy_config = MagicMock() # Call the function - result = _get_dynamic_logging_metadata( - user_api_key_dict=user_api_key_dict, proxy_config=mock_proxy_config - ) + result = _get_dynamic_logging_metadata(user_api_key_dict=user_api_key_dict, proxy_config=mock_proxy_config) # Verify the result assert result is not None @@ -2355,9 +2286,7 @@ def test_add_team_callback_rejects_env_reference(): AddTeamCallback( callback_name="langfuse", callback_type="success", - callback_vars={ - "langfuse_secret_key": "os.environ/LANGFUSE_SECRET_KEY_TEMP" - }, + callback_vars={"langfuse_secret_key": "os.environ/LANGFUSE_SECRET_KEY_TEMP"}, ) assert "os.environ/" in str(exc_info.value) @@ -2388,9 +2317,7 @@ def test_get_dynamic_logging_metadata_ignores_env_reference_from_key_metadata( team_metadata={}, ) - result = _get_dynamic_logging_metadata( - user_api_key_dict=user_api_key_dict, proxy_config=MagicMock() - ) + result = _get_dynamic_logging_metadata(user_api_key_dict=user_api_key_dict, proxy_config=MagicMock()) assert result is None @@ -2401,16 +2328,12 @@ def test_get_num_retries_from_request(): """ # Test case 1: Header is present with valid integer string headers_with_retries = {"x-litellm-num-retries": "3"} - result = LiteLLMProxyRequestSetup._get_num_retries_from_request( - headers_with_retries - ) + result = LiteLLMProxyRequestSetup._get_num_retries_from_request(headers_with_retries) assert result == 3 # Test case 2: Header is not present headers_without_retries = {"Content-Type": "application/json"} - result = LiteLLMProxyRequestSetup._get_num_retries_from_request( - headers_without_retries - ) + result = LiteLLMProxyRequestSetup._get_num_retries_from_request(headers_without_retries) assert result is None # Test case 3: Empty headers dictionary @@ -2425,9 +2348,7 @@ def test_get_num_retries_from_request(): # Test case 5: Header present with large number headers_with_large_number = {"x-litellm-num-retries": "100"} - result = LiteLLMProxyRequestSetup._get_num_retries_from_request( - headers_with_large_number - ) + result = LiteLLMProxyRequestSetup._get_num_retries_from_request(headers_with_large_number) assert result == 100 # Test case 6: Multiple headers with num retries header @@ -2441,19 +2362,17 @@ def test_get_num_retries_from_request(): # Test case 7: Header present with invalid value (should raise ValueError when int() is called) headers_with_invalid = {"x-litellm-num-retries": "invalid"} - with pytest.raises(ValueError, match='invalid literal for int\\(\\) with base'): + with pytest.raises(ValueError, match="invalid literal for int\\(\\) with base"): LiteLLMProxyRequestSetup._get_num_retries_from_request(headers_with_invalid) # Test case 8: Header present with float string (should raise ValueError when int() is called) headers_with_float = {"x-litellm-num-retries": "3.5"} - with pytest.raises(ValueError, match='invalid literal for int\\(\\) with base'): + with pytest.raises(ValueError, match="invalid literal for int\\(\\) with base"): LiteLLMProxyRequestSetup._get_num_retries_from_request(headers_with_float) # Test case 9: Header present with negative number headers_with_negative = {"x-litellm-num-retries": "-1"} - result = LiteLLMProxyRequestSetup._get_num_retries_from_request( - headers_with_negative - ) + result = LiteLLMProxyRequestSetup._get_num_retries_from_request(headers_with_negative) assert result == -1 @@ -2463,15 +2382,11 @@ def test_get_keepalive_seconds_from_request(): """ # Header present with valid float string headers_with_keepalive = {"x-litellm-keepalive-seconds": "15"} - result = LiteLLMProxyRequestSetup._get_keepalive_seconds_from_request( - headers_with_keepalive - ) + result = LiteLLMProxyRequestSetup._get_keepalive_seconds_from_request(headers_with_keepalive) assert result == 15.0 # Header not present - result = LiteLLMProxyRequestSetup._get_keepalive_seconds_from_request( - {"Content-Type": "application/json"} - ) + result = LiteLLMProxyRequestSetup._get_keepalive_seconds_from_request({"Content-Type": "application/json"}) assert result is None # Empty headers dictionary @@ -2479,17 +2394,13 @@ def test_get_keepalive_seconds_from_request(): assert result is None # Header present with a fractional value - result = LiteLLMProxyRequestSetup._get_keepalive_seconds_from_request( - {"x-litellm-keepalive-seconds": "1.5"} - ) + result = LiteLLMProxyRequestSetup._get_keepalive_seconds_from_request({"x-litellm-keepalive-seconds": "1.5"}) assert result == 1.5 # Header present with invalid value raises ValueError, matching the other # x-litellm-* numeric header helpers (_get_timeout_from_request, etc.) with pytest.raises(ValueError, match="could not convert string to float: 'not-a-number"): - LiteLLMProxyRequestSetup._get_keepalive_seconds_from_request( - {"x-litellm-keepalive-seconds": "not-a-number"} - ) + LiteLLMProxyRequestSetup._get_keepalive_seconds_from_request({"x-litellm-keepalive-seconds": "not-a-number"}) def test_add_litellm_data_for_backend_llm_call_merges_keepalive_seconds_header(): @@ -2728,9 +2639,7 @@ def test_management_endpoint_metadata_drops_callback_credentials(): ), ], ) -def test_add_headers_to_llm_call_by_model_group( - data, model_group_settings, expected_headers_added -): +def test_add_headers_to_llm_call_by_model_group(data, model_group_settings, expected_headers_added): """ Test LiteLLMProxyRequestSetup.add_headers_to_llm_call_by_model_group method @@ -2751,9 +2660,7 @@ def test_add_headers_to_llm_call_by_model_group( "X-Custom-Header": "custom-value", } - user_api_key_dict = UserAPIKeyAuth( - api_key="test-key", user_id="test-user", org_id="test-org" - ) + user_api_key_dict = UserAPIKeyAuth(api_key="test-key", user_id="test-user", org_id="test-org") # Mock the model_group_settings original_model_group_settings = getattr(litellm, "model_group_settings", None) @@ -2771,7 +2678,6 @@ def test_add_headers_to_llm_call_by_model_group( "add_headers_to_llm_call", return_value=expected_returned_headers if expected_headers_added else {}, ) as mock_add_headers: - # Make a copy of original data to verify it's not mutated unexpectedly original_data = copy.deepcopy(data) @@ -2828,7 +2734,6 @@ def test_add_headers_to_llm_call_by_model_group_empty_headers_returned(): "add_headers_to_llm_call", return_value={}, # Return empty dict ) as mock_add_headers: - result = LiteLLMProxyRequestSetup.add_headers_to_llm_call_by_model_group( data=data, headers=headers, user_api_key_dict=user_api_key_dict ) @@ -2876,7 +2781,6 @@ def test_add_headers_to_llm_call_by_model_group_existing_headers_in_data(): "add_headers_to_llm_call", return_value=new_headers, ) as mock_add_headers: - result = LiteLLMProxyRequestSetup.add_headers_to_llm_call_by_model_group( data=data, headers=headers, user_api_key_dict=user_api_key_dict ) @@ -2990,13 +2894,9 @@ async def test_add_litellm_metadata_from_request_headers(): general_settings = {} # Create mock select_data_generator with correct signature - def mock_select_data_generator( - response=None, user_api_key_dict=None, request_data=None - ): + def mock_select_data_generator(response=None, user_api_key_dict=None, request_data=None): async def mock_generator(): - yield "data: " + json.dumps( - {"choices": [{"delta": {"content": "Hello"}}]} - ) + "\n\n" + yield "data: " + json.dumps({"choices": [{"delta": {"content": "Hello"}}]}) + "\n\n" yield "data: [DONE]\n\n" return mock_generator() @@ -3023,21 +2923,19 @@ async def test_add_litellm_metadata_from_request_headers(): await asyncio.sleep(3) # Check if standard_logging_object was set - assert ( - test_logger.standard_logging_object is not None - ), "standard_logging_object should be populated after LLM request" + assert test_logger.standard_logging_object is not None, ( + "standard_logging_object should be populated after LLM request" + ) # Verify the logging object contains expected metadata standard_logging_obj = test_logger.standard_logging_object - print( - f"Standard logging object captured: {json.dumps(standard_logging_obj, indent=4, default=str)}" - ) + print(f"Standard logging object captured: {json.dumps(standard_logging_obj, indent=4, default=str)}") SPEND_LOGS_METADATA = standard_logging_obj["metadata"]["spend_logs_metadata"] - assert SPEND_LOGS_METADATA == dict( - json.loads(headers["x-litellm-spend-logs-metadata"]) - ), "spend_logs_metadata should be the same as the headers" + assert SPEND_LOGS_METADATA == dict(json.loads(headers["x-litellm-spend-logs-metadata"])), ( + "spend_logs_metadata should be the same as the headers" + ) finally: litellm.callbacks = original_callbacks @@ -3188,11 +3086,7 @@ def test_add_litellm_metadata_from_request_headers_generic_session_id_header(): def test_add_litellm_metadata_from_anthropic_user_id_sets_session_id(): - data = { - "metadata": { - "user_id": "user_abc123_account__session_e96634a3-fa28-4083-b354-55542e2dca01" - } - } + data = {"metadata": {"user_id": "user_abc123_account__session_e96634a3-fa28-4083-b354-55542e2dca01"}} LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers( headers={}, data=data, _metadata_variable_name="metadata" ) @@ -3308,9 +3202,7 @@ def test_get_chain_id_from_headers_generic_vendor_session_id(): from litellm.proxy.litellm_pre_call_utils import get_chain_id_from_headers assert ( - get_chain_id_from_headers( - {"x-claude-code-session-id": "e96634a3-fa28-4083-b354-55542e2dca01"} - ) + get_chain_id_from_headers({"x-claude-code-session-id": "e96634a3-fa28-4083-b354-55542e2dca01"}) == "e96634a3-fa28-4083-b354-55542e2dca01" ) # Short / non-alphanumeric values should be ignored @@ -3600,19 +3492,13 @@ def test_get_internal_user_header_from_mapping_returns_expected_header(): {"header_name": "X-OpenWebUI-User-Email", "litellm_user_role": "customer"}, ] - header_name = LiteLLMProxyRequestSetup.get_internal_user_header_from_mapping( - mappings - ) + header_name = LiteLLMProxyRequestSetup.get_internal_user_header_from_mapping(mappings) assert header_name == "X-OpenWebUI-User-Id" def test_get_internal_user_header_from_mapping_none_when_absent(): - mappings = [ - {"header_name": "X-OpenWebUI-User-Email", "litellm_user_role": "customer"} - ] - header_name = LiteLLMProxyRequestSetup.get_internal_user_header_from_mapping( - mappings - ) + mappings = [{"header_name": "X-OpenWebUI-User-Email", "litellm_user_role": "customer"}] + header_name = LiteLLMProxyRequestSetup.get_internal_user_header_from_mapping(mappings) assert header_name is None single = {"header_name": "X-Only-Customer", "litellm_user_role": "customer"} @@ -3633,9 +3519,7 @@ def test_add_internal_user_from_user_mapping_sets_user_id_when_header_present(): ] } - result = LiteLLMProxyRequestSetup.add_internal_user_from_user_mapping( - general_settings, user_api_key_dict, headers - ) + result = LiteLLMProxyRequestSetup.add_internal_user_from_user_mapping(general_settings, user_api_key_dict, headers) assert result is user_api_key_dict assert user_api_key_dict.user_id == "internal-user-123" @@ -3651,9 +3535,7 @@ def test_add_internal_user_from_user_mapping_no_header_or_mapping_returns_unchan assert user_api_key_dict.user_id is None general_settings = { - "user_header_mappings": [ - {"header_name": "X-OpenWebUI-User-Id", "litellm_user_role": "internal_user"} - ] + "user_header_mappings": [{"header_name": "X-OpenWebUI-User-Id", "litellm_user_role": "internal_user"}] } result = LiteLLMProxyRequestSetup.add_internal_user_from_user_mapping( general_settings, user_api_key_dict, {"Other": "value"} @@ -3673,9 +3555,7 @@ def test_get_sanitized_user_information_from_key_includes_guardrails_metadata(): metadata={"guardrails": ["presidio", "aporia"], "other_field": "value"}, ) - result = LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key( - user_api_key_dict=user_api_key_dict - ) + result = LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict=user_api_key_dict) assert result["user_api_key_auth_metadata"] is not None assert "guardrails" in result["user_api_key_auth_metadata"] @@ -3704,9 +3584,7 @@ def test_user_and_team_spend_and_budget_flow_to_standard_logging_metadata(): team_max_budget=1000.0, ) - sanitized = LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key( - user_api_key_dict=user_api_key_dict - ) + sanitized = LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict=user_api_key_dict) assert sanitized["user_api_key_spend"] == 1.5 assert sanitized["user_api_key_max_budget"] == 10.0 @@ -3715,9 +3593,7 @@ def test_user_and_team_spend_and_budget_flow_to_standard_logging_metadata(): assert sanitized["user_api_key_team_spend"] == 250.75 assert sanitized["user_api_key_team_max_budget"] == 1000.0 - logging_metadata = StandardLoggingPayloadSetup.get_standard_logging_metadata( - dict(sanitized) - ) + logging_metadata = StandardLoggingPayloadSetup.get_standard_logging_metadata(dict(sanitized)) assert logging_metadata["user_api_key_user_spend"] == 25.5 assert logging_metadata["user_api_key_user_max_budget"] == 100.0 @@ -3734,12 +3610,8 @@ def test_user_and_team_spend_and_budget_default_to_none_in_standard_logging_meta user_api_key_dict = UserAPIKeyAuth(api_key="test-key-hash") - sanitized = LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key( - user_api_key_dict=user_api_key_dict - ) - logging_metadata = StandardLoggingPayloadSetup.get_standard_logging_metadata( - dict(sanitized) - ) + sanitized = LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict=user_api_key_dict) + logging_metadata = StandardLoggingPayloadSetup.get_standard_logging_metadata(dict(sanitized)) assert logging_metadata["user_api_key_user_spend"] is None assert logging_metadata["user_api_key_user_max_budget"] is None @@ -4071,22 +3943,16 @@ async def test_embedding_header_forwarding_with_model_group(): # Verify that only x- prefixed headers (except x-stainless) were forwarded forwarded_headers = updated_data["headers"] - assert ( - "X-Custom-Header" in forwarded_headers - ), "X-Custom-Header should be forwarded" + assert "X-Custom-Header" in forwarded_headers, "X-Custom-Header should be forwarded" assert forwarded_headers["X-Custom-Header"] == "custom-value" assert "X-Request-ID" in forwarded_headers, "X-Request-ID should be forwarded" assert forwarded_headers["X-Request-ID"] == "test-request-123" # Verify that authorization header was NOT forwarded (sensitive header) - assert ( - "Authorization" not in forwarded_headers - ), "Authorization header should not be forwarded" + assert "Authorization" not in forwarded_headers, "Authorization header should not be forwarded" # Verify that Content-Type was NOT forwarded (doesn't start with x-) - assert ( - "Content-Type" not in forwarded_headers - ), "Content-Type should not be forwarded" + assert "Content-Type" not in forwarded_headers, "Content-Type should not be forwarded" # Verify original data fields are preserved assert updated_data["model"] == "local-openai/text-embedding-3-small" @@ -4142,9 +4008,9 @@ async def test_embedding_header_forwarding_without_model_group_config(): ) # Verify that headers were NOT added since model is not in forward list - assert ( - "headers" not in updated_data or updated_data.get("headers") is None - ), "Headers should not be forwarded for models not in forward_client_headers_to_llm_api list" + assert "headers" not in updated_data or updated_data.get("headers") is None, ( + "Headers should not be forwarded for models not in forward_client_headers_to_llm_api list" + ) # Verify original data fields are preserved assert updated_data["model"] == "text-embedding-ada-002" @@ -4198,9 +4064,7 @@ async def test_add_guardrails_from_policy_engine(): attachment_registry = get_attachment_registry() attachment_registry._attachments = [ PolicyAttachment(policy="global-baseline", scope="*"), # applies to all - PolicyAttachment( - policy="healthcare", teams=["healthcare-team"] - ), # applies to healthcare team + PolicyAttachment(policy="healthcare", teams=["healthcare-team"]), # applies to healthcare team ] attachment_registry._initialized = True @@ -4269,9 +4133,9 @@ async def test_add_guardrails_from_policy_engine_accepts_dynamic_policies_and_po ) # Verify that 'policies' was removed from the request body - assert ( - "policies" not in data - ), "'policies' should be removed from request body to prevent forwarding to LLM provider" + assert "policies" not in data, ( + "'policies' should be removed from request body to prevent forwarding to LLM provider" + ) # Verify that other fields are preserved assert "model" in data @@ -4316,9 +4180,7 @@ async def test_api_created_global_policy_applies_to_new_key_without_restart(): "runtime-global-policy", Policy(guardrails=PolicyGuardrails(add=["runtime-guardrail"])), ) - attachment_registry.add_attachment( - PolicyAttachment(policy="runtime-global-policy", scope="*") - ) + attachment_registry.add_attachment(PolicyAttachment(policy="runtime-global-policy", scope="*")) await add_guardrails_from_policy_engine( data=data, @@ -4415,9 +4277,7 @@ async def test_bearer_token_not_in_debug_logs(): from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request from litellm.proxy.proxy_server import ProxyConfig - secret_token = ( - "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.fakesignature" - ) + secret_token = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.fakesignature" mock_request = MagicMock(spec=Request) mock_request.headers = { @@ -4463,8 +4323,7 @@ async def test_bearer_token_not_in_debug_logs(): log_output = log_capture.getvalue() assert secret_token not in log_output, ( - f"Bearer token leaked in debug logs. " - f"Found token in log output:\n{log_output[:500]}" + f"Bearer token leaked in debug logs. Found token in log output:\n{log_output[:500]}" ) @@ -4629,9 +4488,7 @@ def test_apply_overrides_project_model_specific(setup_test_credentials): api_key="test-key", team_metadata={ "model_config": { - "defaultconfig": { - "azure": {"litellm_credentials": "hotel-azure-eastus"} - }, + "defaultconfig": {"azure": {"litellm_credentials": "hotel-azure-eastus"}}, "gpt-4": {"azure": {"litellm_credentials": "hotel-azure-westus"}}, } }, @@ -4642,9 +4499,7 @@ def test_apply_overrides_project_model_specific(setup_test_credentials): } }, ) - _apply_credential_overrides_from_model_config( - data=data, user_api_key_dict=user_api_key_dict - ) + _apply_credential_overrides_from_model_config(data=data, user_api_key_dict=user_api_key_dict) assert data["api_base"] == "https://hotel-rec-vision.openai.azure.com/" assert data["api_key"] == "key-hotel-rec-vision" assert data["api_version"] == "2024-06-01" @@ -4657,9 +4512,7 @@ def test_apply_overrides_project_default(setup_test_credentials): api_key="test-key", team_metadata={ "model_config": { - "defaultconfig": { - "azure": {"litellm_credentials": "hotel-azure-eastus"} - }, + "defaultconfig": {"azure": {"litellm_credentials": "hotel-azure-eastus"}}, "gpt-4": {"azure": {"litellm_credentials": "hotel-azure-westus"}}, } }, @@ -4670,9 +4523,7 @@ def test_apply_overrides_project_default(setup_test_credentials): } }, ) - _apply_credential_overrides_from_model_config( - data=data, user_api_key_dict=user_api_key_dict - ) + _apply_credential_overrides_from_model_config(data=data, user_api_key_dict=user_api_key_dict) assert data["api_base"] == "https://hotel-rec-app.openai.azure.com/" assert data["api_key"] == "key-hotel-rec" @@ -4684,17 +4535,13 @@ def test_apply_overrides_team_model_specific(setup_test_credentials): api_key="test-key", team_metadata={ "model_config": { - "defaultconfig": { - "azure": {"litellm_credentials": "hotel-azure-eastus"} - }, + "defaultconfig": {"azure": {"litellm_credentials": "hotel-azure-eastus"}}, "gpt-4": {"azure": {"litellm_credentials": "hotel-azure-westus"}}, } }, project_metadata={}, ) - _apply_credential_overrides_from_model_config( - data=data, user_api_key_dict=user_api_key_dict - ) + _apply_credential_overrides_from_model_config(data=data, user_api_key_dict=user_api_key_dict) assert data["api_base"] == "https://hotel-westus.openai.azure.com/" assert data["api_key"] == "key-hotel-westus" @@ -4706,17 +4553,13 @@ def test_apply_overrides_team_default(setup_test_credentials): api_key="test-key", team_metadata={ "model_config": { - "defaultconfig": { - "azure": {"litellm_credentials": "hotel-azure-eastus"} - }, + "defaultconfig": {"azure": {"litellm_credentials": "hotel-azure-eastus"}}, "gpt-4": {"azure": {"litellm_credentials": "hotel-azure-westus"}}, } }, project_metadata={}, ) - _apply_credential_overrides_from_model_config( - data=data, user_api_key_dict=user_api_key_dict - ) + _apply_credential_overrides_from_model_config(data=data, user_api_key_dict=user_api_key_dict) assert data["api_base"] == "https://hotel-eastus.openai.azure.com/" assert data["api_key"] == "key-hotel-eastus" @@ -4729,9 +4572,7 @@ def test_apply_overrides_no_config(setup_test_credentials): team_metadata={}, project_metadata={}, ) - _apply_credential_overrides_from_model_config( - data=data, user_api_key_dict=user_api_key_dict - ) + _apply_credential_overrides_from_model_config(data=data, user_api_key_dict=user_api_key_dict) assert "api_base" not in data assert "api_key" not in data @@ -4747,17 +4588,9 @@ def test_apply_overrides_clientside_credentials_take_precedence( } user_api_key_dict = UserAPIKeyAuth( api_key="test-key", - team_metadata={ - "model_config": { - "defaultconfig": { - "azure": {"litellm_credentials": "hotel-azure-eastus"} - } - } - }, - ) - _apply_credential_overrides_from_model_config( - data=data, user_api_key_dict=user_api_key_dict + team_metadata={"model_config": {"defaultconfig": {"azure": {"litellm_credentials": "hotel-azure-eastus"}}}}, ) + _apply_credential_overrides_from_model_config(data=data, user_api_key_dict=user_api_key_dict) assert data["api_base"] == "https://my-custom-endpoint.openai.azure.com/" assert data["api_key"] == "my-custom-key" @@ -4767,15 +4600,9 @@ def test_apply_overrides_missing_credential_name(setup_test_credentials): data = {"model": "gpt-4"} user_api_key_dict = UserAPIKeyAuth( api_key="test-key", - team_metadata={ - "model_config": { - "gpt-4": {"azure": {"litellm_credentials": "nonexistent-credential"}} - } - }, - ) - _apply_credential_overrides_from_model_config( - data=data, user_api_key_dict=user_api_key_dict + team_metadata={"model_config": {"gpt-4": {"azure": {"litellm_credentials": "nonexistent-credential"}}}}, ) + _apply_credential_overrides_from_model_config(data=data, user_api_key_dict=user_api_key_dict) assert "api_base" not in data assert "api_key" not in data @@ -4785,17 +4612,9 @@ def test_apply_overrides_api_version_only_if_present(setup_test_credentials): data = {"model": "gpt-3.5"} user_api_key_dict = UserAPIKeyAuth( api_key="test-key", - team_metadata={ - "model_config": { - "defaultconfig": { - "azure": {"litellm_credentials": "hotel-azure-eastus"} - } - } - }, - ) - _apply_credential_overrides_from_model_config( - data=data, user_api_key_dict=user_api_key_dict + team_metadata={"model_config": {"defaultconfig": {"azure": {"litellm_credentials": "hotel-azure-eastus"}}}}, ) + _apply_credential_overrides_from_model_config(data=data, user_api_key_dict=user_api_key_dict) assert data["api_base"] == "https://hotel-eastus.openai.azure.com/" assert data["api_key"] == "key-hotel-eastus" assert "api_version" not in data @@ -4806,15 +4625,9 @@ def test_apply_overrides_no_model_in_data(setup_test_credentials): data = {"messages": [{"role": "user", "content": "hello"}]} user_api_key_dict = UserAPIKeyAuth( api_key="test-key", - team_metadata={ - "model_config": { - "defaultconfig": {"azure": {"litellm_credentials": "some-cred"}} - } - }, - ) - _apply_credential_overrides_from_model_config( - data=data, user_api_key_dict=user_api_key_dict + team_metadata={"model_config": {"defaultconfig": {"azure": {"litellm_credentials": "some-cred"}}}}, ) + _apply_credential_overrides_from_model_config(data=data, user_api_key_dict=user_api_key_dict) assert "api_base" not in data @@ -4826,9 +4639,7 @@ def test_apply_overrides_none_metadata(setup_test_credentials): team_metadata=None, project_metadata=None, ) - _apply_credential_overrides_from_model_config( - data=data, user_api_key_dict=user_api_key_dict - ) + _apply_credential_overrides_from_model_config(data=data, user_api_key_dict=user_api_key_dict) assert "api_base" not in data @@ -4837,15 +4648,9 @@ def test_apply_overrides_clientside_api_version_preserved(setup_test_credentials data = {"model": "gpt-4-vision", "api_version": "2025-01-01"} user_api_key_dict = UserAPIKeyAuth( api_key="test-key", - team_metadata={ - "model_config": { - "gpt-4-vision": {"azure": {"litellm_credentials": "hotel-rec-vision"}} - } - }, - ) - _apply_credential_overrides_from_model_config( - data=data, user_api_key_dict=user_api_key_dict + team_metadata={"model_config": {"gpt-4-vision": {"azure": {"litellm_credentials": "hotel-rec-vision"}}}}, ) + _apply_credential_overrides_from_model_config(data=data, user_api_key_dict=user_api_key_dict) # api_base and api_key should be set from credential assert data["api_base"] == "https://hotel-rec-vision.openai.azure.com/" assert data["api_key"] == "key-hotel-rec-vision" @@ -4858,9 +4663,7 @@ def test_resolve_non_dict_model_config_ignored(): result = _resolve_credential_from_model_config("gpt-4", "not-a-dict", None) assert result is None - result = _resolve_credential_from_model_config( - "gpt-4", None, ["also", "not", "a", "dict"] - ) + result = _resolve_credential_from_model_config("gpt-4", None, ["also", "not", "a", "dict"]) assert result is None # Valid config still works alongside invalid one @@ -4878,9 +4681,7 @@ def test_resolve_pre_alias_model_name_fallback(): "gpt-4": {"azure": {"litellm_credentials": "team-gpt4"}}, } # Post-alias name doesn't match, but pre-alias does (team scope) - result = _resolve_credential_from_model_config( - "azure/gpt-4-0613", None, team_config, pre_alias_model_name="gpt-4" - ) + result = _resolve_credential_from_model_config("azure/gpt-4-0613", None, team_config, pre_alias_model_name="gpt-4") assert result == "team-gpt4" # Same test for project scope @@ -4900,15 +4701,11 @@ def test_resolve_post_alias_name_takes_priority(): "gpt-4o-team-1": {"azure": {"litellm_credentials": "post-alias-cred"}}, } # Team scope - result = _resolve_credential_from_model_config( - "gpt-4o-team-1", None, team_config, pre_alias_model_name="gpt-4" - ) + result = _resolve_credential_from_model_config("gpt-4o-team-1", None, team_config, pre_alias_model_name="gpt-4") assert result == "post-alias-cred" # Project scope - result = _resolve_credential_from_model_config( - "gpt-4o-team-1", team_config, None, pre_alias_model_name="gpt-4" - ) + result = _resolve_credential_from_model_config("gpt-4o-team-1", team_config, None, pre_alias_model_name="gpt-4") assert result == "post-alias-cred" @@ -4940,15 +4737,9 @@ def test_apply_overrides_feature_flag_disabled_by_default(): data = {"model": "gpt-4"} user_api_key_dict = UserAPIKeyAuth( api_key="test-key", - team_metadata={ - "model_config": { - "gpt-4": {"azure": {"litellm_credentials": "hotel-azure-eastus"}} - } - }, - ) - _apply_credential_overrides_from_model_config( - data=data, user_api_key_dict=user_api_key_dict + team_metadata={"model_config": {"gpt-4": {"azure": {"litellm_credentials": "hotel-azure-eastus"}}}}, ) + _apply_credential_overrides_from_model_config(data=data, user_api_key_dict=user_api_key_dict) assert "api_base" not in data assert "api_key" not in data @@ -5108,9 +4899,7 @@ async def test_team_guardrail_merges_with_global_policy(): policy_registry = get_policy_registry() policy_registry._policies = { "global-policy": Policy( - guardrails=PolicyGuardrails( - add=["policy-guardrail-1", "policy-guardrail-2"] - ), + guardrails=PolicyGuardrails(add=["policy-guardrail-1", "policy-guardrail-2"]), ), } policy_registry._initialized = True @@ -5131,18 +4920,10 @@ async def test_team_guardrail_merges_with_global_policy(): guardrails = data["metadata"].get("guardrails", []) - assert ( - "team-direct-guardrail" in guardrails - ), f"Team guardrail missing from merged list: {guardrails}" - assert ( - "policy-guardrail-1" in guardrails - ), f"policy-guardrail-1 missing: {guardrails}" - assert ( - "policy-guardrail-2" in guardrails - ), f"policy-guardrail-2 missing: {guardrails}" - assert len(guardrails) == len( - set(guardrails) - ), f"Duplicates in guardrails list: {guardrails}" + assert "team-direct-guardrail" in guardrails, f"Team guardrail missing from merged list: {guardrails}" + assert "policy-guardrail-1" in guardrails, f"policy-guardrail-1 missing: {guardrails}" + assert "policy-guardrail-2" in guardrails, f"policy-guardrail-2 missing: {guardrails}" + assert len(guardrails) == len(set(guardrails)), f"Duplicates in guardrails list: {guardrails}" # Verify get_guardrail_from_metadata returns the merged list even # when litellm_metadata is present (the bug: it returned [] before fix) @@ -5153,9 +4934,9 @@ async def test_team_guardrail_merges_with_global_policy(): dummy = _DummyGuardrail(guardrail_name="team-direct-guardrail") returned = dummy.get_guardrail_from_metadata(data) - assert ( - "team-direct-guardrail" in returned - ), f"get_guardrail_from_metadata shadowed by litellm_metadata; got: {returned}" + assert "team-direct-guardrail" in returned, ( + f"get_guardrail_from_metadata shadowed by litellm_metadata; got: {returned}" + ) finally: policy_registry._policies = {} @@ -5208,9 +4989,7 @@ def test_get_guardrail_from_metadata_reads_litellm_metadata_when_no_metadata(): } result = dummy.get_guardrail_from_metadata(data) - assert result == [ - "my-guardrail" - ], f"Expected guardrails from litellm_metadata fallback, got: {result}" + assert result == ["my-guardrail"], f"Expected guardrails from litellm_metadata fallback, got: {result}" def _build_request_mock_with_headers(headers: dict) -> Request: @@ -5237,9 +5016,7 @@ class TestApplyClientTagPolicyPreAuth: """ def test_merges_header_tags_into_metadata(self): - request_mock = _build_request_mock_with_headers( - {"x-litellm-tags": "tenant:acme,env:prod"} - ) + request_mock = _build_request_mock_with_headers({"x-litellm-tags": "tenant:acme,env:prod"}) data = {"model": "gpt-3.5-turbo"} user_api_key_dict = UserAPIKeyAuth( api_key="hashed-key", @@ -5256,9 +5033,7 @@ class TestApplyClientTagPolicyPreAuth: assert data["metadata"]["tags"] == ["tenant:acme", "env:prod"] def test_unions_header_tags_with_existing_metadata_tags(self): - request_mock = _build_request_mock_with_headers( - {"x-litellm-tags": "tenant:acme,env:prod"} - ) + request_mock = _build_request_mock_with_headers({"x-litellm-tags": "tenant:acme,env:prod"}) data = { "model": "gpt-3.5-turbo", "metadata": {"tags": ["env:prod", "team:platform"]}, @@ -5283,9 +5058,7 @@ class TestApplyClientTagPolicyPreAuth: # (inside common_checks) enforces per-tag budgets on whatever tags # it sees in request_data, including body tags. The helper only # adds header tags to metadata.tags. - request_mock = _build_request_mock_with_headers( - {"x-litellm-tags": "tenant:acme"} - ) + request_mock = _build_request_mock_with_headers({"x-litellm-tags": "tenant:acme"}) data = { "model": "gpt-3.5-turbo", "tags": ["root-tag"], @@ -5312,9 +5085,7 @@ class TestApplyClientTagPolicyPreAuth: ] def test_uses_litellm_metadata_when_present(self): - request_mock = _build_request_mock_with_headers( - {"x-litellm-tags": "tenant:acme"} - ) + request_mock = _build_request_mock_with_headers({"x-litellm-tags": "tenant:acme"}) data = { "model": "gpt-3.5-turbo", "litellm_metadata": {"foo": "bar"}, @@ -5409,9 +5180,7 @@ class TestApplyClientTagPolicyPreAuth: litellm_budget_table=LiteLLM_BudgetTable(max_budget=0.10), ) - async def mock_get_current_spend( - counter_key, fallback_spend, max_budget=None, **kwargs - ): + async def mock_get_current_spend(counter_key, fallback_spend, max_budget=None, **kwargs): if counter_key == "spend:tag:paid": return 0.50 return fallback_spend @@ -5446,9 +5215,7 @@ class TestApplyClientTagPolicyPreAuth: from litellm.proxy.auth.auth_checks import _tag_max_budget_check from litellm.proxy.utils import ProxyLogging - request_mock = _build_request_mock_with_headers( - {"x-litellm-tags": "tenant:acme"} - ) + request_mock = _build_request_mock_with_headers({"x-litellm-tags": "tenant:acme"}) data = {"model": "gpt-3.5-turbo"} user_api_key_dict = UserAPIKeyAuth( api_key="hashed-key", @@ -5468,9 +5235,7 @@ class TestApplyClientTagPolicyPreAuth: litellm_budget_table=LiteLLM_BudgetTable(max_budget=0.10), ) - async def mock_get_current_spend( - counter_key, fallback_spend, max_budget=None, **kwargs - ): + async def mock_get_current_spend(counter_key, fallback_spend, max_budget=None, **kwargs): if counter_key == "spend:tag:tenant:acme": return 0.50 return fallback_spend @@ -5505,9 +5270,7 @@ class TestApplyClientTagPolicyPreAuth: "/v1/messages", ], ) - async def test_header_tags_visible_to_tag_max_budget_check_on_metadata_route( - self, route - ): + async def test_header_tags_visible_to_tag_max_budget_check_on_metadata_route(self, route): """Regression: on LITELLM_METADATA_ROUTES (bedrock, /v1/messages, ...), common_checks pre-seeds ``litellm_metadata`` and writes key tags there before ``_tag_max_budget_check`` reads from the same key. The auth wrapper @@ -5521,9 +5284,7 @@ class TestApplyClientTagPolicyPreAuth: from litellm.proxy.auth.auth_checks import common_checks from litellm.proxy.utils import ProxyLogging - request_mock = _build_request_mock_with_headers( - {"x-litellm-tags": "tenant:acme"} - ) + request_mock = _build_request_mock_with_headers({"x-litellm-tags": "tenant:acme"}) data = {"model": "us.anthropic.claude-sonnet-4-6"} valid_token = UserAPIKeyAuth( token="test-token", @@ -5548,9 +5309,7 @@ class TestApplyClientTagPolicyPreAuth: litellm_budget_table=LiteLLM_BudgetTable(max_budget=0.10), ) - async def mock_get_current_spend( - counter_key, fallback_spend, max_budget=None, **kwargs - ): + async def mock_get_current_spend(counter_key, fallback_spend, max_budget=None, **kwargs): if counter_key == "spend:tag:tenant:acme": return 0.50 return fallback_spend @@ -5718,9 +5477,7 @@ class TestApplyKeyTagsPreAuth: litellm_budget_table=LiteLLM_BudgetTable(max_budget=0.10), ) - async def mock_get_current_spend( - counter_key, fallback_spend, max_budget=None, **kwargs - ): + async def mock_get_current_spend(counter_key, fallback_spend, max_budget=None, **kwargs): if counter_key == "spend:tag:engineering": return 0.50 return fallback_spend @@ -5771,9 +5528,7 @@ class TestApplyKeyTagsPreAuth: litellm_budget_table=LiteLLM_BudgetTable(max_budget=0.10), ) - async def mock_get_current_spend( - counter_key, fallback_spend, max_budget=None, **kwargs - ): + async def mock_get_current_spend(counter_key, fallback_spend, max_budget=None, **kwargs): if counter_key == "spend:tag:engineering": return 0.05 return fallback_spend @@ -5854,9 +5609,7 @@ def test_resolve_provider_from_deployment_falls_back_to_pre_alias(): router.get_deployment_by_model_group_name.side_effect = lookup - result = _resolve_provider_from_deployment( - router, "post-alias-name", pre_alias_model_name="pre-alias-name" - ) + result = _resolve_provider_from_deployment(router, "post-alias-name", pre_alias_model_name="pre-alias-name") assert result == "bedrock" @@ -5921,17 +5674,9 @@ def test_apply_overrides_no_router_keeps_legacy_behaviour(setup_test_credentials data = {"model": "gpt-4"} user_api_key_dict = UserAPIKeyAuth( api_key="test-key", - team_metadata={ - "model_config": { - "defaultconfig": { - "azure": {"litellm_credentials": "hotel-azure-eastus"} - } - } - }, - ) - _apply_credential_overrides_from_model_config( - data=data, user_api_key_dict=user_api_key_dict, llm_router=None + team_metadata={"model_config": {"defaultconfig": {"azure": {"litellm_credentials": "hotel-azure-eastus"}}}}, ) + _apply_credential_overrides_from_model_config(data=data, user_api_key_dict=user_api_key_dict, llm_router=None) assert data["api_base"] == "https://hotel-eastus.openai.azure.com/" assert data["api_key"] == "key-hotel-eastus" @@ -5957,9 +5702,7 @@ def test_apply_overrides_provider_prefix_in_model_skips_router_lookup( ) router = MagicMock() - _apply_credential_overrides_from_model_config( - data=data, user_api_key_dict=user_api_key_dict, llm_router=router - ) + _apply_credential_overrides_from_model_config(data=data, user_api_key_dict=user_api_key_dict, llm_router=router) assert data["api_base"] == "https://hotel-eastus.openai.azure.com/" assert data["api_key"] == "key-hotel-eastus" router.get_deployment_by_model_group_name.assert_not_called() @@ -6338,9 +6081,7 @@ def test_get_sanitized_user_information_from_key_drops_callback_config(): }, ) - result = LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key( - user_api_key_dict=user_api_key_dict - ) + result = LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict=user_api_key_dict) auth_metadata = result["user_api_key_auth_metadata"] assert "logging" not in auth_metadata @@ -6380,9 +6121,7 @@ def test_team_alias_targeting_deleted_team_deployment_keeps_requested_model(monk ) with patch("litellm.proxy.proxy_server.llm_router", _MockRouter()): - _update_model_if_team_alias_exists( - data=test_data, user_api_key_dict=user_api_key_dict - ) + _update_model_if_team_alias_exists(data=test_data, user_api_key_dict=user_api_key_dict) assert test_data.get("model") == "gpt-4" @@ -6406,9 +6145,7 @@ def test_team_alias_targeting_live_team_deployment_still_rewrites(monkeypatch): ) with patch("litellm.proxy.proxy_server.llm_router", _MockRouter()): - _update_model_if_team_alias_exists( - data=test_data, user_api_key_dict=user_api_key_dict - ) + _update_model_if_team_alias_exists(data=test_data, user_api_key_dict=user_api_key_dict) assert test_data.get("model") == "model_name_team-1_live-uuid" @@ -6545,7 +6282,6 @@ async def test_add_litellm_data_to_request_keeps_every_forwarded_credential_out_ assert value not in logged - @pytest.mark.parametrize( "header, expected_redacted", [ @@ -6571,7 +6307,6 @@ def test_redact_credential_headers_classifies_each_header(header, expected_redac assert headers[header] == "secret-value" - @pytest.mark.asyncio async def test_add_litellm_data_to_request_debug_log_does_not_print_credentials(): """The request-header debug line carries values the stdout secret filter does not match.""" @@ -7191,8 +6926,7 @@ AUTHORIZATION_HEADER_CASINGS = ["authorization", "Authorization", "AUTHORIZATION LEAK_TARGET_PROVIDERS = ["bedrock", "bedrock_converse", "vertex_ai"] BEDROCK_ENDPOINT = ( - "https://bedrock-runtime.us-west-2.amazonaws.com" - "/model/us.anthropic.claude-sonnet-4-5-20250929-v1:0/invoke" + "https://bedrock-runtime.us-west-2.amazonaws.com/model/us.anthropic.claude-sonnet-4-5-20250929-v1:0/invoke" ) BEDROCK_REGION = "us-west-2" BEDROCK_REQUEST_DATA = {"messages": [{"role": "user", "content": "Say OK"}], "max_tokens": 32} @@ -7250,9 +6984,7 @@ def _signed_headers_component(signature: str, component: str) -> str: @pytest.mark.parametrize("authorization_header_name", AUTHORIZATION_HEADER_CASINGS) @pytest.mark.parametrize("custom_llm_provider", LEAK_TARGET_PROVIDERS) -def test_oauth_credential_is_never_forwarded_to_bedrock_or_vertex( - authorization_header_name, custom_llm_provider -): +def test_oauth_credential_is_never_forwarded_to_bedrock_or_vertex(authorization_header_name, custom_llm_provider): """ A client's Anthropic OAuth credential is meaningless to AWS and Google, and sending it there both breaks the request and hands a third-party cloud a credential it should @@ -7280,9 +7012,7 @@ def test_oauth_credential_entry_is_scoped_to_anthropic_alone(): if not isinstance(scoped_headers, list): scoped_headers = [scoped_headers] - credential_entries = [ - entry for entry in scoped_headers if OAUTH_TOKEN in entry["extra_headers"].values() - ] + credential_entries = [entry for entry in scoped_headers if OAUTH_TOKEN in entry["extra_headers"].values()] assert [entry["custom_llm_provider"] for entry in credential_entries] == ["anthropic"] @@ -7344,9 +7074,7 @@ def test_bedrock_get_request_headers_keeps_the_sigv4_signature(): def test_bedrock_api_key_deployment_keeps_its_own_bearer_token(): forwarded = _headers_forwarded_to(_client_headers(), "bedrock") - signed = _signed_headers_for_bedrock( - {"Content-Type": "application/json", **forwarded}, api_key=BEDROCK_API_KEY - ) + signed = _signed_headers_for_bedrock({"Content-Type": "application/json", **forwarded}, api_key=BEDROCK_API_KEY) assert _authorization_values(signed) == [f"Bearer {BEDROCK_API_KEY}"] @@ -7369,6 +7097,8 @@ def test_vertex_sends_exactly_one_authorization_header(): vertex_request_headers.update(forwarded) assert _authorization_values(vertex_request_headers) == [GOOGLE_ACCESS_TOKEN] + + @pytest.mark.asyncio async def test_newrelic_team_callback_vars_reach_trusted_field(): """A key with a newrelic team callback stamps its vars into the proxy-owned @@ -7737,13 +7467,13 @@ def _request_for(path: str) -> MagicMock: return request -def _spend_log_session_id(data: dict[str, object]) -> str: - """Resolve session_id the way LiteLLM_SpendLogs does: standard_logging_payload.trace_id.""" +def _spend_log_session_id(data: dict[str, object], metadata_key: str = "metadata") -> str | None: + """Resolve session_id the way LiteLLM_SpendLogs does, reading the omit decision stamped on the request.""" from litellm.litellm_core_utils.get_litellm_params import get_litellm_params from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup from litellm.proxy.spend_tracking.spend_tracking_utils import _get_session_id_for_spend_log - metadata = data["metadata"] + metadata = data[metadata_key] assert isinstance(metadata, dict) litellm_params = get_litellm_params( litellm_session_id=str(data["litellm_session_id"]) if "litellm_session_id" in data else None, @@ -7754,7 +7484,12 @@ def _spend_log_session_id(data: dict[str, object]) -> str: logging_obj=SimpleNamespace(litellm_trace_id="per-call-random-trace-id"), litellm_params=litellm_params, ) - return _get_session_id_for_spend_log(kwargs={}, standard_logging_payload={"trace_id": trace_id}) + return _get_session_id_for_spend_log( + kwargs={}, + metadata=metadata, + standard_logging_payload={"trace_id": trace_id}, + omit_when_missing=bool(metadata.get(SESSION_ID_OMITTED_METADATA_KEY)), + ) @pytest.mark.asyncio @@ -7780,9 +7515,10 @@ async def test_missing_session_id_generate_makes_spend_log_and_callback_session_ assert isinstance(callback_session_id, str) and len(callback_session_id) == 36 assert _spend_log_session_id(updated) == callback_session_id assert updated["metadata"][SESSION_ID_GENERATED_METADATA_KEY] is True - assert get_fireworks_session_id( - {"litellm_session_id": updated["litellm_session_id"], "metadata": updated["metadata"]} - ) is None + assert ( + get_fireworks_session_id({"litellm_session_id": updated["litellm_session_id"], "metadata": updated["metadata"]}) + is None + ) @pytest.mark.asyncio @@ -7800,6 +7536,44 @@ async def test_missing_session_id_unset_keeps_legacy_divergence(): assert _spend_log_session_id(updated) == "per-call-random-trace-id" +@pytest.mark.asyncio +async def test_missing_session_id_omit_leaves_spend_log_session_id_null(): + """Under `omit` a traceparent still becomes the trace id but never a session id, so SpendLogs and + Langfuse agree on having no session.""" + request = _request_for("/v1/chat/completions") + request.headers = {"traceparent": "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"} + + updated = await add_litellm_data_to_request( + data={"model": "gpt-4o", "messages": []}, + request=request, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": "omit"}, + ) + + assert "session_id" not in updated["metadata"] + assert updated["litellm_trace_id"] == "4bf92f3577b34da6a3ce929d0e0e4736" + assert updated["metadata"][SESSION_ID_OMITTED_METADATA_KEY] is True + assert _spend_log_session_id(updated) is None + + +@pytest.mark.asyncio +async def test_missing_session_id_omit_keeps_client_supplied_session_id(): + request = _request_for("/v1/chat/completions") + request.headers = {"x-litellm-session-id": "client-session-1"} + + updated = await add_litellm_data_to_request( + data={"model": "gpt-4o", "messages": []}, + request=request, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": "omit"}, + ) + + assert updated["metadata"]["session_id"] == "client-session-1" + assert _spend_log_session_id(updated) == "client-session-1" + + @pytest.mark.asyncio async def test_missing_session_id_generate_reuses_traceparent_trace_id(): """A W3C traceparent already decides SpendLogs.session_id, so the callback session id must reuse it.""" @@ -7895,3 +7669,38 @@ async def test_missing_session_id_unknown_value_is_ignored(): ) assert "session_id" not in updated["metadata"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "path, sent_in, metadata_key, general_settings", + [ + ("/v1/chat/completions", "metadata", "metadata", {}), + ("/v1/chat/completions", "litellm_metadata", "metadata", {}), + ("/v1/chat/completions", "litellm_metadata", "metadata", {"missing_session_id": "generate"}), + ("/v1/messages", "litellm_metadata", "litellm_metadata", {}), + ("/v1/messages", "metadata", "litellm_metadata", {}), + ("/mcp/tools", "metadata", "metadata", {"missing_session_id": "omit"}), + ("/mcp/tools", "litellm_metadata", "metadata", {"missing_session_id": "omit"}), + ], +) +async def test_client_supplied_omit_marker_never_reaches_the_spend_log( + path: str, sent_in: str, metadata_key: str, general_settings: dict[str, str] +): + """The omit marker is proxy-owned: only the pre-call policy may set it. A caller that sends it in either + metadata bucket, including the one later merged into the route's bucket, must not be able to null out + SpendLogs.session_id on a request the proxy did not omit.""" + updated = await add_litellm_data_to_request( + data={"model": "gpt-4o", "messages": [], sent_in: {SESSION_ID_OMITTED_METADATA_KEY: True}}, + request=_request_for(path), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings=general_settings, + ) + + assert SESSION_ID_OMITTED_METADATA_KEY not in updated[metadata_key] + assert _spend_log_session_id(updated, metadata_key) == ( + updated[metadata_key]["session_id"] + if general_settings.get("missing_session_id") == "generate" + else "per-call-random-trace-id" + ) diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index dcaad968663..f16c6c937d0 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -942,6 +942,47 @@ def test_create_model_info_response_uses_deployment_limits_when_not_in_cost_map( assert response["max_output_tokens"] == 8000 +def test_create_model_info_response_uses_deployment_mode_for_auto_router(): + router = litellm.Router( + model_list=[ + { + "model_name": "claude-sonnet", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "test-key"}, + }, + { + "model_name": "claude-auto", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": { + "tiers": { + "SIMPLE": "claude-sonnet", + "MEDIUM": "claude-sonnet", + "COMPLEX": "claude-sonnet", + } + }, + "complexity_router_default_model": "claude-sonnet", + }, + "model_info": { + "mode": "chat", + "max_input_tokens": 1_000_000, + "max_output_tokens": 128_000, + }, + }, + ] + ) + + response = create_model_info_response( + model_id="claude-auto", + provider="openai", + llm_router=router, + get_model_info=_raise_unmapped, + ) + + assert response["mode"] == "chat" + assert response["max_input_tokens"] == 1_000_000 + assert response["max_output_tokens"] == 128_000 + + def test_create_model_info_response_deployment_limits_override_cost_map(): router = MagicMock() router.get_configured_token_limits.return_value = (200000, None) diff --git a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_access_control.py b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_access_control.py index 7d72121456a..93049b21460 100644 --- a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_access_control.py +++ b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_access_control.py @@ -120,9 +120,7 @@ async def test_delete_vector_store_checks_access(): "team_id": "team_456", } ) - mock_prisma.db.litellm_managedvectorstorestable.find_unique = AsyncMock( - return_value=mock_vector_store - ) + mock_prisma.db.litellm_managedvectorstorestable.find_unique = AsyncMock(return_value=mock_vector_store) # User from different team should get 403 user_api_key_dict = UserAPIKeyAuth(team_id="team_789") @@ -134,9 +132,115 @@ async def test_delete_vector_store_checks_access(): ): with patch("litellm.vector_store_registry", None): with pytest.raises(HTTPException) as exc_info: - await delete_vector_store( - data=request, user_api_key_dict=user_api_key_dict - ) + await delete_vector_store(data=request, user_api_key_dict=user_api_key_dict) assert exc_info.value.status_code == 403 assert "Access denied" in exc_info.value.detail + + +_UNSCOPED: LiteLLM_ManagedVectorStore = { + "vector_store_id": "vs_unscoped", + "custom_llm_provider": "openai", + "team_id": None, +} +_TEAM_A_OWNED: LiteLLM_ManagedVectorStore = { + "vector_store_id": "vs_team_a", + "custom_llm_provider": "openai", + "team_id": "team_a", +} +_UI_CREATED: LiteLLM_ManagedVectorStore = { + "vector_store_id": "vs_ui_created", + "custom_llm_provider": "openai", + "team_id": "litellm-dashboard", +} + + +async def _listed_ids(user_api_key_dict: UserAPIKeyAuth) -> list[str]: + from litellm.proxy.vector_store_endpoints.management_endpoints import ( + list_vector_stores, + ) + + with patch( # test-quality-ok: the list route reads rows through this module-level DB helper, no injection seam + "litellm.proxy.vector_store_endpoints.management_endpoints.VectorStoreRegistry._get_vector_stores_from_db", + new=AsyncMock(return_value=[_UNSCOPED, _TEAM_A_OWNED, _UI_CREATED]), + ): + response = await list_vector_stores(user_api_key_dict=user_api_key_dict) + return sorted(vs["vector_store_id"] for vs in response["data"]) + + +@pytest.mark.asyncio +async def test_list_vector_stores_hides_ungranted_stores_from_non_admin_keys(): + """A store with no team_id and no allowlist entry is not listed for a key it was never granted to; + only team ownership or an explicit object_permission grant makes a store visible.""" + assert await _listed_ids(UserAPIKeyAuth()) == [] + assert await _listed_ids(UserAPIKeyAuth(team_id="team_a")) == ["vs_team_a"] + assert await _listed_ids( + UserAPIKeyAuth( + team_id="team_b", + object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="op-1", vector_stores=["vs_unscoped"]), + ) + ) == ["vs_unscoped"] + assert await _listed_ids( + UserAPIKeyAuth( + team_id="team_b", + team_object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id="op-2", vector_stores=["vs_unscoped"] + ), + ) + ) == ["vs_unscoped"] + assert await _listed_ids(UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN)) == [ + "vs_team_a", + "vs_ui_created", + "vs_unscoped", + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("user_team_ids", "session_key_grants", "expected"), + [ + ([], None, []), + ([], ["vs_unscoped"], ["vs_unscoped"]), + (["team_a"], None, ["vs_team_a"]), + (["team_a", "team_granted"], None, ["vs_team_a", "vs_unscoped"]), + ], +) +async def test_list_vector_stores_dashboard_session_resolves_real_teams( + user_team_ids: list[str], session_key_grants: list[str] | None, expected: list[str] +): + """A dashboard session lists through the user's real teams plus the session key's own grants: stores created + from the dashboard (team_id litellm-dashboard) are not visible just because every session shares that team id, + while stores owned by or granted to one of the user's teams, or granted to the session key itself, are.""" + from litellm.models.team import LiteLLM_TeamTableCachedObj + + alice = UserAPIKeyAuth( + team_id="litellm-dashboard", + user_id="alice", + user_role=LitellmUserRoles.INTERNAL_USER, + object_permission=( + LiteLLM_ObjectPermissionTable(object_permission_id="op-4", vector_stores=session_key_grants) + if session_key_grants is not None + else None + ), + ) + teams = { + "team_a": LiteLLM_TeamTableCachedObj(team_id="team_a"), + "team_granted": LiteLLM_TeamTableCachedObj( + team_id="team_granted", + object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="op-3", vector_stores=["vs_unscoped"]), + ), + } + + async def fake_get_team_object(team_id: str, **_kwargs: object) -> LiteLLM_TeamTableCachedObj: + return teams[team_id] + + with ( + patch( # test-quality-ok: team rows come from the module-level prisma client, no injection seam + "litellm.proxy.auth.auth_checks.get_team_object", new=fake_get_team_object + ), + patch( # test-quality-ok: the user row comes from the module-level prisma client, no injection seam + "litellm.proxy.vector_store_endpoints.utils.resolve_ui_session_team_ids", + new=AsyncMock(return_value=user_team_ids), + ), + ): + assert await _listed_ids(alice) == expected diff --git a/tests/test_litellm/realtime_api/test_main.py b/tests/test_litellm/realtime_api/test_main.py index a3dd5688ad1..761e87ac764 100644 --- a/tests/test_litellm/realtime_api/test_main.py +++ b/tests/test_litellm/realtime_api/test_main.py @@ -1,6 +1,7 @@ import asyncio import time from types import TracebackType +from typing import Final from unittest.mock import MagicMock, patch @@ -294,3 +295,39 @@ async def test_azure_health_check_honors_deployment_realtime_protocol(): model_params={"realtime_protocol": "GA"}, ) assert connect.url == "wss://my-endpoint.openai.azure.com/openai/v1/realtime?model=gpt-4o-realtime-preview" + + +class _ConnectThatStopsAfterCapturingTheUrl: + url: str | None = None + + def __call__(self, url: str, **kwargs: object) -> "_ConnectThatStopsAfterCapturingTheUrl": + self.url = url + return self + + async def __aenter__(self) -> None: + raise RuntimeError("backend url captured, nothing to bridge") + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, + ) -> None: + return None + + +@pytest.mark.asyncio +async def test_arealtime_azure_ai_on_a_foundry_host_connects_to_the_azure_openai_realtime_route(): + connect: Final = _ConnectThatStopsAfterCapturingTheUrl() + with patch("websockets.connect", connect): + await realtime_main._arealtime.__wrapped__( + model="azure_ai/gpt-realtime-mini", + websocket=MagicMock(), + api_base="https://my-project.services.ai.azure.com", + api_key="fake-key", + litellm_logging_obj=FakeLogging(), + ) + assert connect.url == ( + "wss://my-project.services.ai.azure.com/openai/realtime" + "?api-version=2024-10-01-preview&deployment=gpt-realtime-mini" + ) diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index 4a03913f55a..719d51c11e3 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -25,6 +25,7 @@ from litellm.types.utils import ( ModelResponse, ModelResponseStream, StreamingChoices, + Usage, ) CHAT_COMPLETION_ID = "chatcmpl-77d33d09-effa-4cd2-9c0d-c742d4358256" @@ -527,6 +528,20 @@ async def test_streaming_response_id_falls_back_when_upstream_yields_nothing(): assert response_ids[0].startswith("resp_") +def test_completed_event_restores_usage_hidden_by_stream_options_none(): + final_chunk = _chunk("", finish_reason="stop") + final_chunk._hidden_params = {"usage": Usage(prompt_tokens=117, completion_tokens=5, total_tokens=122)} + iterator = _build_iterator([_chunk("the document says hello"), final_chunk]) + + events = list(iterator) + + completed = next( + event for event in events if getattr(event, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + ) + assert completed.response.usage.input_tokens == 117 + assert completed.response.usage.output_tokens == 5 + + def test_object_tool_call_arguments_stream_as_valid_json(): """A provider that sends decoded object arguments must still stream valid JSON. diff --git a/tests/test_litellm/router_utils/pre_call_checks/test_deployment_affinity_check.py b/tests/test_litellm/router_utils/pre_call_checks/test_deployment_affinity_check.py index 60433921de6..b5651062098 100644 --- a/tests/test_litellm/router_utils/pre_call_checks/test_deployment_affinity_check.py +++ b/tests/test_litellm/router_utils/pre_call_checks/test_deployment_affinity_check.py @@ -1,11 +1,12 @@ import asyncio +import itertools +import json +from collections.abc import Sequence +from typing import Final from unittest.mock import AsyncMock, patch import pytest - -import json - import litellm from litellm.caching.dual_cache import DualCache from litellm.router_utils.pre_call_checks.deployment_affinity_check import ( @@ -102,11 +103,10 @@ async def test_async_user_key_affinity_routes_to_same_deployment(): # Deterministic routing: first selection uses seq[0], second selection attempts seq[1] # unless the list has been filtered to length=1 by deployment affinity. - choice_calls = {"count": 0} + choice_calls: Final = itertools.count(1) - def deterministic_choice(seq): - choice_calls["count"] += 1 - if choice_calls["count"] == 1: + def deterministic_choice(seq: Sequence[dict[str, object]]) -> dict[str, object]: + if next(choice_calls) == 1: return seq[0] return seq[1] if len(seq) > 1 else seq[0] @@ -998,3 +998,212 @@ async def test_model_group_affinity_config_overrides_global(): ) # All deployments returned (user-key affinity disabled for this group) assert len(filtered) == 2 + + +def _jwt_metadata(user_id: str) -> dict[str, str | None]: + return {"user_api_key_hash": None, "user_api_key_user_id": user_id} + + +def _two_deployments(model_group: str) -> list[dict]: + return [ + { + "model_name": model_group, + "litellm_params": {"model": "openai/gpt-5.4-mini"}, + "model_info": {"id": "openai-deployment-a"}, + }, + { + "model_name": model_group, + "litellm_params": {"model": "openai/gpt-5.4-mini"}, + "model_info": {"id": "openai-deployment-b"}, + }, + ] + + +@pytest.mark.asyncio +async def test_async_jwt_user_affinity_routes_to_same_deployment(): + """ + JWT-authenticated proxy requests carry no `user_api_key_hash`, only `user_api_key_user_id`. + They must still pin to one deployment per user. + """ + model_group = "gpt-5.4-mini" + router = litellm.Router( + model_list=[ + { + "model_name": model_group, + "litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "mock-api-key-a"}, + "model_info": {"id": "openai-deployment-a"}, + }, + { + "model_name": model_group, + "litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "mock-api-key-b"}, + "model_info": {"id": "openai-deployment-b"}, + }, + ], + optional_pre_call_checks=["deployment_affinity"], + ) + + choice_calls = {"count": 0} + + def deterministic_choice(seq): + choice_calls["count"] += 1 + if choice_calls["count"] == 1: + return seq[0] + return seq[1] if len(seq) > 1 else seq[0] + + with patch( # test-quality-ok: simple-shuffle has no injectable RNG; forcing the other pick is what proves the pin overrides the strategy + "litellm.router_strategy.simple_shuffle.random.choice", + side_effect=deterministic_choice, + ): + first_response = await router.acompletion( + model=model_group, + messages=[{"role": "user", "content": "Reply with the single word ok"}], + mock_response="ok", + metadata=_jwt_metadata("jwt-user-alice"), + ) + second_response = await router.acompletion( + model=model_group, + messages=[{"role": "user", "content": "Reply with the single word ok"}], + mock_response="ok", + metadata=_jwt_metadata("jwt-user-alice"), + ) + + first_model_id = first_response._hidden_params["model_id"] + assert first_model_id in ("openai-deployment-a", "openai-deployment-b") + assert second_response._hidden_params["model_id"] == first_model_id + + +@pytest.mark.asyncio +async def test_proxy_jwt_auth_metadata_pins_per_user(): + """ + The metadata the proxy stamps for a JWT caller (`UserAPIKeyAuth(api_key=None, user_id=)`) + must claim a pin and be read back by the filter, and another JWT user must not inherit it. + """ + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + + model_group = "gpt-5.4-mini" + healthy_deployments = _two_deployments(model_group) + callback = DeploymentAffinityCheck( + cache=DualCache(), + ttl_seconds=60, + enable_user_key_affinity=True, + enable_responses_api_affinity=False, + ) + + def proxy_request(user_id: str) -> dict[str, object]: + return LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata( + data={"model": model_group, "messages": [{"role": "user", "content": "hi"}], "metadata": {}}, + user_api_key_dict=UserAPIKeyAuth(api_key=None, user_id=user_id), + _metadata_variable_name="metadata", + ) + + alice_request = proxy_request("jwt-user-alice") + alice_metadata = alice_request["metadata"] + assert isinstance(alice_metadata, dict) + assert alice_metadata["user_api_key_hash"] is None + + await callback.async_pre_call_deployment_hook( + kwargs={ + **alice_request, + "metadata": {**alice_metadata, "deployment_model_name": model_group}, + "model_info": {"id": "openai-deployment-b"}, + }, + call_type=None, + ) + + alice_pinned = await callback.async_filter_deployments( + model=model_group, + healthy_deployments=healthy_deployments, + messages=None, + request_kwargs=alice_request, + parent_otel_span=None, + ) + assert [deployment["model_info"]["id"] for deployment in alice_pinned] == ["openai-deployment-b"] + + bob_filtered = await callback.async_filter_deployments( + model=model_group, + healthy_deployments=healthy_deployments, + messages=None, + request_kwargs=proxy_request("jwt-user-bob"), + parent_otel_span=None, + ) + assert bob_filtered == healthy_deployments + + +@pytest.mark.asyncio +async def test_jwt_user_id_never_reads_a_virtual_key_pin(): + """ + A JWT user id that happens to equal a virtual key's 64-hex hash must not read that key's pin. + """ + model_group = "gpt-5.4-mini" + healthy_deployments = _two_deployments(model_group) + callback = DeploymentAffinityCheck( + cache=DualCache(), + ttl_seconds=60, + enable_user_key_affinity=True, + enable_responses_api_affinity=False, + ) + key_hash = "a" * 64 + + await callback.async_pre_call_deployment_hook( + kwargs={ + "metadata": {"user_api_key_hash": key_hash, "deployment_model_name": model_group}, + "model_info": {"id": "openai-deployment-b"}, + }, + call_type=None, + ) + + key_pinned = await callback.async_filter_deployments( + model=model_group, + healthy_deployments=healthy_deployments, + messages=None, + request_kwargs={"metadata": {"user_api_key_hash": key_hash}}, + parent_otel_span=None, + ) + assert [deployment["model_info"]["id"] for deployment in key_pinned] == ["openai-deployment-b"] + + lookalike_jwt_user = await callback.async_filter_deployments( + model=model_group, + healthy_deployments=healthy_deployments, + messages=None, + request_kwargs={"metadata": _jwt_metadata(key_hash)}, + parent_otel_span=None, + ) + assert lookalike_jwt_user == healthy_deployments + + +@pytest.mark.asyncio +async def test_virtual_key_hash_wins_over_user_id_for_affinity(): + """ + A virtual-key caller with a user id pins on the key hash, so two keys owned by one user + keep independent pins. + """ + model_group = "gpt-5.4-mini" + healthy_deployments = _two_deployments(model_group) + callback = DeploymentAffinityCheck( + cache=DualCache(), + ttl_seconds=60, + enable_user_key_affinity=True, + enable_responses_api_affinity=False, + ) + + await callback.async_pre_call_deployment_hook( + kwargs={ + "metadata": { + "user_api_key_hash": "key-one", + "user_api_key_user_id": "shared-user", + "deployment_model_name": model_group, + }, + "model_info": {"id": "openai-deployment-b"}, + }, + call_type=None, + ) + + other_key_same_user = await callback.async_filter_deployments( + model=model_group, + healthy_deployments=healthy_deployments, + messages=None, + request_kwargs={"metadata": {"user_api_key_hash": "key-two", "user_api_key_user_id": "shared-user"}}, + parent_otel_span=None, + ) + assert other_key_same_user == healthy_deployments diff --git a/tests/test_litellm/router_utils/test_reasoning_effort_capability.py b/tests/test_litellm/router_utils/test_reasoning_effort_capability.py index a0dbf3b6637..7b2e45ab3ed 100644 --- a/tests/test_litellm/router_utils/test_reasoning_effort_capability.py +++ b/tests/test_litellm/router_utils/test_reasoning_effort_capability.py @@ -371,3 +371,20 @@ class TestKimiK3AdvertisesItsDocumentedLevels: "low", "high", ) + + +class TestGpt6AstraAdvertisesItsDocumentedLevels: + def test_the_entry_advertises_low_through_max_without_none(self, local_model_cost_map): + """OpenAI documents low, medium, high, xhigh and max for gpt-6-astra. Unlike gpt-5.6-sol it + does not take none, so a group must not offer none and must offer max.""" + from litellm.utils import _get_model_info_helper + + model_info = dict(_get_model_info_helper(model="gpt-6-astra", custom_llm_provider="openai")) + + assert resolve_supported_reasoning_efforts(model_info, deployment_is_mapped=True) == ( + "low", + "medium", + "high", + "xhigh", + "max", + ) diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index db08ee486bf..6f9548dc1d7 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -4485,3 +4485,17 @@ def test_cost_per_token_mistral_voxtral_tts_bills_per_input_character(_local_mod assert prompt_usd == pytest.approx(1000 * 1.6e-05) assert completion_usd == 0.0 + + +def test_batch_cost_calculator_gpt_6_astra_bills_half_the_standard_rate(_local_model_cost_map): + """gpt-6-astra batch pricing is 50% off the standard $10 input and $50 output rates per 1M tokens.""" + from litellm.cost_calculator import batch_cost_calculator + + usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + + prompt_cost, completion_cost = batch_cost_calculator( + usage=usage, model="gpt-6-astra", custom_llm_provider="openai" + ) + + assert prompt_cost == pytest.approx(1000 * 5e-6) + assert completion_cost == pytest.approx(500 * 2.5e-5) diff --git a/tests/test_litellm/test_gpt_5_5_model_metadata.py b/tests/test_litellm/test_gpt_5_5_model_metadata.py index 1c12a48ed9d..a60fa9466e6 100644 --- a/tests/test_litellm/test_gpt_5_5_model_metadata.py +++ b/tests/test_litellm/test_gpt_5_5_model_metadata.py @@ -47,8 +47,7 @@ def test_azure_ai_gpt_5_5_model_info(model): routed_model, provider, _, _ = get_llm_provider(model=model) assert routed_model == model.split("/", 1)[1] - # azure_ai/* models resolve under the azure provider in get_llm_provider - assert provider == "azure" + assert provider == "azure_ai" def test_azure_ai_gpt_5_5_backup_matches_main(): diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index e01679048e6..7682a6412b9 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -1133,6 +1133,82 @@ def test_responses_api_bridge_check_custom_api_base_via_env_with_unset_effort_st assert model_info.get("mode") != "responses" +@pytest.mark.parametrize( + "api_base", + [ + "https://southcentralus.privatelink.api.openai.com/v1", + "https://privatelink.corp.api.openai.com/v1", + "https://api.openai.com:443/v1", + "https://api.openai.com/v1/", + "HTTPS://API.OPENAI.COM/v1", + ], +) +def test_responses_api_bridge_check_openai_backed_custom_api_base_with_unset_effort_routes_to_responses(api_base): + """ + A custom api_base whose host is api.openai.com or a subdomain of it (a PrivateLink hostname, a + port-qualified or trailing-slash default) still reaches the real OpenAI backend, which rejects + function tools with reasoning on Chat Completions, so the unset-effort arm must bridge exactly as + it does for the literal default URL. Regression guard for GH #39353. + """ + from litellm.main import responses_api_bridge_check + + model_info, model = responses_api_bridge_check( + model="gpt-5.6", + custom_llm_provider="openai", + tools=[{"type": "function", "function": {"name": "get_capital"}}], + reasoning_effort=None, + api_base=api_base, + ) + + assert model == "gpt-5.6" + assert model_info.get("mode") == "responses" + + +@pytest.mark.parametrize( + "api_base", + [ + "https://api.openai.com.evil.example/v1", + "https://notapi.openai.com/v1", + "https://gateway.example/v1?upstream=api.openai.com", + "https://openai.internal.example/api.openai.com/v1", + ], +) +def test_responses_api_bridge_check_lookalike_custom_api_base_with_unset_effort_stays_chat(api_base): + """Only the host decides: api.openai.com appearing elsewhere in the URL is still a foreign backend.""" + from litellm.main import responses_api_bridge_check + + model_info, model = responses_api_bridge_check( + model="gpt-5.6", + custom_llm_provider="openai", + tools=[{"type": "function", "function": {"name": "get_capital"}}], + reasoning_effort=None, + api_base=api_base, + ) + + assert model == "gpt-5.6" + assert model_info.get("mode") != "responses" + + +def test_responses_api_bridge_check_privatelink_api_base_via_env_with_unset_effort_routes_to_responses(monkeypatch): + """A PrivateLink base set through OPENAI_BASE_URL resolves the way the chat handler's does and still bridges.""" + import litellm + from litellm.main import responses_api_bridge_check + + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setenv("OPENAI_BASE_URL", "https://southcentralus.privatelink.api.openai.com/v1") + model_info, model = responses_api_bridge_check( + model="gpt-5.6", + custom_llm_provider="openai", + tools=[{"type": "function", "function": {"name": "get_capital"}}], + reasoning_effort=None, + api_base=None, + ) + + assert model == "gpt-5.6" + assert model_info.get("mode") == "responses" + + def test_responses_api_bridge_check_custom_api_base_with_explicit_effort_still_routes(): """Explicit reasoning_effort keeps its pre-existing bridging behavior on any api_base.""" from litellm.main import responses_api_bridge_check @@ -3305,3 +3381,43 @@ def test_speech_mistral_routes_to_configured_api_base(respx_mock: respx.MockRout assert gateway_route.called assert response.content == audio_bytes + + +FOUNDRY_HOST: Final = "https://my-project.services.ai.azure.com" + + +def test_azure_ai_transcription_on_a_foundry_host_uses_the_azure_openai_deployment_route( + respx_mock: respx.MockRouter, +): + route: Final = respx_mock.post( + url__regex=r"https://my-project\.services\.ai\.azure\.com/openai/deployments/whisper-1/audio/transcriptions\?api-version=.+" + ).mock(return_value=httpx.Response(200, json={"text": "hello"})) + + response: Final = litellm.transcription( + model="azure_ai/whisper-1", + file=("tone.wav", b"RIFF\x00\x00\x00\x00WAVE", "audio/wav"), + api_base=FOUNDRY_HOST, + api_key="fake-key", + ) + + assert route.called + assert response.text == "hello" + + +def test_azure_ai_speech_on_a_foundry_host_uses_the_azure_openai_deployment_route( + respx_mock: respx.MockRouter, +): + route: Final = respx_mock.post( + url__regex=r"https://my-project\.services\.ai\.azure\.com/openai/deployments/tts-1/audio/speech\?api-version=.+" + ).mock(return_value=httpx.Response(200, content=b"mp3-bytes")) + + response: Final = litellm.speech( + model="azure_ai/tts-1", + input="hello", + voice="alloy", + api_base=FOUNDRY_HOST, + api_key="fake-key", + ) + + assert route.called + assert response.content == b"mp3-bytes" diff --git a/tests/test_litellm/test_openai_service_tier_long_context_pricing.py b/tests/test_litellm/test_openai_service_tier_long_context_pricing.py index c0860a5b55f..bdb2dc26813 100644 --- a/tests/test_litellm/test_openai_service_tier_long_context_pricing.py +++ b/tests/test_litellm/test_openai_service_tier_long_context_pricing.py @@ -52,6 +52,12 @@ PRIORITY_LONG_CONTEXT = { "cache_read_input_token_cost_above_272k_tokens_priority": 8e-08, "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-06, }, + "gpt-6-astra": { + "input_cost_per_token_above_272k_tokens_priority": 4e-05, + "output_cost_per_token_above_272k_tokens_priority": 0.00015, + "cache_read_input_token_cost_above_272k_tokens_priority": 4e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-05, + }, } EXPECTED = {**FLEX_LONG_CONTEXT, **PRIORITY_LONG_CONTEXT} @@ -63,6 +69,7 @@ NO_PUBLISHED_PRIORITY_LONG_CONTEXT = ("gpt-5.4", "gpt-5.5") def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + litellm.add_known_models() @lru_cache(maxsize=2) @@ -114,6 +121,7 @@ TIERED_COST_CASES = [ ("gpt-5.6-sol", "priority", 1.6e-05, 6e-05), ("gpt-5.6-terra", "priority", 8e-06, 3.6e-05), ("gpt-5.6-luna", "priority", 8e-07, 3.6e-06), + ("gpt-6-astra", "priority", 4e-05, 0.00015), ] diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 200cfd02197..99c69d6bc31 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -330,6 +330,37 @@ def test_get_optional_params_image_gen(): assert optional_params["n"] == 3 +@pytest.mark.parametrize("custom_llm_provider", ["openai", "azure"]) +def test_get_optional_params_image_gen_keeps_gpt_image_supported_params(custom_llm_provider): + """https://github.com/BerriAI/litellm/issues/38649""" + from litellm.types.utils import LlmProviders + + provider_config = ProviderConfigManager.get_provider_image_generation_config( + model="gpt-image-2", provider=LlmProviders(custom_llm_provider) + ) + optional_params = get_optional_params_image_gen( + model="gpt-image-2", + n=1, + size="1024x1024", + custom_llm_provider=custom_llm_provider, + provider_config=provider_config, + background="transparent", + output_format="png", + moderation="low", + output_compression=50, + unknown_param="kept-in-extra-body", + ) + assert optional_params == { + "n": 1, + "size": "1024x1024", + "background": "transparent", + "output_format": "png", + "moderation": "low", + "output_compression": 50, + "extra_body": {"unknown_param": "kept-in-extra-body"}, + } + + def test_get_optional_params_image_gen_vertex_ai_size(): """Test that Vertex AI image generation properly handles size parameter and maps it to aspectRatio""" # Test with various size parameters diff --git a/tests/test_litellm/types/test_types_utils.py b/tests/test_litellm/types/test_types_utils.py index c081b9e8e0d..554604ab200 100644 --- a/tests/test_litellm/types/test_types_utils.py +++ b/tests/test_litellm/types/test_types_utils.py @@ -759,3 +759,12 @@ def test_delta_function_tool_call_unchanged_by_custom_support(): delta = Delta(tool_calls=[{"index": 0, "id": "c2", "type": "function", "function": {"name": "g", "arguments": ""}}]) assert isinstance(delta.tool_calls[0], ChatCompletionDeltaToolCall) assert "custom" not in delta.model_dump()["tool_calls"][0] + + +def test_image_response_keeps_background(): + """https://github.com/BerriAI/litellm/issues/38649""" + from litellm.types.utils import ImageResponse + + response = ImageResponse(created=1, data=[{"b64_json": "aGk="}], background="transparent", output_format="png") + assert response.background == "transparent" + assert response.model_dump()["background"] == "transparent" diff --git a/ui/litellm-dashboard/src/components/common_components/OrganizationDropdown.test.tsx b/ui/litellm-dashboard/src/components/common_components/OrganizationDropdown.test.tsx index e0b2b897b36..524ec6a715e 100644 --- a/ui/litellm-dashboard/src/components/common_components/OrganizationDropdown.test.tsx +++ b/ui/litellm-dashboard/src/components/common_components/OrganizationDropdown.test.tsx @@ -58,6 +58,17 @@ describe("OrganizationDropdown", () => { expect(onChange.mock.calls[0][0]).toBe("org-1"); }); + it("emits null, never the empty string, when the selection is cleared", async () => { + const onChange = vi.fn(); + const user = userEvent.setup(); + render(); + + await user.click(screen.getByRole("button", { name: "Clear" })); + + expect(onChange).toHaveBeenCalledTimes(1); + expect(onChange).toHaveBeenCalledWith(null); + }); + it("should filter options by organization id", async () => { const user = userEvent.setup(); render(); diff --git a/ui/litellm-dashboard/src/components/common_components/OrganizationDropdown.tsx b/ui/litellm-dashboard/src/components/common_components/OrganizationDropdown.tsx index 8da35ecd02e..663028b2d92 100644 --- a/ui/litellm-dashboard/src/components/common_components/OrganizationDropdown.tsx +++ b/ui/litellm-dashboard/src/components/common_components/OrganizationDropdown.tsx @@ -5,7 +5,7 @@ import { Organization } from "../networking"; interface OrganizationDropdownProps { organizations?: Organization[] | null; value?: string; - onChange?: (value: string) => void; + onChange?: (value: string | null) => void; disabled?: boolean; loading?: boolean; style?: React.CSSProperties; @@ -32,7 +32,7 @@ const OrganizationDropdown: React.FC = ({ sublabel: org.organization_id, }))} value={value} - onValueChange={(organizationId) => onChange?.(organizationId)} + onValueChange={(organizationId) => onChange?.(organizationId || null)} placeholder={placeholder} emptyText={loading ? "Loading organizations…" : "No organizations found"} disabled={disabled} diff --git a/ui/litellm-dashboard/src/components/object_permissions_view.tsx b/ui/litellm-dashboard/src/components/object_permissions_view.tsx index 011791fa18d..e90645a9e9f 100644 --- a/ui/litellm-dashboard/src/components/object_permissions_view.tsx +++ b/ui/litellm-dashboard/src/components/object_permissions_view.tsx @@ -3,9 +3,12 @@ import VectorStorePermissions from "./permissions/VectorStorePermissions"; import MCPServerPermissions from "./permissions/MCPServerPermissions"; import AgentPermissions from "./permissions/AgentPermissions"; import type { ObjectPermission } from "./object_permission_types"; +import type { InheritedGrant } from "./permissions/inheritedGrants"; interface ObjectPermissionsViewProps { objectPermission?: ObjectPermission | null; + inheritedMcpServers?: InheritedGrant[]; + inheritedAgents?: InheritedGrant[]; variant?: "card" | "inline"; className?: string; accessToken?: string | null; @@ -13,6 +16,8 @@ interface ObjectPermissionsViewProps { export function ObjectPermissionsView({ objectPermission, + inheritedMcpServers = [], + inheritedAgents = [], variant = "card", className = "", accessToken, @@ -34,9 +39,15 @@ export function ObjectPermissionsView({ mcpAccessGroups={mcpAccessGroups} mcpToolPermissions={mcpToolPermissions} mcpToolsets={mcpToolsets} + inheritedMcpServers={inheritedMcpServers} + accessToken={accessToken} + /> + -

Search tools

{searchTools.length === 0 ? ( diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx index 5749541dcea..ee3a88acba0 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx @@ -587,9 +587,9 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp } }; - const changeOrganization = (write: FieldWrite) => (orgId: string) => { - write(orgId || undefined); - setSelectedOrganizationId(orgId || null); + const changeOrganization = (write: FieldWrite) => (orgId: string | null) => { + write(orgId ?? undefined); + setSelectedOrganizationId(orgId); // Clear team and project when org changes setSelectedCreateKeyTeam(null); setSelectedProjectId(null); diff --git a/ui/litellm-dashboard/src/components/permissions/AgentPermissions.test.tsx b/ui/litellm-dashboard/src/components/permissions/AgentPermissions.test.tsx new file mode 100644 index 00000000000..01c7b60b8a5 --- /dev/null +++ b/ui/litellm-dashboard/src/components/permissions/AgentPermissions.test.tsx @@ -0,0 +1,71 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import AgentPermissions from "./AgentPermissions"; +import * as networking from "../networking"; + +vi.mock("../networking"); + +describe("AgentPermissions", () => { + const accessToken = "test-token"; + const agentId = "90337622-756e-4f25-98f0-01fc8174aa24"; + + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("lists agents inherited from access groups, counts them, and names the groups on hover", async () => { + const user = userEvent.setup(); + vi.mocked(networking.getAgentsList).mockResolvedValue({ + agents: [{ agent_id: agentId, agent_name: "support_agent" }], + }); + + render( + , + ); + + const row = await screen.findByText(/support_agent/); + expect(screen.getByText("1")).toBeInTheDocument(); + expect(screen.queryByText("No agents or access groups configured")).not.toBeInTheDocument(); + expect(networking.getAgentsList).toHaveBeenCalledWith(accessToken); + + await user.hover(row); + expect( + await screen.findByText(`Granted via access groups platform-tools, support. Full ID: ${agentId}`), + ).toBeInTheDocument(); + }); + + it("does not double-list an agent that is both granted directly and inherited", async () => { + const user = userEvent.setup(); + vi.mocked(networking.getAgentsList).mockResolvedValue({ + agents: [{ agent_id: agentId, agent_name: "support_agent" }], + }); + + render( + , + ); + + const row = await screen.findByText(/support_agent/); + expect(screen.getAllByText(/support_agent/)).toHaveLength(1); + expect(screen.getByText("1")).toBeInTheDocument(); + + await user.hover(row); + expect(await screen.findByText(`Full ID: ${agentId}`)).toBeInTheDocument(); + }); + + it("shows the empty state when nothing is granted directly or inherited", () => { + render(); + + expect(screen.getByText("No agents or access groups configured")).toBeInTheDocument(); + expect(screen.getByText("0")).toBeInTheDocument(); + expect(networking.getAgentsList).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/permissions/AgentPermissions.tsx b/ui/litellm-dashboard/src/components/permissions/AgentPermissions.tsx index d84bd1834f8..d1ca25c975b 100644 --- a/ui/litellm-dashboard/src/components/permissions/AgentPermissions.tsx +++ b/ui/litellm-dashboard/src/components/permissions/AgentPermissions.tsx @@ -3,6 +3,7 @@ import { UserGroupIcon } from "@heroicons/react/outline"; import { Badge } from "@/components/ui/badge"; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; import { getAgentsList } from "../networking"; +import { InheritedGrant, inheritedGrantTooltip } from "./inheritedGrants"; interface Agent { agent_id: string; @@ -14,16 +15,24 @@ interface Agent { interface AgentPermissionsProps { agents: string[]; agentAccessGroups?: string[]; + inheritedAgents?: InheritedGrant[]; accessToken?: string | null; } -export function AgentPermissions({ agents, agentAccessGroups = [], accessToken }: AgentPermissionsProps) { +export function AgentPermissions({ + agents, + agentAccessGroups = [], + inheritedAgents = [], + accessToken, +}: AgentPermissionsProps) { const [agentDetails, setAgentDetails] = useState([]); + const inheritedOnlyAgents = inheritedAgents.filter((grant) => !agents.includes(grant.id)); + const agentIdCount = agents.length + inheritedOnlyAgents.length; // Fetch agent details when component mounts useEffect(() => { const fetchAgentDetails = async () => { - if (accessToken && agents.length > 0) { + if (accessToken && agentIdCount > 0) { try { const response = await getAgentsList(accessToken); if (response && response.agents && Array.isArray(response.agents)) { @@ -35,7 +44,7 @@ export function AgentPermissions({ agents, agentAccessGroups = [], accessToken } } }; fetchAgentDetails(); - }, [accessToken, agents.length]); + }, [accessToken, agentIdCount]); // Function to get display name for agent const getAgentDisplayName = (agentId: string) => { @@ -47,10 +56,10 @@ export function AgentPermissions({ agents, agentAccessGroups = [], accessToken } return agentId; }; - // Merge agents and access groups into one list const mergedItems = [ - ...agents.map((agent) => ({ type: "agent", value: agent })), - ...agentAccessGroups.map((group) => ({ type: "accessGroup", value: group })), + ...agents.map((agent) => ({ type: "agent", value: agent, tooltip: `Full ID: ${agent}` })), + ...inheritedOnlyAgents.map((grant) => ({ type: "agent", value: grant.id, tooltip: inheritedGrantTooltip(grant) })), + ...agentAccessGroups.map((group) => ({ type: "accessGroup", value: group, tooltip: "" })), ]; const totalCount = mergedItems.length; @@ -77,7 +86,7 @@ export function AgentPermissions({ agents, agentAccessGroups = [], accessToken } {getAgentDisplayName(item.value)} - {`Full ID: ${item.value}`} + {item.tooltip} ) : ( diff --git a/ui/litellm-dashboard/src/components/permissions/MCPServerPermissions.test.tsx b/ui/litellm-dashboard/src/components/permissions/MCPServerPermissions.test.tsx index c2df945f367..78ede9865a4 100644 --- a/ui/litellm-dashboard/src/components/permissions/MCPServerPermissions.test.tsx +++ b/ui/litellm-dashboard/src/components/permissions/MCPServerPermissions.test.tsx @@ -406,4 +406,55 @@ describe("MCPServerPermissions", () => { ); await waitFor(() => expect(screen.getByText("Blocked")).toHaveAttribute("data-variant", "destructive")); }); + + it("lists servers inherited from access groups, counts them, and names the group on hover", async () => { + const user = userEvent.setup(); + vi.mocked(networking.fetchMCPServers).mockResolvedValue([ + { server_id: mockServerId1, server_name: mockServerName1, alias: mockServerName1 }, + ]); + + render( + , + ); + + const row = await screen.findByText(/DW_MCP/); + expect(screen.getByText("1")).toBeInTheDocument(); + expect(screen.queryByText("No MCP servers, access groups, or toolsets configured")).not.toBeInTheDocument(); + expect(networking.fetchMCPServers).toHaveBeenCalledWith(mockAccessToken); + + await user.hover(row); + expect( + await screen.findByText(`Granted via access group platform-tools. Full ID: ${mockServerId1}`), + ).toBeInTheDocument(); + }); + + it("does not double-list a server that is both granted directly and inherited", async () => { + const user = userEvent.setup(); + vi.mocked(networking.fetchMCPServers).mockResolvedValue([ + { server_id: mockServerId2, server_name: mockServerName2, alias: mockServerName2 }, + ]); + + render( + , + ); + + const row = await screen.findByText(/Test Server/); + expect(screen.getAllByText(/Test Server/)).toHaveLength(1); + expect(screen.getByText("1")).toBeInTheDocument(); + + await user.hover(row); + expect(await screen.findByText(`Full ID: ${mockServerId2}`)).toBeInTheDocument(); + }); }); diff --git a/ui/litellm-dashboard/src/components/permissions/MCPServerPermissions.tsx b/ui/litellm-dashboard/src/components/permissions/MCPServerPermissions.tsx index f11bc6b9627..65f210addfa 100644 --- a/ui/litellm-dashboard/src/components/permissions/MCPServerPermissions.tsx +++ b/ui/litellm-dashboard/src/components/permissions/MCPServerPermissions.tsx @@ -5,12 +5,14 @@ import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip import { fetchMCPServers, fetchMCPToolsets } from "../networking"; import { MCPServer, MCPToolset } from "../mcp_tools/types"; import { ALL_PROXY_MCP_SERVERS_SENTINEL, NO_MCP_SERVERS_SENTINEL } from "../mcp_tools/constants"; +import { InheritedGrant, inheritedGrantTooltip } from "./inheritedGrants"; interface MCPServerPermissionsProps { mcpServers: string[]; mcpAccessGroups?: string[]; mcpToolPermissions?: Record; mcpToolsets?: string[]; + inheritedMcpServers?: InheritedGrant[]; accessToken?: string | null; } @@ -19,6 +21,7 @@ export function MCPServerPermissions({ mcpAccessGroups = [], mcpToolPermissions = {}, mcpToolsets = [], + inheritedMcpServers = [], accessToken, }: MCPServerPermissionsProps) { const [mcpServerDetails, setMCPServerDetails] = useState([]); @@ -50,10 +53,16 @@ export function MCPServerPermissions({ }); }; + const directServerIds = mcpServers.filter( + (server) => server !== NO_MCP_SERVERS_SENTINEL && server !== ALL_PROXY_MCP_SERVERS_SENTINEL, + ); + const inheritedOnlyServers = inheritedMcpServers.filter((grant) => !mcpServers.includes(grant.id)); + const serverIdCount = directServerIds.length + inheritedOnlyServers.length; + // Fetch MCP server details when component mounts useEffect(() => { const fetchMCPServerDetails = async () => { - if (accessToken && mcpServers.length > 0) { + if (accessToken && serverIdCount > 0) { try { const response = await fetchMCPServers(accessToken); if (response && Array.isArray(response)) { @@ -67,7 +76,7 @@ export function MCPServerPermissions({ } }; fetchMCPServerDetails(); - }, [accessToken, mcpServers.length]); + }, [accessToken, serverIdCount]); // Fetch toolset details useEffect(() => { @@ -98,12 +107,14 @@ export function MCPServerPermissions({ const blocksAllMcpServers = mcpServers.includes(NO_MCP_SERVERS_SENTINEL); const grantsAllProxyMcpServers = mcpServers.includes(ALL_PROXY_MCP_SERVERS_SENTINEL); - // Merge servers and access groups into one list const mergedItems = [ - ...mcpServers - .filter((server) => server !== NO_MCP_SERVERS_SENTINEL && server !== ALL_PROXY_MCP_SERVERS_SENTINEL) - .map((server) => ({ type: "server", value: server })), - ...mcpAccessGroups.map((group) => ({ type: "accessGroup", value: group })), + ...directServerIds.map((server) => ({ type: "server", value: server, tooltip: `Full ID: ${server}` })), + ...inheritedOnlyServers.map((grant) => ({ + type: "server", + value: grant.id, + tooltip: inheritedGrantTooltip(grant), + })), + ...mcpAccessGroups.map((group) => ({ type: "accessGroup", value: group, tooltip: "" })), ]; const totalCount = mergedItems.length + mcpToolsets.length; @@ -153,7 +164,7 @@ export function MCPServerPermissions({ {getMCPServerDisplayName(item.value)} - {`Full ID: ${item.value}`} + {item.tooltip} ) : (
diff --git a/ui/litellm-dashboard/src/components/permissions/inheritedGrants.test.ts b/ui/litellm-dashboard/src/components/permissions/inheritedGrants.test.ts new file mode 100644 index 00000000000..766f8f23f0b --- /dev/null +++ b/ui/litellm-dashboard/src/components/permissions/inheritedGrants.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from "vitest"; +import { computeInheritedGrants, inheritedGrantTooltip } from "./inheritedGrants"; +import { TeamAccessGroupModelGrant } from "../team/teamModelAccess"; + +const GRANTS: TeamAccessGroupModelGrant[] = [ + { access_group_id: "ag-1", access_group_name: "platform-tools", models: [], mcp_server_ids: ["mcp-1", "mcp-2"] }, + { + access_group_id: "ag-2", + access_group_name: "support", + models: [], + mcp_server_ids: ["mcp-2"], + agent_ids: ["agent-1"], + }, +]; + +describe("computeInheritedGrants", () => { + it("attributes each id to every group that grants it, in group order", () => { + expect(computeInheritedGrants(["mcp-1", "mcp-2"], GRANTS, (g) => g.mcp_server_ids)).toEqual([ + { id: "mcp-1", accessGroupNames: ["platform-tools"] }, + { id: "mcp-2", accessGroupNames: ["platform-tools", "support"] }, + ]); + }); + + it("keeps ids the flat list carries but no group detail explains, with no group names", () => { + expect(computeInheritedGrants(["agent-1", "agent-legacy"], GRANTS, (g) => g.agent_ids)).toEqual([ + { id: "agent-1", accessGroupNames: ["support"] }, + { id: "agent-legacy", accessGroupNames: [] }, + ]); + }); + + it("falls back to the group details when the flat list is missing, without duplicates", () => { + expect(computeInheritedGrants(undefined, GRANTS, (g) => g.mcp_server_ids).map((g) => g.id)).toEqual([ + "mcp-1", + "mcp-2", + ]); + }); + + it("returns nothing when neither source has ids", () => { + expect(computeInheritedGrants(undefined, undefined, (g) => g.agent_ids)).toEqual([]); + }); +}); + +describe("inheritedGrantTooltip", () => { + it("names a single group", () => { + expect(inheritedGrantTooltip({ id: "mcp-1", accessGroupNames: ["platform-tools"] })).toBe( + "Granted via access group platform-tools. Full ID: mcp-1", + ); + }); + + it("lists several groups", () => { + expect(inheritedGrantTooltip({ id: "mcp-2", accessGroupNames: ["platform-tools", "support"] })).toBe( + "Granted via access groups platform-tools, support. Full ID: mcp-2", + ); + }); + + it("stays generic when the proxy did not say which group granted it", () => { + expect(inheritedGrantTooltip({ id: "agent-legacy", accessGroupNames: [] })).toBe( + "Granted via an access group. Full ID: agent-legacy", + ); + }); +}); diff --git a/ui/litellm-dashboard/src/components/permissions/inheritedGrants.ts b/ui/litellm-dashboard/src/components/permissions/inheritedGrants.ts new file mode 100644 index 00000000000..fb78de5ed77 --- /dev/null +++ b/ui/litellm-dashboard/src/components/permissions/inheritedGrants.ts @@ -0,0 +1,26 @@ +import { describeGroups, TeamAccessGroupModelGrant } from "../team/teamModelAccess"; + +export interface InheritedGrant { + id: string; + accessGroupNames: string[]; +} + +export function computeInheritedGrants( + ids: string[] | undefined, + grants: TeamAccessGroupModelGrant[] | undefined, + idsOf: (grant: TeamAccessGroupModelGrant) => string[] | undefined, +): InheritedGrant[] { + const known = grants ?? []; + const allIds = [...new Set([...(ids ?? []), ...known.flatMap((grant) => idsOf(grant) ?? [])])]; + return allIds.map((id) => ({ + id, + accessGroupNames: known + .filter((grant) => (idsOf(grant) ?? []).includes(id)) + .map((grant) => grant.access_group_name), + })); +} + +export const inheritedGrantTooltip = (grant: InheritedGrant): string => { + const source = grant.accessGroupNames.length > 0 ? describeGroups(grant.accessGroupNames) : "an access group"; + return `Granted via ${source}. Full ID: ${grant.id}`; +}; diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx index ca1e0413dcb..a9c1077e96b 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx @@ -38,6 +38,9 @@ vi.mock("@/components/networking", () => ({ organizationInfoCall: vi.fn(), getRouterSettingsCall: vi.fn().mockResolvedValue({ fields: [] }), getPassThroughEndpointsCall: vi.fn(), + fetchMCPServers: vi.fn().mockResolvedValue([]), + fetchMCPToolsets: vi.fn().mockResolvedValue([]), + getAgentsList: vi.fn().mockResolvedValue({ agents: [] }), })); const can = vi.fn(); @@ -302,6 +305,47 @@ describe("TeamInfoView", () => { ); }); + it("shows MCP servers and agents inherited from access groups in the Object Permissions card, naming the group on hover", async () => { + const user = userEvent.setup(); + vi.mocked(networking.fetchMCPServers).mockResolvedValue([ + { server_id: "mcp-github-1234", server_name: "github", alias: "github" }, + ]); + vi.mocked(networking.getAgentsList).mockResolvedValue({ + agents: [{ agent_id: "agent-support-5678", agent_name: "support_agent" }], + }); + const platformToolsGroup = { + access_group_id: "ag-1", + access_group_name: "platform-tools", + models: [], + mcp_server_ids: ["mcp-github-1234"], + agent_ids: ["agent-support-5678"], + }; + const inheritedGrants = { + object_permission: null, + access_group_ids: ["ag-1"], + access_group_mcp_server_ids: ["mcp-github-1234"], + access_group_agent_ids: ["agent-support-5678"], + access_group_details: [platformToolsGroup], + }; + vi.mocked(networking.teamInfoCall).mockResolvedValue(createMockTeamData(inheritedGrants)); + + renderWithProviders(); + + const serverRow = await screen.findByText(/github \(mcp\.\.\.1234\)/); + const agentRow = await screen.findByText(/support_agent \(age\.\.\.5678\)/); + expect(screen.queryByText("No MCP servers, access groups, or toolsets configured")).not.toBeInTheDocument(); + expect(screen.queryByText("No agents or access groups configured")).not.toBeInTheDocument(); + + await user.hover(serverRow); + expect( + await screen.findByText("Granted via access group platform-tools. Full ID: mcp-github-1234"), + ).toBeInTheDocument(); + await user.hover(agentRow); + expect( + await screen.findByText("Granted via access group platform-tools. Full ID: agent-support-5678"), + ).toBeInTheDocument(); + }); + it("keeps the all-proxy-models badge non-clickable", async () => { vi.mocked(networking.teamInfoCall).mockResolvedValue(createMockTeamData({ models: ["all-proxy-models"] })); diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx index 44f0d420fd6..3f6d6a96972 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx @@ -58,6 +58,7 @@ import { TeamModelBadge, TeamModelBadgeKind, } from "./teamModelAccess"; +import { computeInheritedGrants } from "../permissions/inheritedGrants"; import MetadataKeyValueFields, { metadataObjectToPairs, metadataPairsSchema, @@ -936,6 +937,17 @@ const TeamInfoView: React.FC = ({ const { team_info: info } = teamData; + const inheritedMcpServers = computeInheritedGrants( + info.access_group_mcp_server_ids, + info.access_group_details, + (grant) => grant.mcp_server_ids, + ); + const inheritedAgents = computeInheritedGrants( + info.access_group_agent_ids, + info.access_group_details, + (grant) => grant.agent_ids, + ); + const initialKillSwitchOn = info.metadata?.disable_global_guardrails === true; const allGuardrails: GuardrailListItem[] = guardrailsData?.guardrails ?? []; @@ -1033,7 +1045,13 @@ const TeamInfoView: React.FC = ({
- + = ({ 0 ? models : [NO_DEFAULT_MODELS]; } -const describeGroups = (names: string[]): string => +export const describeGroups = (names: string[]): string => names.length > 1 ? `access groups ${names.join(", ")}` : `access group ${names[0]}`; export function computeTeamModelBadges( diff --git a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx index 3e772fd0e9b..1330be2788b 100644 --- a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx @@ -303,9 +303,9 @@ export function KeyEditView({ } }; - const handleOrganizationChange = (setField: (value: string | null) => void, orgId: string | undefined) => { - setField(orgId || null); - setSelectedOrganizationId(orgId || null); + const handleOrganizationChange = (setField: (value: string | null) => void, orgId: string | null) => { + setField(orgId); + setSelectedOrganizationId(orgId); form.setValue("team_id", undefined); }; diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx index e3bacc0908a..3c9e6543c1c 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx @@ -75,6 +75,41 @@ describe("Cost column", () => { }); }); +describe("Tokens column", () => { + const sessionRow: Partial = { + request_id: "req-session-tokens", + total_tokens: 10, + prompt_tokens: 7, + completion_tokens: 3, + session_id: "sess-1", + session_total_count: 3, + }; + + it("shows the summed session token usage, not the representative call's tokens, for a multi-round session", () => { + const aggregatedRow: Partial = { + ...sessionRow, + session_total_tokens: 60, + session_total_prompt_tokens: 42, + session_total_completion_tokens: 18, + }; + renderRows([logEntry(aggregatedRow)]); + + const tokensCell = screen.getByRole("cell", { name: /\(42\+18\)/ }); + expect(tokensCell).toHaveTextContent("60"); + expect(tokensCell).toHaveTextContent("session total"); + expect(screen.queryByText("10")).not.toBeInTheDocument(); + expect(screen.queryByText("(7+3)")).not.toBeInTheDocument(); + }); + + it("falls back to the call's own tokens with no session label when the backend sent no session token sums", () => { + renderRows([logEntry(sessionRow)]); + + const tokensCell = screen.getByRole("cell", { name: /\(7\+3\)/ }); + expect(tokensCell).toHaveTextContent("10"); + expect(tokensCell).not.toHaveTextContent("session total"); + }); +}); + describe("Type column", () => { it("shows the conversation badge and composition even when an MCP call represents the conversation", async () => { const user = userEvent.setup(); diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx index 8db0b106851..9d4dc4f7898 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx @@ -263,13 +263,20 @@ export const getRequestLogsTableColumns = ({ meta: { numeric: true }, cell: ({ row }) => { const log = row.original; + const showSessionTotal = (log.session_total_count || 1) > 1 && log.session_total_tokens != null; + const total = showSessionTotal ? log.session_total_tokens : log.total_tokens; + const prompt = showSessionTotal ? log.session_total_prompt_tokens : log.prompt_tokens; + const completion = showSessionTotal ? log.session_total_completion_tokens : log.completion_tokens; return ( - - {String(log.total_tokens || "0")} - - ({String(log.prompt_tokens || "0")}+{String(log.completion_tokens || "0")}) +
+ + {String(total || "0")} + + ({String(prompt || "0")}+{String(completion || "0")}) + - + {showSessionTotal && session total} +
); }, }, diff --git a/ui/litellm-dashboard/src/components/view_logs/columns.tsx b/ui/litellm-dashboard/src/components/view_logs/columns.tsx index 21e09faf454..b2e29c3a0c1 100644 --- a/ui/litellm-dashboard/src/components/view_logs/columns.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/columns.tsx @@ -42,6 +42,9 @@ export type LogEntry = { request_duration_ms?: number; session_total_count?: number; session_total_spend?: number; + session_total_tokens?: number; + session_total_prompt_tokens?: number; + session_total_completion_tokens?: number; session_cache_hit_count?: number; mcp_tool_call_count?: number; mcp_tool_call_spend?: number; diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 098b43f6433..5246aaf6904 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -25779,9 +25779,9 @@ export interface components { mcp_xff_num_trusted_hops?: number | null; /** * Missing Session Id - * @description What to do with LLM API requests that carry no session id (x-litellm-session-id header, metadata.session_id, etc.). 'generate' stamps one id into litellm_session_id, litellm_trace_id and metadata.session_id so SpendLogs and logging callbacks agree; 'reject' returns 400. Unset keeps the legacy behavior where SpendLogs falls back to the trace id while callbacks get no session id. + * @description What to do with LLM API requests that carry no session id (x-litellm-session-id header, metadata.session_id, etc.). 'generate' stamps one id into litellm_session_id, litellm_trace_id and metadata.session_id so SpendLogs and logging callbacks agree; 'reject' returns 400; 'omit' leaves SpendLogs.session_id null, matching callbacks such as Langfuse that only record a client-established metadata.session_id. Unset keeps the legacy behavior where SpendLogs falls back to the trace id while callbacks get no session id. */ - missing_session_id?: ("generate" | "reject") | null; + missing_session_id?: ("generate" | "reject" | "omit") | null; /** * Model List Healthy Only * @description When true, `/models`, `/v1/models/{id}` and `/model/info` hide models whose backing deployments are all unhealthy, for every caller, without needing `healthy_only=true` per request. Requires `background_health_checks: true`, and keeps deployment health state cached without turning on `enable_health_check_routing`, so routing is unaffected. With no health state nothing is hidden. Hiding is presentation-only, a hidden model can still be called. diff --git a/uv.lock b/uv.lock index de3181edd5a..dfa77c66dfe 100644 --- a/uv.lock +++ b/uv.lock @@ -10,7 +10,7 @@ resolution-markers = [ ] [options] -exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values. +exclude-newer = "2026-08-31T17:52:45.782441Z" exclude-newer-span = "P3D" [manifest] @@ -4765,12 +4765,12 @@ proxy-dev = [ [[package]] name = "litellm-enterprise" -version = "0.1.63" +version = "0.1.64" source = { editable = "enterprise" } [[package]] name = "litellm-proxy-extras" -version = "0.4.92" +version = "0.4.93" source = { editable = "litellm-proxy-extras" } [[package]]