diff --git a/README.md b/README.md index d600f3952c6..d7dc665dcec 100644 --- a/README.md +++ b/README.md @@ -327,6 +327,7 @@ curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \ | [Maritalk (`maritalk`)](https://docs.litellm.ai/docs/providers/maritalk) | ✅ | ✅ | ✅ | | | | | | | | | [Meta - Llama API (`meta_llama`)](https://docs.litellm.ai/docs/providers/meta_llama) | ✅ | ✅ | ✅ | | | | | | | | | [Mistral AI API (`mistral`)](https://docs.litellm.ai/docs/providers/mistral) | ✅ | ✅ | ✅ | ✅ | | | | | | | +| [ModelScope (`modelscope`)](https://docs.litellm.ai/docs/providers/modelscope) | ✅ | ✅ | ✅ | | ✅ | | | | | | | [Moonshot (`moonshot`)](https://docs.litellm.ai/docs/providers/moonshot) | ✅ | ✅ | ✅ | | | | | | | | | [Morph (`morph`)](https://docs.litellm.ai/docs/providers/morph) | ✅ | ✅ | ✅ | | | | | | | | | [Nebius AI Studio (`nebius`)](https://docs.litellm.ai/docs/providers/nebius) | ✅ | ✅ | ✅ | ✅ | | | | | | | diff --git a/litellm/__init__.py b/litellm/__init__.py index d5fbb41c462..0d6a788e368 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -73,6 +73,7 @@ from litellm.constants import ( replicate_models, clarifai_models, huggingface_models, + modelscope_models, empower_models, together_ai_models, baseten_models, @@ -900,6 +901,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None): heroku_models.add(key) elif value.get("litellm_provider") == "dashscope": dashscope_models.add(key) + elif value.get("litellm_provider") == "modelscope": + modelscope_models.add(key) elif value.get("litellm_provider") == "moonshot": moonshot_models.add(key) elif value.get("litellm_provider") == "publicai": @@ -1019,6 +1022,7 @@ model_list = list( | zai_models | fal_ai_models | deepseek_models + | modelscope_models | azure_ai_models | voyage_models | infinity_models @@ -1152,6 +1156,7 @@ models_by_provider: dict = { "elevenlabs": elevenlabs_models, "heroku": heroku_models, "dashscope": dashscope_models, + "modelscope": modelscope_models, "moonshot": moonshot_models, "publicai": publicai_models, "v0": v0_models, @@ -1975,6 +1980,9 @@ if TYPE_CHECKING: from .llms.dashscope.rerank.transformation import ( DashScopeRerankConfig as DashScopeRerankConfig, ) + from .llms.modelscope.chat.transformation import ( + ModelScopeChatConfig as ModelScopeChatConfig, + ) from .llms.moonshot.chat.transformation import ( MoonshotChatConfig as MoonshotChatConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 6073b6b2833..e653b40fd04 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -306,6 +306,7 @@ LLM_CONFIG_NAMES = ( "GigaChatConfig", "GigaChatEmbeddingConfig", "DashScopeChatConfig", + "ModelScopeChatConfig", "MoonshotChatConfig", "DockerModelRunnerChatConfig", "V0ChatConfig", @@ -1161,6 +1162,10 @@ _LLM_CONFIGS_IMPORT_MAP = { ".llms.dashscope.chat.transformation", "DashScopeChatConfig", ), + "ModelScopeChatConfig": ( + ".llms.modelscope.chat.transformation", + "ModelScopeChatConfig", + ), "MoonshotChatConfig": (".llms.moonshot.chat.transformation", "MoonshotChatConfig"), "DockerModelRunnerChatConfig": ( ".llms.docker_model_runner.chat.transformation", diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 6d8b5cf8a57..78826f822bc 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -1293,9 +1293,15 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): provider_specific_fields ) + from litellm.responses.litellm_completion_transformation.transformation import ( + LiteLLMCompletionResponsesConfig, + ) + tool_call_index = parsed_chunk.get("output_index", 0) tool_call_chunk = ChatCompletionToolCallChunk( - id=output_item.get("call_id"), + id=LiteLLMCompletionResponsesConfig._tool_call_id_from_responses_item( + output_item.get("id"), output_item.get("call_id") + ), index=tool_call_index, type="function", function=function_chunk, diff --git a/litellm/constants.py b/litellm/constants.py index b44c204432c..b51d15b6d25 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -622,6 +622,7 @@ LITELLM_CHAT_PROVIDERS = [ "nscale", "nebius", "dashscope", + "modelscope", "moonshot", "publicai", "v0", @@ -780,6 +781,7 @@ openai_compatible_endpoints: List = [ "inference.api.nscale.com/v1", "api.studio.nebius.ai/v1", "https://dashscope-intl.aliyuncs.com/compatible-mode/v1", + "https://api-inference.modelscope.cn/v1", "https://api.moonshot.ai/v1", "https://api.publicai.co/v1", "https://api.synthetic.new/openai/v1", @@ -797,6 +799,7 @@ openai_compatible_endpoints: List = [ "https://ai-gateway.vercel.sh/v1", "https://api.inference.wandb.ai/v1", "https://api.clarifai.com/v2/ext/openai/v1", + "https://api.libertai.io/v1", ] @@ -840,10 +843,12 @@ openai_compatible_providers: List = [ "poe", # Poe - JSON-configured provider "chutes", # Chutes - JSON-configured provider "parasail", # Parasail - JSON-configured provider + "libertai", # LibertAI - JSON-configured provider "featherless_ai", "nscale", "nebius", "dashscope", + "modelscope", "moonshot", "v0", "helicone", @@ -869,6 +874,7 @@ openai_text_completion_compatible_providers: List = ( "featherless_ai", "nebius", "dashscope", + "modelscope", "moonshot", "publicai", "synthetic", @@ -1129,6 +1135,48 @@ WANDB_MODELS: set = set( ] ) +modelscope_models: set = set( + [ + # Qwen series models + "Qwen/Qwen3-0.6B", + "Qwen/Qwen3-1.7B", + "Qwen/Qwen3-4B", + "Qwen/Qwen3-8B", + "Qwen/Qwen3-14B", + "Qwen/Qwen3-30B-A3B", + "Qwen/Qwen3-32B", + "Qwen/Qwen3-235B-A22B", + "Qwen/Qwen3-235B-A22B-Instruct-2507", + "Qwen/Qwen3-235B-A22B-Thinking-2507", + "Qwen/Qwen3-30B-A3B-Thinking-2507", + "Qwen/Qwen3-Coder-30B-A3B-Instruct", + "Qwen/Qwen3-Coder-480B-A35B-Instruct", + "Qwen/Qwen3-Next-80B-A3B-Instruct", + "Qwen/Qwen3-Next-80B-A3B-Thinking", + "Qwen/Qwen3-VL-235B-A22B-Instruct", + "Qwen/Qwen3-VL-8B-Instruct", + "Qwen/Qwen3-VL-8B-Thinking", + "Qwen/Qwen3.5-122B-A10B", + "Qwen/Qwen3.5-27B", + "Qwen/Qwen3.5-35B-A3B", + "Qwen/Qwen3.5-397B-A17B", + "Qwen/QwQ-32B", + "Qwen/QwQ-32B-Preview", + "Qwen/QVQ-72B-Preview", + "Qwen/Qwen-Image-Edit", + # DeepSeek series models + "deepseek-ai/DeepSeek-R1-0528", + "deepseek-ai/DeepSeek-R1-Distill-Llama-70B", + "deepseek-ai/DeepSeek-R1-Distill-Llama-8B", + "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B", + "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B", + "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", + "deepseek-ai/DeepSeek-V3.2", + "deepseek-ai/DeepSeek-V4-Flash", + ] +) + BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[ "cohere", "anthropic", diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 059658991fb..38245a2e5ba 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -621,6 +621,9 @@ class CustomGuardrail(CustomLogger): ): return False + if self.default_on is True and disable_global_guardrail is True: + return False + if self.default_on is True and disable_global_guardrail is not True: if self._event_hook_is_event_type(event_type): if isinstance(self.event_hook, Mode): diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index e984df82140..98b792efa59 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -242,9 +242,28 @@ def _get_parent_otel_span_from_kwargs( return None -def process_response_headers(response_headers: Union[httpx.Headers, dict]) -> dict: +def process_response_headers( + response_headers: Union[httpx.Headers, dict], + preserve_litellm_internal_headers: bool = False, +) -> dict: + """ + `preserve_litellm_internal_headers` must only be True when the input is a + LiteLLM-owned dict (e.g. `_hidden_params["additional_headers"]` that has + already been through one round of processing). For raw upstream provider + headers — whether passed as `httpx.Headers` or a plain dict — it must + remain False, otherwise a malicious provider returning `x-litellm-*` could + spoof LiteLLM-internal markers (e.g. `x-litellm-attempted-fallbacks`). + + When the input is an `httpx.Headers` object the flag is always treated as + False regardless of what the caller requested, because `httpx.Headers` is + always a raw provider response and can never be LiteLLM-owned. + """ from litellm.types.utils import OPENAI_RESPONSE_HEADERS + # Raw httpx.Headers objects come directly from provider HTTP responses and + # must never be treated as LiteLLM-owned, regardless of caller intent. + _preserve = preserve_litellm_internal_headers and isinstance(response_headers, dict) + openai_headers = {} processed_headers = {} additional_headers = {} @@ -256,6 +275,12 @@ def process_response_headers(response_headers: Union[httpx.Headers, dict]) -> di "llm_provider-" ): # return raw provider headers (incl. openai-compatible ones) processed_headers[k] = v + elif _preserve and k.startswith("x-litellm-"): + # LiteLLM's own internal headers (e.g. x-litellm-attempted-fallbacks, + # x-litellm-model-group) are not LLM provider headers and must not be + # prefixed. Downstream consumers (proxy override, callers checking + # whether a fallback happened) look up the bare key. + processed_headers[k] = v else: additional_headers["{}-{}".format("llm_provider", k)] = v diff --git a/litellm/litellm_core_utils/fallback_utils.py b/litellm/litellm_core_utils/fallback_utils.py index daacca85c8a..1606b53e1f9 100644 --- a/litellm/litellm_core_utils/fallback_utils.py +++ b/litellm/litellm_core_utils/fallback_utils.py @@ -7,6 +7,9 @@ from litellm.litellm_core_utils.core_helpers import ( safe_deep_copy, filter_internal_params, ) +from litellm.router_utils.add_retry_fallback_headers import ( + add_fallback_headers_to_response, +) from .asyncify import run_async_function @@ -42,7 +45,7 @@ async def async_completion_with_fallbacks(**kwargs): # Try each fallback model most_recent_exception_str: Optional[str] = None - for fallback in fallbacks: + for attempted_fallbacks, fallback in enumerate(fallbacks): try: completion_kwargs = safe_deep_copy(base_kwargs) # Handle dictionary fallback configurations @@ -63,7 +66,10 @@ async def async_completion_with_fallbacks(**kwargs): ) if response is not None: - return response + return add_fallback_headers_to_response( + response=response, + attempted_fallbacks=attempted_fallbacks, + ) except Exception as e: verbose_logger.exception( diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 5dc3f5c6868..80ee406b7e7 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -334,6 +334,9 @@ def get_llm_provider( # noqa: PLR0915 elif endpoint == "dashscope-intl.aliyuncs.com/compatible-mode/v1": custom_llm_provider = "dashscope" dynamic_api_key = get_secret_str("DASHSCOPE_API_KEY") + elif endpoint == "https://api-inference.modelscope.cn/v1": + custom_llm_provider = "modelscope" + dynamic_api_key = get_secret_str("MODELSCOPE_API_KEY") elif endpoint == "api.moonshot.ai/v1": custom_llm_provider = "moonshot" dynamic_api_key = get_secret_str("MOONSHOT_API_KEY") @@ -927,6 +930,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 ) = litellm.DashScopeChatConfig()._get_openai_compatible_provider_info( api_base, api_key ) + elif custom_llm_provider == "modelscope": + ( + api_base, + dynamic_api_key, + ) = litellm.ModelScopeChatConfig()._get_openai_compatible_provider_info( + api_base, api_key + ) elif custom_llm_provider == "moonshot": ( api_base, diff --git a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py index 06933a6fbcb..ba870eb9459 100644 --- a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py +++ b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py @@ -49,7 +49,8 @@ class ResponseMetadata: result=self.result, litellm_model_name=model, router_model_id=model_id ), "additional_headers": process_response_headers( - self._get_value_from_hidden_params("additional_headers") or {} + self._get_value_from_hidden_params("additional_headers") or {}, + preserve_litellm_internal_headers=True, ), "litellm_model_name": model, } diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index b495b183ec0..d51b937d434 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -604,6 +604,8 @@ class ChunkProcessor: usage_chunk = chunk._hidden_params.get("usage", None) if usage_chunk is not None: + if isinstance(usage_chunk, dict): + usage_chunk = Usage(**usage_chunk) usage_chunk_dict = self._usage_chunk_calculation_helper(usage_chunk) if ( usage_chunk_dict["prompt_tokens"] is not None diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 9ecd0df0cb8..e8c1e659e9f 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -2214,18 +2214,33 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if "inference_geo" in _usage and _usage["inference_geo"] is not None: inference_geo = _usage["inference_geo"] - if ( - "cache_creation_input_tokens" in _usage - and _usage["cache_creation_input_tokens"] is not None - ): - cache_creation_input_tokens = _usage["cache_creation_input_tokens"] - prompt_tokens += cache_creation_input_tokens - if ( - "cache_read_input_tokens" in _usage - and _usage["cache_read_input_tokens"] is not None - ): - cache_read_input_tokens = _usage["cache_read_input_tokens"] - prompt_tokens += cache_read_input_tokens + iterations: Optional[List[Any]] = _usage.get("iterations") + if iterations: + prompt_tokens = sum(it.get("input_tokens", 0) or 0 for it in iterations) + completion_tokens = sum( + it.get("output_tokens", 0) or 0 for it in iterations + ) + cache_creation_input_tokens = sum( + it.get("cache_creation_input_tokens", 0) or 0 for it in iterations + ) + cache_read_input_tokens = sum( + it.get("cache_read_input_tokens", 0) or 0 for it in iterations + ) + prompt_tokens += cache_creation_input_tokens + cache_read_input_tokens + + if not iterations: + if ( + "cache_creation_input_tokens" in _usage + and _usage["cache_creation_input_tokens"] is not None + ): + cache_creation_input_tokens = _usage["cache_creation_input_tokens"] + prompt_tokens += cache_creation_input_tokens + if ( + "cache_read_input_tokens" in _usage + and _usage["cache_read_input_tokens"] is not None + ): + cache_read_input_tokens = _usage["cache_read_input_tokens"] + prompt_tokens += cache_read_input_tokens if "server_tool_use" in _usage and _usage["server_tool_use"] is not None: if ( "web_search_requests" in _usage["server_tool_use"] @@ -2264,7 +2279,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ), ) - raw_input_tokens = usage_object.get("input_tokens", 0) or 0 + raw_input_tokens = ( + prompt_tokens - cache_read_input_tokens - cache_creation_input_tokens + ) prompt_tokens_details = PromptTokensDetailsWrapper( cached_tokens=cache_read_input_tokens, cache_creation_tokens=cache_creation_input_tokens, @@ -2296,6 +2313,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): cache_creation_input_tokens=cache_creation_input_tokens, cache_read_input_tokens=cache_read_input_tokens, completion_tokens_details=completion_token_details, + iterations=iterations, server_tool_use=( ServerToolUse( web_search_requests=web_search_requests, diff --git a/litellm/llms/base_llm/base_model_iterator.py b/litellm/llms/base_llm/base_model_iterator.py index bf1bfd06537..422ae947997 100644 --- a/litellm/llms/base_llm/base_model_iterator.py +++ b/litellm/llms/base_llm/base_model_iterator.py @@ -1,8 +1,11 @@ import json from abc import abstractmethod -from typing import List, Optional, Union, cast +from typing import TYPE_CHECKING, List, Optional, Union, cast import litellm + +if TYPE_CHECKING: + import httpx from litellm.types.utils import ( Choices, Delta, @@ -69,6 +72,18 @@ class BaseModelResponseIterator: self.streaming_response = streaming_response self.response_iterator = self.streaming_response self.json_mode = json_mode + self.http_response: Optional["httpx.Response"] = None + + async def aclose(self) -> None: + """Close the upstream HTTP response so the provider connection is + released (and a backend like vLLM aborts generation) when the stream + is abandoned before its natural end. + + ``streaming_response`` is usually a bare ``aiter_lines()`` generator + that holds no reference to the response, so the handler that owns the + response attaches it here after construction.""" + if self.http_response is not None: + await self.http_response.aclose() def chunk_parser( self, chunk: dict diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index c3f487997c3..5575385fb28 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -33,7 +33,10 @@ from litellm.llms.base_llm.anthropic_messages.transformation import ( from litellm.llms.base_llm.audio_transcription.transformation import ( BaseAudioTranscriptionConfig, ) -from litellm.llms.base_llm.base_model_iterator import MockResponseIterator +from litellm.llms.base_llm.base_model_iterator import ( + BaseModelResponseIterator, + MockResponseIterator, +) from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig from litellm.llms.base_llm.chat.transformation import BaseConfig from litellm.llms.base_llm.containers.transformation import BaseContainerConfig @@ -814,6 +817,8 @@ class BaseLLMHTTPHandler: completion_stream = provider_config.get_model_response_iterator( streaming_response=response.aiter_lines(), sync_stream=False ) + if isinstance(completion_stream, BaseModelResponseIterator): + completion_stream.http_response = response # LOGGING logging_obj.post_call( input=messages, diff --git a/litellm/llms/fastcrw/__init__.py b/litellm/llms/fastcrw/__init__.py new file mode 100644 index 00000000000..d65ed8d3fa1 --- /dev/null +++ b/litellm/llms/fastcrw/__init__.py @@ -0,0 +1,7 @@ +""" +fastCRW API integration module. +""" + +from litellm.llms.fastcrw.search.transformation import FastCRWSearchConfig + +__all__ = ["FastCRWSearchConfig"] diff --git a/litellm/llms/fastcrw/search/__init__.py b/litellm/llms/fastcrw/search/__init__.py new file mode 100644 index 00000000000..4f8023b2db4 --- /dev/null +++ b/litellm/llms/fastcrw/search/__init__.py @@ -0,0 +1,7 @@ +""" +fastCRW Search API module. +""" + +from litellm.llms.fastcrw.search.transformation import FastCRWSearchConfig + +__all__ = ["FastCRWSearchConfig"] diff --git a/litellm/llms/fastcrw/search/transformation.py b/litellm/llms/fastcrw/search/transformation.py new file mode 100644 index 00000000000..ce702266e7b --- /dev/null +++ b/litellm/llms/fastcrw/search/transformation.py @@ -0,0 +1,182 @@ +""" +Calls fastCRW's /v1/search endpoint to search the web. + +fastCRW is a Firecrawl-compatible web data engine (single Rust binary; self-host +or cloud). The search response uses the Firecrawl-compatible envelope +{ "success": true, "data": [ { "title", "url", "description", "markdown"? } ] }. + +fastCRW API Reference: https://fastcrw.com/docs/rest-api +""" + +from typing import Optional, TypedDict, Union + +import httpx + +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.search.transformation import ( + BaseSearchConfig, + SearchResponse, + SearchResult, +) +from litellm.secret_managers.main import get_secret_str + + +class _FastCRWSearchRequestRequired(TypedDict): + """Required fields for fastCRW Search API request.""" + + query: str # Required - search query + + +class FastCRWSearchRequest(_FastCRWSearchRequestRequired, total=False): + """ + fastCRW Search API request format. + Based on: https://fastcrw.com/docs/rest-api + """ + + limit: int # Optional - maximum number of results to return + sources: list[ + str + ] # Optional - sources to search ('web', 'images'), default ['web'] + scrapeOptions: dict # Optional - options for scraping search results + + +class FastCRWSearchConfig(BaseSearchConfig): + FASTCRW_API_BASE = "https://fastcrw.com/api/v1" + + @staticmethod + def ui_friendly_name() -> str: + return "fastCRW" + + def validate_environment( + self, + headers: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + **kwargs, + ) -> dict: + """ + Validate environment and return headers. + """ + api_key = api_key or get_secret_str("CRW_API_KEY") + if not api_key: + raise ValueError( + "CRW_API_KEY is not set. Set `CRW_API_KEY` environment variable." + ) + headers["Authorization"] = f"Bearer {api_key}" + headers["Content-Type"] = "application/json" + return headers + + def get_complete_url( + self, + api_base: Optional[str], + optional_params: dict, + data: Optional[Union[dict, list[dict]]] = None, + **kwargs, + ) -> str: + """ + Get complete URL for Search endpoint. + """ + api_base = api_base or get_secret_str("CRW_API_BASE") or self.FASTCRW_API_BASE + + # Append "/search" to the api base if it's not already there + if not api_base.endswith("/search"): + api_base = f"{api_base}/search" + + return api_base + + def transform_search_request( + self, + query: Union[str, list[str]], + optional_params: dict, + **kwargs, + ) -> dict: + """ + Transform Search request to fastCRW API format. + + Transforms Perplexity unified spec parameters: + - query -> query (same) + - max_results -> limit + + All other fastCRW-specific parameters are passed through as-is. + + Args: + query: Search query (string or list of strings). fastCRW only supports single string queries. + optional_params: Optional parameters for the request + + Returns: + Dict with typed request data following FastCRWSearchRequest spec + """ + if isinstance(query, list): + # fastCRW only supports single string queries, join with spaces + query = " ".join(query) + + request_data: FastCRWSearchRequest = { + "query": query, + } + + # Transform Perplexity unified spec parameters to fastCRW format + if "max_results" in optional_params: + request_data["limit"] = optional_params["max_results"] + + # Convert to dict before dynamic key assignments + result_data = dict(request_data) + + # pass through all other parameters as-is + for param, value in optional_params.items(): + if ( + param not in self.get_supported_perplexity_optional_params() + and param not in result_data + ): + result_data[param] = value + + # By default, request markdown content if not explicitly specified + # fastCRW doesn't return content unless explicitly requested via scrapeOptions + if "scrapeOptions" not in result_data: + result_data["scrapeOptions"] = { + "formats": ["markdown"], + "onlyMainContent": True, + } + + return result_data + + def transform_search_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + **kwargs, + ) -> SearchResponse: + """ + Transform fastCRW API response to LiteLLM unified SearchResponse format. + + fastCRW (Firecrawl-compatible) returns: + {"success": true, "data": [{"url": "...", "title": "...", "description": "...", "markdown"?: "..."}, ...]} + + Args: + raw_response: Raw httpx response from fastCRW API + logging_obj: Logging object for tracking + + Returns: + SearchResponse with standardized format + """ + response_json = raw_response.json() + + results = [] + + data = response_json.get("data", []) + + if isinstance(data, list): + for result in data: + snippet = result.get("markdown") or result.get("description", "") + search_result = SearchResult( + title=result.get("title", ""), + url=result.get("url", ""), + snippet=snippet, + date=None, + last_updated=None, + ) + results.append(search_result) + + return SearchResponse( + results=results, + object="search", + ) diff --git a/litellm/llms/modelscope/chat/transformation.py b/litellm/llms/modelscope/chat/transformation.py new file mode 100644 index 00000000000..162ef1a236c --- /dev/null +++ b/litellm/llms/modelscope/chat/transformation.py @@ -0,0 +1,93 @@ +""" +Translates from OpenAI's `/v1/chat/completions` to ModelScope's `/v1/chat/completions` +""" + +from typing import Any, Coroutine, Literal, Optional, Tuple, Union, cast, overload + +from typing_extensions import override + +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import AllMessageValues + +from ...openai.chat.gpt_transformation import OpenAIGPTConfig + + +def _has_non_text_content(message: AllMessageValues) -> bool: + """Check if a message has non-text content items (e.g. image_url).""" + content = message.get("content") + if not isinstance(content, list): + return False + return any(item.get("type") != "text" for item in content) + + +class ModelScopeChatConfig(OpenAIGPTConfig): + DEFAULT_BASE_URL: str = "https://api-inference.modelscope.cn/v1" + + @overload + def _transform_messages( + self, messages: list[AllMessageValues], model: str, is_async: Literal[True] + ) -> Coroutine[Any, Any, list[AllMessageValues]]: ... + + @overload + def _transform_messages( + self, + messages: list[AllMessageValues], + model: str, + is_async: Literal[False] = False, + ) -> list[AllMessageValues]: ... + + def _transform_messages( + self, messages: list[AllMessageValues], model: str, is_async: bool = False + ) -> Union[list[AllMessageValues], Coroutine[Any, Any, list[AllMessageValues]]]: + """ + Flatten text-only content lists to strings for ModelScope. + + Messages with non-text content (e.g. image_url for vision models) + are kept as lists so the parent class can normalize them properly. + """ + messages = [cast(AllMessageValues, {**m}) for m in messages] + for message in messages: + if _has_non_text_content(message): + continue + content = message.get("content") + if isinstance(content, list): + message["content"] = "".join(item.get("text") or "" for item in content) + + if is_async: + return super()._transform_messages( + messages=messages, model=model, is_async=True + ) + else: + return super()._transform_messages( + messages=messages, model=model, is_async=False + ) + + def _get_openai_compatible_provider_info( + self, api_base: Optional[str], api_key: Optional[str] + ) -> Tuple[Optional[str], Optional[str]]: + api_base = ( + api_base or get_secret_str("MODELSCOPE_API_BASE") or self.DEFAULT_BASE_URL + ) # type: ignore + dynamic_api_key = api_key or get_secret_str("MODELSCOPE_API_KEY") + return api_base, dynamic_api_key + + @override + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + If api_base is not provided, use the default ModelScope /chat/completions endpoint. + """ + if not api_base: + api_base = self.DEFAULT_BASE_URL + + if not api_base.endswith("/chat/completions"): + api_base = f"{api_base}/chat/completions" + + return api_base diff --git a/litellm/llms/modelscope/image_generation/__init__.py b/litellm/llms/modelscope/image_generation/__init__.py new file mode 100644 index 00000000000..8b28ea962ce --- /dev/null +++ b/litellm/llms/modelscope/image_generation/__init__.py @@ -0,0 +1,31 @@ +""" +ModelScope Image Generation Module + +Factory function for getting the appropriate config class. +""" + +from litellm.llms.base_llm.image_generation.transformation import ( + BaseImageGenerationConfig, +) + +from .transformation import ModelScopeImageGenerationConfig + +__all__ = [ + "ModelScopeImageGenerationConfig", + "get_modelscope_image_generation_config", +] + + +def get_modelscope_image_generation_config( + model: str, +) -> BaseImageGenerationConfig: + """ + Get the ModelScope config for image generation. + + Args: + model: The model name (e.g., "modelscope/Qwen/Qwen-Image-Edit") + + Returns: + BaseImageGenerationConfig instance for ModelScope + """ + return ModelScopeImageGenerationConfig() diff --git a/litellm/llms/modelscope/image_generation/transformation.py b/litellm/llms/modelscope/image_generation/transformation.py new file mode 100644 index 00000000000..0d85f7796fb --- /dev/null +++ b/litellm/llms/modelscope/image_generation/transformation.py @@ -0,0 +1,248 @@ +""" +ModelScope Image Generation Config + +Handles transformation between OpenAI-compatible format and ModelScope API format. + +API Reference: https://modelscope.cn/docs/model-service/API-Inference/intro +""" + +from typing import TYPE_CHECKING, Optional, Union + +import httpx +from typing_extensions import override + +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.image_generation.transformation import ( + BaseImageGenerationConfig, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import ( + AllMessageValues, + OpenAIImageGenerationOptionalParams, +) +from litellm.types.utils import ImageObject, ImageResponse + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = object + + +class ModelScopeImageGenerationConfig(BaseImageGenerationConfig): + """ + Configuration for ModelScope image generation. + + Supports text-to-image models like: + - Qwen/Qwen-Image-Edit + - And other ModelScope-hosted image generation models + """ + + DEFAULT_BASE_URL: str = "https://api-inference.modelscope.cn/v1" + + def get_supported_openai_params( + self, model: str + ) -> list[OpenAIImageGenerationOptionalParams]: + """ + Return list of OpenAI params supported by ModelScope. + + ModelScope supports standard OpenAI image generation parameters. + """ + return [ + "n", # Number of images to generate + "size", # Size of the generated images + "response_format", # url or b64_json + "user", # User identifier + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + """ + Map OpenAI parameters to ModelScope parameters. + + ModelScope uses the same parameter names as OpenAI. + """ + supported_params = self.get_supported_openai_params(model) + if drop_params: + non_default_params = { + k: v for k, v in non_default_params.items() if k in supported_params + } + optional_params.update(non_default_params) + return optional_params + + @override + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + Get the complete URL for the ModelScope image generation API request. + """ + base_url: str = ( + api_base or get_secret_str("MODELSCOPE_API_BASE") or self.DEFAULT_BASE_URL + ) + base_url = base_url.rstrip("/") + + # Return the images endpoint + return f"{base_url}/images/generations" + + @override + def validate_environment( + self, + headers: dict, + model: str, + messages: list[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + """ + Validate environment and set up headers for ModelScope. + """ + final_api_key: Optional[str] = api_key or get_secret_str("MODELSCOPE_API_KEY") + + if not final_api_key: + raise ValueError( + "MODELSCOPE_API_KEY is not set. " + "Please set it via environment variable or pass api_key parameter." + ) + + default_headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {final_api_key}", + } + + headers = {**headers, **default_headers} + return headers + + def transform_image_generation_request( + self, + model: str, + prompt: str, + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + """ + Transform OpenAI-style request to ModelScope request format. + + ModelScope uses the same format as OpenAI for image generation. + """ + # Build the request body (same as OpenAI) + request_data: dict = { + "model": model, + "prompt": prompt, + } + + # Add optional params + for key, value in optional_params.items(): + if key.startswith("_"): + continue + request_data[key] = value + + return request_data + + @override + def transform_image_generation_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ImageResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + optional_params: dict, + litellm_params: dict, + encoding: object, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ImageResponse: + """ + Transform ModelScope response to OpenAI-compatible ImageResponse. + + ModelScope returns the same format as OpenAI: + {"created": timestamp, "data": [{"url": "..."}]} + """ + try: + response_data = raw_response.json() + except Exception as e: + raise self.get_error_class( + error_message=f"Error parsing ModelScope response: {e}", + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + + # Check for errors in response + if "error" in response_data: + error_msg = response_data["error"].get( + "message", str(response_data["error"]) + ) + raise self.get_error_class( + error_message=f"ModelScope error: {error_msg}", + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + + # Extract images from response + data_list = response_data.get("data", []) + if not model_response.data: + model_response.data = [] + + for item in data_list: + image_obj = ImageObject( + url=item.get("url"), + b64_json=item.get("b64_json"), + revised_prompt=item.get("revised_prompt"), + ) + model_response.data.append(image_obj) + + return model_response + + def get_error_class( + self, + error_message: str, + status_code: int, + headers: Union[dict, httpx.Headers], + ) -> BaseLLMException: + """Return the appropriate error class for ModelScope.""" + from litellm.exceptions import ( + AuthenticationError, + BadRequestError, + InternalServerError, + ) + + if status_code == 400: + return BadRequestError( # type: ignore[return-value] + message=error_message, + model="", + llm_provider="modelscope", + ) + elif status_code == 401: + return AuthenticationError( # type: ignore[return-value] + message=error_message, + model="", + llm_provider="modelscope", + ) + elif status_code >= 500: + return InternalServerError( # type: ignore[return-value] + message=error_message, + model="", + llm_provider="modelscope", + ) + else: + return BadRequestError( # type: ignore[return-value] + message=error_message, + model="", + llm_provider="modelscope", + ) diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index 303e9ba8f9e..0dda047d1ca 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -143,6 +143,14 @@ "force_store_false": true } }, + "libertai": { + "base_url": "https://api.libertai.io/v1", + "api_key_env": "LIBERTAI_API_KEY", + "api_base_env": "LIBERTAI_API_BASE", + "param_mappings": { + "max_completion_tokens": "max_tokens" + } + }, "empiriolabs": { "base_url": "https://api.empiriolabs.ai/v1", "api_key_env": "EMPIRIOLABS_API_KEY", diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 76a7c0640af..f563ad0c5b5 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -18822,6 +18822,38 @@ "supports_response_schema": true, "supports_vision": true }, + "github_copilot/mai-code-1-flash": { + "cache_read_input_token_cost": 7.5e-08, + "input_cost_per_token": 7.5e-07, + "litellm_provider": "github_copilot", + "max_input_tokens": 128000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 4.5e-06, + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true + }, + "github_copilot/mai-code-1-flash-internal": { + "cache_read_input_token_cost": 7.5e-08, + "input_cost_per_token": 7.5e-07, + "litellm_provider": "github_copilot", + "max_input_tokens": 128000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 4.5e-06, + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true + }, "github_copilot/text-embedding-3-small": { "litellm_provider": "github_copilot", "max_input_tokens": 8191, @@ -40784,6 +40816,174 @@ "litellm_provider": "llamagate", "mode": "embedding" }, + "libertai/hermes-3-8b-tee": { + "max_tokens": 16000, + "max_input_tokens": 16000, + "max_output_tokens": 16000, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "libertai", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_vision": false, + "source": "https://docs.libertai.io/apis/text/" + }, + "libertai/gemma-4-31b-it": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 4e-07, + "litellm_provider": "libertai", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_vision": true, + "source": "https://docs.libertai.io/apis/text/" + }, + "libertai/gemma-4-31b-it-thinking": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 4e-07, + "litellm_provider": "libertai", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "source": "https://docs.libertai.io/apis/text/" + }, + "libertai/qwen3.6-27b": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 5e-07, + "litellm_provider": "libertai", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_vision": true, + "source": "https://docs.libertai.io/apis/text/" + }, + "libertai/qwen3.6-27b-thinking": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 5e-07, + "litellm_provider": "libertai", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "source": "https://docs.libertai.io/apis/text/" + }, + "libertai/qwen3.6-35b-a3b": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 5e-07, + "litellm_provider": "libertai", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_vision": true, + "source": "https://docs.libertai.io/apis/text/" + }, + "libertai/qwen3.6-35b-a3b-thinking": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 5e-07, + "litellm_provider": "libertai", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "source": "https://docs.libertai.io/apis/text/" + }, + "libertai/qwen3.5-122b-a10b": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 1.75e-06, + "litellm_provider": "libertai", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_vision": true, + "source": "https://docs.libertai.io/apis/text/" + }, + "libertai/qwen3.5-122b-a10b-thinking": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 1.75e-06, + "litellm_provider": "libertai", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "source": "https://docs.libertai.io/apis/text/" + }, + "libertai/deepseek-v4-flash": { + "max_tokens": 200000, + "max_input_tokens": 200000, + "max_output_tokens": 200000, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 1.75e-06, + "litellm_provider": "libertai", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_vision": false, + "source": "https://docs.libertai.io/apis/text/" + }, + "libertai/deepseek-v4-flash-thinking": { + "max_tokens": 200000, + "max_input_tokens": 200000, + "max_output_tokens": 200000, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 1.75e-06, + "litellm_provider": "libertai", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_vision": false, + "supports_reasoning": true, + "source": "https://docs.libertai.io/apis/text/" + }, + "libertai/bge-m3": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "input_cost_per_token": 1e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "libertai", + "mode": "embedding", + "source": "https://docs.libertai.io/apis/text/" + }, "sarvam/sarvam-m": { "cache_creation_input_token_cost": 0, "cache_creation_input_token_cost_above_1hr": 0, diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index e0eeb014c51..db6183edaa0 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -1288,6 +1288,23 @@ "interactions": true } }, + "libertai": { + "display_name": "LibertAI (`libertai`)", + "url": "https://docs.litellm.ai/docs/providers/libertai", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, "litellm_proxy": { "display_name": "LiteLLM Proxy (`litellm_proxy`)", "url": "https://docs.litellm.ai/docs/providers/litellm_proxy", diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 90ad0f28808..8f330ada7d3 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -17,10 +17,12 @@ from typing import ( Union, ) +import anyio import httpx import orjson from fastapi import HTTPException, Request, status from fastapi.responses import JSONResponse, Response, StreamingResponse +from starlette.types import Receive, Scope, Send import litellm from litellm._logging import _redact_string, verbose_proxy_logger @@ -240,6 +242,64 @@ def _extract_error_from_sse_chunk(event_line: Union[str, bytes]) -> dict: return default_error +async def _aclose_upstream_response(response: Any) -> None: + """Release the upstream HTTP connection when a stream ends for any + reason, including client disconnect. Mirrors the finally block of + async_data_generator in proxy_server.py.""" + with anyio.CancelScope(shield=True): + if hasattr(response, "aclose"): + try: + await response.aclose() + except BaseException as e: + verbose_proxy_logger.debug( + "error closing upstream response stream: %s", e + ) + + +class _UpstreamClosingStreamingResponse(StreamingResponse): + """StreamingResponse that always closes its body iterator and the wrapped + upstream generator. + + When the client disconnects mid-stream, Starlette abandons the body + iterator without calling aclose(), leaving the upstream LLM connection + open until garbage collection; the backend (e.g. vLLM) keeps generating + into a dead pipe. The upstream generator is closed directly (not via the + body iterator) because aclose() on a never-started generator skips its + body, so a cascade through it would be a no-op if the client disconnects + before the first chunk is sent. + """ + + def __init__( + self, + content: AsyncGenerator[str, None], + *, + media_type: Optional[str] = None, + headers: Optional[dict] = None, + status_code: int = status.HTTP_200_OK, + upstream_generator: Optional[AsyncGenerator[str, None]] = None, + ) -> None: + super().__init__( + content, status_code=status_code, headers=headers, media_type=media_type + ) + self._upstream_generator = upstream_generator + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + try: + await super().__call__(scope, receive, send) + finally: + with anyio.CancelScope(shield=True): + for target in (self.body_iterator, self._upstream_generator): + aclose = getattr(target, "aclose", None) + if aclose is None: + continue + try: + await aclose() + except BaseException as e: + verbose_proxy_logger.debug( + "error closing streaming generator: %s", e + ) + + async def create_response( # noqa: PLR0915 generator: AsyncGenerator[str, None], media_type: str, @@ -366,11 +426,12 @@ async def create_response( # noqa: PLR0915 with tracer.trace(DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE): yield chunk - return StreamingResponse( + return _UpstreamClosingStreamingResponse( combined_generator(), media_type=media_type, headers=streaming_headers, status_code=final_status_code, + upstream_generator=generator, ) @@ -1702,6 +1763,23 @@ class ProxyBaseLLMRequestProcessing: response=completed_obj, user_api_key_dict=user_api_key_dict, ) + else: + # Silent skip caused #30210: the proxy's Router wrapper + # of the responses streaming iterator wasn't propagating + # ``completed_response``, so this hook recorded nothing + # and follow-up /v1/containers//files calls 403'd + # for non-admin keys with no proxy-side hint. Log a + # warning so future regressions of the same shape + # surface in operator logs. + verbose_proxy_logger.warning( + "Container ownership recording skipped on streaming " + "/v1/responses: no completed_response on stream " + "iterator %s. If this stream created any tool " + "container (e.g. code_interpreter), follow-up " + "/v1/containers//files calls will 403 for " + "non-admin keys.", + type(original_stream_response).__name__, + ) except Exception as e: verbose_proxy_logger.exception( "Container ownership recording failed after streaming responses call: %s", @@ -2424,6 +2502,8 @@ class ProxyBaseLLMRequestProcessing: code=getattr(e, "status_code", 500), ) yield serialize_error(proxy_exception) + finally: + await _aclose_upstream_response(response) @staticmethod def async_sse_data_generator( diff --git a/litellm/proxy/management_endpoints/customer_endpoints.py b/litellm/proxy/management_endpoints/customer_endpoints.py index f1a34bb0ed4..50a1bc23a6d 100644 --- a/litellm/proxy/management_endpoints/customer_endpoints.py +++ b/litellm/proxy/management_endpoints/customer_endpoints.py @@ -885,6 +885,19 @@ async def get_customer_daily_activity( """ Get daily activity for specific organizations or all accessible organizations. """ + if ( + user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN + and user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY + ): + raise HTTPException( + status_code=401, + detail={ + "error": "Admin-only endpoint. Your user role={}".format( + user_api_key_dict.user_role + ) + }, + ) + from litellm.proxy.proxy_server import prisma_client if prisma_client is None: diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 1a0a57c71fd..43a36fcb2f1 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -4244,6 +4244,14 @@ async def _enforce_list_team_v2_access( status_code=403, detail={"error": "You can only view teams within your organizations."}, ) + # When the caller is an org admin querying their own teams (or no + # specific user), null out user_id so that + # _build_team_list_where_conditions scopes only by organization_id + # — org admins should see all teams in their orgs, not just teams + # they are a direct member of. Keep user_id when the org admin + # explicitly queries a *different* user's teams. + if user_id is None or user_id == user_api_key_dict.user_id: + user_id = None verbose_proxy_logger.debug( "list_team_v2: org admin access for user=%s, org_ids=%s, user_id_filter=%s", user_api_key_dict.user_id, diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index d3d30642216..5b5ff122c50 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -2,6 +2,7 @@ Handles transforming from Responses API -> LiteLLM completion (Chat Completion API) """ +import re from collections.abc import Sequence from typing import Any, Dict, List, Literal, Optional, Set, Tuple, Union, cast @@ -1554,6 +1555,20 @@ class LiteLLMCompletionResponsesConfig: # Default to completed for unknown finish reasons return "completed" + @staticmethod + def _tool_call_id_from_responses_item( + item_id: Optional[str], call_id: Optional[str] + ) -> str: + """Bedrock Mantle returns a non-unique, index-based ``call_id`` (``call_0``, + ``call_1``, ... that resets every response) alongside a unique ``id`` + (``fc_...``). ``call_id`` is the canonical Responses API correlation key, so + prefer it; fall back to the unique ``id`` only when ``call_id`` is absent or + in that degenerate index form, otherwise multi-turn tool calls collide and an + agent cannot correlate its tool results.""" + if call_id and re.fullmatch(r"call_\d+", call_id) is None: + return call_id + return item_id or call_id or "" + @staticmethod def convert_response_function_tool_call_to_chat_completion_tool_call( tool_call_item: Any, @@ -1601,7 +1616,10 @@ class LiteLLMCompletionResponsesConfig: function_dict["provider_specific_fields"] = provider_specific_fields tool_call_dict: Dict[str, Any] = { - "id": tool_call_item.call_id, + "id": LiteLLMCompletionResponsesConfig._tool_call_id_from_responses_item( + getattr(tool_call_item, "id", None), + getattr(tool_call_item, "call_id", None), + ), "function": function_dict, "type": "function", "index": 0, diff --git a/litellm/router.py b/litellm/router.py index 80584858311..148c8cf5771 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2521,10 +2521,22 @@ class Router: from litellm.exceptions import MidStreamFallbackError from litellm.responses.streaming_iterator import ( BaseResponsesAPIStreamingIterator, + _get_openai_response_types, ) source_iterator = response + # Pre-resolve the set of terminal stream event types so the + # per-chunk type check inside FallbackResponsesStreamWrapper + # stays cheap; mirrors the source-iterator filter at + # responses/streaming_iterator.py:243-247. + _openai_types = _get_openai_response_types() + _RESPONSES_TERMINAL_EVENT_TYPES = ( + _openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + _openai_types.ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, + _openai_types.ResponsesAPIStreamEvents.RESPONSE_FAILED, + ) + class FallbackResponsesStreamWrapper(BaseResponsesAPIStreamingIterator): """ Subclasses BaseResponsesAPIStreamingIterator only for isinstance @@ -2550,9 +2562,16 @@ class Router: # is missing many of these attributes — use getattr fallbacks # so wrapper construction never raises AttributeError. The # bridge stores the logging object as `litellm_logging_obj`. - self.response = getattr(source_iterator, "response", None) - self.model = getattr(source_iterator, "model", None) - self.logging_obj = getattr( + # base class declares non-Optional types for these + # fields but the bridge path (LiteLLMCompletionStreamingIterator) + # can legitimately omit them at runtime — keep the None + # fallback. Same lines passed mypy on the pre-fix file + # because the surrounding function body wasn't fully + # type-narrowed; the new typed terminal-event tuple above + # is what made these surface. + self.response = getattr(source_iterator, "response", None) # type: ignore[assignment] + self.model = getattr(source_iterator, "model", None) # type: ignore[assignment] + self.logging_obj = getattr( # type: ignore[assignment] source_iterator, "logging_obj", getattr(source_iterator, "litellm_logging_obj", None), @@ -2587,7 +2606,23 @@ class Router: return self async def __anext__(self): - return await self._async_generator.__anext__() + chunk = await self._async_generator.__anext__() + # Sniff the terminal stream event off each forwarded chunk + # so ``self.completed_response`` is populated regardless of + # which inner iterator produced it (source_iterator, + # fallback_iterator, or any future wrapper). Without this + # the proxy's container-ownership hook (which reads + # ``getattr(stream_response, "completed_response", None)`` + # via _extract_completed_responses_response) silently + # records nothing on streaming /v1/responses calls — every + # follow-up /v1/containers//files call then 403s for + # the very key that created the container (#30210). + if ( + self.completed_response is None + and getattr(chunk, "type", None) in _RESPONSES_TERMINAL_EVENT_TYPES + ): + self.completed_response = chunk + return chunk async def aclose(self): # async generators always expose aclose — no defensive check needed. diff --git a/litellm/types/router.py b/litellm/types/router.py index 5047cee424b..1611f1e5538 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -166,6 +166,13 @@ class CredentialLiteLLMParams(BaseModel): api_key: Optional[str] = None api_base: Optional[str] = None api_version: Optional[str] = None + ## AZURE OAUTH ## + # Without this field, ``get_deployment_credentials_with_provider`` + # round-trips ``litellm_params`` through a strict Pydantic dump and + # silently drops the OAuth token before the files/batch/passthrough + # callers see it, breaking Azure deployments configured with + # ``azure_ad_token`` instead of a static ``api_key`` (#30235). + azure_ad_token: Optional[str] = None ## VERTEX AI ## vertex_project: Optional[str] = None vertex_location: Optional[str] = None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index d3dc7eadb94..a5032942011 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3338,6 +3338,7 @@ class LlmProviders(str, Enum): CODESTRAL = "codestral" TEXT_COMPLETION_CODESTRAL = "text-completion-codestral" DASHSCOPE = "dashscope" + MODELSCOPE = "modelscope" MOONSHOT = "moonshot" PUBLICAI = "publicai" V0 = "v0" @@ -3420,6 +3421,7 @@ class LlmProviders(str, Enum): PARASAIL = "parasail" XIAOMI_MIMO = "xiaomi_mimo" TENSORMESH = "tensormesh" + LIBERTAI = "libertai" LITELLM_AGENT = "litellm_agent" CURSOR = "cursor" BEDROCK_MANTLE = "bedrock_mantle" @@ -3455,6 +3457,7 @@ class SearchProviders(str, Enum): GOOGLE_PSE = "google_pse" DATAFORSEO = "dataforseo" FIRECRAWL = "firecrawl" + FASTCRW = "fastcrw" SEARXNG = "searxng" LINKUP = "linkup" DUCKDUCKGO = "duckduckgo" diff --git a/litellm/utils.py b/litellm/utils.py index 4c67abdf937..3748cbd8b95 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3015,6 +3015,21 @@ def register_model(model_cost: Union[str, dict]): # noqa: PLR0915 # custom pricing on subsequent cost lookups. if existing_model.get("litellm_provider") is None: existing_model.pop("litellm_provider", None) + # Same pattern for cost fields (#30198): ``_get_model_info_helper`` + # synthesizes ``input_cost_per_token`` / ``output_cost_per_token`` + # = 0 when they are absent from the raw entry. Writing those zeros + # back flips a sparse entry from "no cost keys" (priced via name) + # to "cost keys = 0" (free), which makes + # ``_is_cost_explicitly_configured`` return True and silently + # disables budget enforcement on the next re-registration. + _raw_entry = litellm.model_cost.get(model_cost_key) + if _raw_entry is None: + _raw_entry = litellm.model_cost.get(key) + if _raw_entry is None: + _raw_entry = {} + for _cost_field in ("input_cost_per_token", "output_cost_per_token"): + if _cost_field not in _raw_entry and _cost_field not in value: + existing_model.pop(_cost_field, None) ## override / add new keys to the existing model cost dictionary updated_dictionary = _update_dictionary(existing_model, value) litellm.model_cost.setdefault(model_cost_key, {}).update(updated_dictionary) @@ -6833,6 +6848,11 @@ def validate_environment( # noqa: PLR0915 keys_in_environment = True else: missing_keys.append("DASHSCOPE_API_KEY") + elif custom_llm_provider == "modelscope": + if "MODELSCOPE_API_KEY" in os.environ: + keys_in_environment = True + else: + missing_keys.append("MODELSCOPE_API_KEY") elif custom_llm_provider == "moonshot": if "MOONSHOT_API_KEY" in os.environ: keys_in_environment = True @@ -8504,6 +8524,7 @@ class ProviderConfigManager: LlmProviders.NEBIUS: (lambda: litellm.NebiusConfig(), False), LlmProviders.WANDB: (lambda: litellm.WandbConfig(), False), LlmProviders.DASHSCOPE: (lambda: litellm.DashScopeChatConfig(), False), + LlmProviders.MODELSCOPE: (lambda: litellm.ModelScopeChatConfig(), False), LlmProviders.MOONSHOT: (lambda: litellm.MoonshotChatConfig(), False), LlmProviders.DOCKER_MODEL_RUNNER: ( lambda: litellm.DockerModelRunnerChatConfig(), @@ -9442,6 +9463,12 @@ class ProviderConfigManager: ) return get_dashscope_image_generation_config(model) + elif LlmProviders.MODELSCOPE == provider: + from litellm.llms.modelscope.image_generation import ( + get_modelscope_image_generation_config, + ) + + return get_modelscope_image_generation_config(model) return None @staticmethod @@ -9647,6 +9674,7 @@ class ProviderConfigManager: from litellm.llms.dataforseo.search.transformation import DataForSEOSearchConfig from litellm.llms.duckduckgo.search.transformation import DuckDuckGoSearchConfig from litellm.llms.exa_ai.search.transformation import ExaAISearchConfig + from litellm.llms.fastcrw.search.transformation import FastCRWSearchConfig from litellm.llms.firecrawl.search.transformation import FirecrawlSearchConfig from litellm.llms.google_pse.search.transformation import GooglePSESearchConfig from litellm.llms.linkup.search.transformation import LinkupSearchConfig @@ -9669,6 +9697,7 @@ class ProviderConfigManager: SearchProviders.GOOGLE_PSE: GooglePSESearchConfig, SearchProviders.DATAFORSEO: DataForSEOSearchConfig, SearchProviders.FIRECRAWL: FirecrawlSearchConfig, + SearchProviders.FASTCRW: FastCRWSearchConfig, SearchProviders.SEARXNG: SearXNGSearchConfig, SearchProviders.LINKUP: LinkupSearchConfig, SearchProviders.DUCKDUCKGO: DuckDuckGoSearchConfig, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b181df94131..f0c15654cfe 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -18822,6 +18822,38 @@ "supports_response_schema": true, "supports_vision": true }, + "github_copilot/mai-code-1-flash": { + "cache_read_input_token_cost": 7.5e-08, + "input_cost_per_token": 7.5e-07, + "litellm_provider": "github_copilot", + "max_input_tokens": 128000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 4.5e-06, + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true + }, + "github_copilot/mai-code-1-flash-internal": { + "cache_read_input_token_cost": 7.5e-08, + "input_cost_per_token": 7.5e-07, + "litellm_provider": "github_copilot", + "max_input_tokens": 128000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 4.5e-06, + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true + }, "github_copilot/text-embedding-3-small": { "litellm_provider": "github_copilot", "max_input_tokens": 8191, @@ -40986,6 +41018,174 @@ "litellm_provider": "llamagate", "mode": "embedding" }, + "libertai/hermes-3-8b-tee": { + "max_tokens": 16000, + "max_input_tokens": 16000, + "max_output_tokens": 16000, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "libertai", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_vision": false, + "source": "https://docs.libertai.io/apis/text/" + }, + "libertai/gemma-4-31b-it": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 4e-07, + "litellm_provider": "libertai", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_vision": true, + "source": "https://docs.libertai.io/apis/text/" + }, + "libertai/gemma-4-31b-it-thinking": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 4e-07, + "litellm_provider": "libertai", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "source": "https://docs.libertai.io/apis/text/" + }, + "libertai/qwen3.6-27b": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 5e-07, + "litellm_provider": "libertai", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_vision": true, + "source": "https://docs.libertai.io/apis/text/" + }, + "libertai/qwen3.6-27b-thinking": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 5e-07, + "litellm_provider": "libertai", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "source": "https://docs.libertai.io/apis/text/" + }, + "libertai/qwen3.6-35b-a3b": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 5e-07, + "litellm_provider": "libertai", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_vision": true, + "source": "https://docs.libertai.io/apis/text/" + }, + "libertai/qwen3.6-35b-a3b-thinking": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 5e-07, + "litellm_provider": "libertai", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "source": "https://docs.libertai.io/apis/text/" + }, + "libertai/qwen3.5-122b-a10b": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 1.75e-06, + "litellm_provider": "libertai", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_vision": true, + "source": "https://docs.libertai.io/apis/text/" + }, + "libertai/qwen3.5-122b-a10b-thinking": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 1.75e-06, + "litellm_provider": "libertai", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "source": "https://docs.libertai.io/apis/text/" + }, + "libertai/deepseek-v4-flash": { + "max_tokens": 200000, + "max_input_tokens": 200000, + "max_output_tokens": 200000, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 1.75e-06, + "litellm_provider": "libertai", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_vision": false, + "source": "https://docs.libertai.io/apis/text/" + }, + "libertai/deepseek-v4-flash-thinking": { + "max_tokens": 200000, + "max_input_tokens": 200000, + "max_output_tokens": 200000, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 1.75e-06, + "litellm_provider": "libertai", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_vision": false, + "supports_reasoning": true, + "source": "https://docs.libertai.io/apis/text/" + }, + "libertai/bge-m3": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "input_cost_per_token": 1e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "libertai", + "mode": "embedding", + "source": "https://docs.libertai.io/apis/text/" + }, "sarvam/sarvam-m": { "cache_creation_input_token_cost": 0, "cache_creation_input_token_cost_above_1hr": 0, diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 2ad2b3ec982..b90e5d2698d 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -972,6 +972,23 @@ "search": true } }, + "fastcrw": { + "display_name": "fastCRW (`fastcrw`)", + "url": "https://docs.litellm.ai/docs/search/fastcrw", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "search": true + } + }, "linkup": { "display_name": "Linkup (`linkup`)", "url": "https://docs.litellm.ai/docs/search/linkup", @@ -1359,6 +1376,23 @@ "interactions": true } }, + "libertai": { + "display_name": "LibertAI (`libertai`)", + "url": "https://docs.litellm.ai/docs/providers/libertai", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, "litellm_proxy": { "display_name": "LiteLLM Proxy (`litellm_proxy`)", "url": "https://docs.litellm.ai/docs/providers/litellm_proxy", @@ -1468,6 +1502,24 @@ "interactions": true } }, + "modelscope": { + "display_name": "ModelScope (`modelscope`)", + "url": "https://docs.litellm.ai/docs/providers/modelscope", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": true, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false, + "interactions": false + } + }, "moonshot": { "display_name": "Moonshot (`moonshot`)", "url": "https://docs.litellm.ai/docs/providers/moonshot", diff --git a/tests/code_coverage_tests/enforce_llms_folder_style.py b/tests/code_coverage_tests/enforce_llms_folder_style.py index 43ab81b6c60..cbf5cd5266e 100644 --- a/tests/code_coverage_tests/enforce_llms_folder_style.py +++ b/tests/code_coverage_tests/enforce_llms_folder_style.py @@ -14,6 +14,7 @@ SEARCH_PROVIDERS = [ "exa_ai", "brave", "firecrawl", + "fastcrw", "searxng", "linkup", "duckduckgo", diff --git a/tests/test_anthropic_compaction_usage.py b/tests/test_anthropic_compaction_usage.py new file mode 100644 index 00000000000..1758a94fffc --- /dev/null +++ b/tests/test_anthropic_compaction_usage.py @@ -0,0 +1,96 @@ +from litellm.llms.anthropic.chat.transformation import AnthropicConfig + + +def test_anthropic_compaction_usage_calculation(): + """ + Test that calculate_usage correctly sums tokens from the iterations array + as requested in Issue #27060. + """ + anthropic_config = AnthropicConfig() + + # Mock usage object with compaction iterations + usage_object = { + "input_tokens": 100, # Top-level (excludes compaction) + "output_tokens": 50, # Top-level (excludes compaction) + "iterations": [ + { + "iteration": 1, + "type": "compaction", + "input_tokens": 1000, + "output_tokens": 500, + }, + { + "iteration": 2, + "type": "message", + "input_tokens": 100, + "output_tokens": 50, + }, + ], + } + + usage = anthropic_config.calculate_usage( + usage_object=usage_object, reasoning_content=None + ) + + # Assertions + # Total prompt tokens should be 1000 + 100 = 1100 + assert usage.prompt_tokens == 1100 + # Total completion tokens should be 500 + 50 = 550 + assert usage.completion_tokens == 550 + # Total tokens should be 1650 + assert usage.total_tokens == 1650 + + # Assert details + assert usage.prompt_tokens_details.text_tokens == 1100 + + # Assert iterations passthrough + assert usage.iterations is not None + assert len(usage.iterations) == 2 + assert usage.iterations[0]["type"] == "compaction" + + +def test_anthropic_compaction_usage_with_iteration_cache(): + """ + Test that calculate_usage correctly sums caching tokens FROM iterations. + This covers the specific case mentioned by JasonPan. + """ + anthropic_config = AnthropicConfig() + + usage_object = { + "input_tokens": 100, + "output_tokens": 50, + "iterations": [ + { + "type": "compaction", + "input_tokens": 500, + "output_tokens": 200, + "cache_creation_input_tokens": 50, + "cache_read_input_tokens": 17000, + }, + { + "type": "message", + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 10, + "cache_read_input_tokens": 20, + }, + ], + } + + usage = anthropic_config.calculate_usage( + usage_object=usage_object, reasoning_content=None + ) + + # input_tokens sum = 500 + 100 = 600 + # cache_creation sum = 50 + 10 = 60 + # cache_read sum = 17000 + 20 = 17020 + # Total prompt tokens = 600 + 60 + 17020 = 17680 + assert usage.prompt_tokens == 17680 + assert usage.completion_tokens == 250 + assert usage.prompt_tokens_details.cache_creation_tokens == 60 + assert usage.prompt_tokens_details.cached_tokens == 17020 + + +if __name__ == "__main__": + test_anthropic_compaction_usage_calculation() + test_anthropic_compaction_usage_with_iteration_cache() diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 06457dfebff..6a1de0586dd 100644 --- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -2819,3 +2819,37 @@ def test_reasoning_items_streaming_emitted_on_response_completed(): ri["encrypted_content"] == encrypted ), "encrypted_content must be preserved in streaming" assert ri["summary"][0]["text"] == summary_text + + +def test_streaming_function_call_tool_id_for_degenerate_call_id(): + """In streaming, Bedrock Mantle's function_call event carries a unique ``id`` + (``fc_...``) and a non-unique, index-based ``call_id`` (``call_0``). For that + degenerate form the chat tool-call chunk must use the unique ``id`` so multi-turn + streaming agents don't collapse every tool call to the same id (which makes the + agent loop). A normal (unique) ``call_id`` must be preserved. Regression for the + bedrock-mantle gpt-5.5 streaming path.""" + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + OpenAiResponsesToChatCompletionStreamIterator, + ) + + def stream_tool_id(item_id, call_id): + chunk = { + "type": "response.output_item.added", + "output_index": 0, + "item": { + "type": "function_call", + "id": item_id, + "call_id": call_id, + "name": "get_weather", + "arguments": "", + }, + } + out = OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream( + chunk + ) + tool_calls = out.model_dump()["choices"][0]["delta"]["tool_calls"] + assert tool_calls, "expected a tool_call chunk in the streaming delta" + return tool_calls[0]["id"] + + assert stream_tool_id("fc_unique_abc123", "call_0") == "fc_unique_abc123" + assert stream_tool_id("fc_2", "call_tokyo") == "call_tokyo" diff --git a/tests/test_litellm/integrations/test_custom_guardrail.py b/tests/test_litellm/integrations/test_custom_guardrail.py index 57fb0fe6714..29e9f4529fc 100644 --- a/tests/test_litellm/integrations/test_custom_guardrail.py +++ b/tests/test_litellm/integrations/test_custom_guardrail.py @@ -363,6 +363,54 @@ class TestCustomGuardrailShouldRunGuardrail: result is False ), "Admin config in metadata must be respected when other metadata key is empty" + def test_should_run_guardrail_key_disable_global_not_overruled_by_team_guardrail_list( + self, + ): + """Key disable_global_guardrails must take precedence over the guardrail + appearing in the team's explicit guardrails list.""" + from litellm.types.guardrails import GuardrailEventHooks + + custom_guardrail = CustomGuardrail( + guardrail_name="global_guardrail", + default_on=True, + event_hook=GuardrailEventHooks.pre_call, + ) + + # Key disabled globals; team added the same guardrail to its explicit list + # (simulates what _add_guardrails_from_key_or_team_metadata produces). + data_key_disabled_team_listed = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "test"}], + "metadata": { + "user_api_key_metadata": {"disable_global_guardrails": True}, + "guardrails": ["global_guardrail"], + }, + } + assert ( + custom_guardrail.should_run_guardrail( + data=data_key_disabled_team_listed, + event_type=GuardrailEventHooks.pre_call, + ) + is False + ), "Key disable_global_guardrails must win over team's explicit guardrail list" + + # Complementary: key NOT disabled, team added guardrail → should run + data_key_enabled_team_listed = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "test"}], + "metadata": { + "user_api_key_metadata": {}, + "guardrails": ["global_guardrail"], + }, + } + assert ( + custom_guardrail.should_run_guardrail( + data=data_key_enabled_team_listed, + event_type=GuardrailEventHooks.pre_call, + ) + is True + ), "Guardrail in team's explicit list should run when key has not disabled globals" + def test_should_run_guardrail_with_opted_out_global_guardrails(self): """Test that per-guardrail opt-out only works from admin metadata""" from litellm.types.guardrails import GuardrailEventHooks diff --git a/tests/test_litellm/litellm_core_utils/test_fallback_utils.py b/tests/test_litellm/litellm_core_utils/test_fallback_utils.py index 0c542ff6a1b..90a61696e9d 100644 --- a/tests/test_litellm/litellm_core_utils/test_fallback_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_fallback_utils.py @@ -1,7 +1,13 @@ +"""Tests for litellm.litellm_core_utils.fallback_utils.""" + import pytest +import httpx import litellm -from litellm.litellm_core_utils.fallback_utils import async_completion_with_fallbacks +from litellm.litellm_core_utils.core_helpers import process_response_headers +from litellm.litellm_core_utils.fallback_utils import ( + async_completion_with_fallbacks, +) @pytest.mark.asyncio @@ -41,3 +47,123 @@ async def test_fallback_dict_not_mutated(monkeypatch): "primary-model", "fallback-model", ] + + +@pytest.mark.asyncio +async def test_async_completion_with_fallbacks_sets_attempted_fallbacks_header(): + """ + When a fallback succeeds, the response must carry the + `x-litellm-attempted-fallbacks` header so the proxy and other callers can + detect that a fallback occurred. Without it, + `_override_openai_response_model` stamps the requested model back over the + fallback model used. See issue #28241. + """ + response = await async_completion_with_fallbacks( + model="openai/primary-llm", + messages=[{"role": "user", "content": "hi"}], + api_key="fake-key", + mock_response=Exception("forced failure"), + kwargs={ + "fallbacks": [ + { + "model": "openai/backup-llm", + "api_key": "fake-key", + "mock_response": "backup-resp", + } + ] + }, + ) + + hidden_params = getattr(response, "_hidden_params", None) + assert isinstance(hidden_params, dict) + headers = hidden_params.get("additional_headers") or {} + assert headers.get("x-litellm-attempted-fallbacks") == 1 + + +@pytest.mark.asyncio +async def test_async_completion_with_fallbacks_header_is_zero_when_primary_succeeds(): + """ + When the primary model succeeds on the first attempt, the header should be + `0` (no fallback was used). This mirrors the existing router-level + semantics in `async_function_with_fallbacks`. + """ + response = await async_completion_with_fallbacks( + model="openai/primary-llm", + messages=[{"role": "user", "content": "hi"}], + api_key="fake-key", + mock_response="primary-resp", + kwargs={ + "fallbacks": [ + { + "model": "openai/backup-llm", + "api_key": "fake-key", + "mock_response": "backup-resp", + } + ] + }, + ) + + hidden_params = getattr(response, "_hidden_params", None) + assert isinstance(hidden_params, dict) + headers = hidden_params.get("additional_headers") or {} + assert headers.get("x-litellm-attempted-fallbacks") == 0 + assert response.choices[0].message.content == "primary-resp" + + +def test_process_response_headers_preserves_x_litellm_headers_when_internal(): + """ + `process_response_headers` must not add the `llm_provider-` prefix to + LiteLLM's own internal headers (anything starting with `x-litellm-`) when + the caller has marked the input as LiteLLM-owned. These are markers set by + LiteLLM (e.g. fallback / retry headers); the proxy and other callers look + up the bare key. + """ + result = process_response_headers( + { + "x-litellm-attempted-fallbacks": 1, + "x-litellm-model-group": "gpt-4", + "x-stainless-arch": "arm64", + }, + preserve_litellm_internal_headers=True, + ) + assert result["x-litellm-attempted-fallbacks"] == 1 + assert result["x-litellm-model-group"] == "gpt-4" + assert result["llm_provider-x-stainless-arch"] == "arm64" + + +def test_process_response_headers_prefixes_x_litellm_from_raw_provider(): + """ + On raw upstream-provider headers (default `preserve_litellm_internal_headers=False`), + a header whose name starts with `x-litellm-` MUST still get the + `llm_provider-` prefix. Otherwise a malicious provider could return + `x-litellm-attempted-fallbacks` and spoof a LiteLLM-internal marker, + bypassing the proxy model-override guard. + """ + result = process_response_headers( + { + "x-litellm-attempted-fallbacks": 99, + "x-stainless-arch": "arm64", + } + ) + assert "x-litellm-attempted-fallbacks" not in result + assert result["llm_provider-x-litellm-attempted-fallbacks"] == 99 + assert result["llm_provider-x-stainless-arch"] == "arm64" + + +def test_process_response_headers_ignores_preserve_flag_for_httpx_headers(): + """ + Some providers store raw httpx.Headers directly in _hidden_params["additional_headers"] + without a prior normalization pass. If preserve_litellm_internal_headers=True were + honored for httpx.Headers inputs, a provider returning x-litellm-attempted-fallbacks + could spoof it as a bare LiteLLM-internal marker and make the proxy skip + stamping the correct response model. The flag must be ignored for httpx.Headers. + """ + raw = httpx.Headers( + { + "x-litellm-attempted-fallbacks": "1", + "content-type": "application/json", + } + ) + result = process_response_headers(raw, preserve_litellm_internal_headers=True) + assert "x-litellm-attempted-fallbacks" not in result + assert result["llm_provider-x-litellm-attempted-fallbacks"] == "1" diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py index c5794194528..b5eb7af88b3 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -8,7 +8,8 @@ sys.path.insert( 0, os.path.abspath("../../..") ) # Adds the parent directory to the system path -from litellm import stream_chunk_builder +from litellm import ChatCompletionUsageBlock, stream_chunk_builder +from litellm.types.utils import GenericStreamingChunk from litellm.litellm_core_utils.streaming_chunk_builder_utils import ChunkProcessor from litellm.types.utils import ( ChatCompletionDeltaToolCall, @@ -324,6 +325,42 @@ def test_cache_read_input_tokens_retained(): assert usage.cache_read_input_tokens == 11775 assert usage.prompt_tokens_details.cached_tokens == 11775 +def test_cache_read_input_tokens_retained_genericstreamingchunk(): + chunk1 = GenericStreamingChunk( + text="Test1", + is_finished=False, + finish_reason="", + usage=None, + index=1, + ) + + chunk2 = GenericStreamingChunk( + text="Test2", + is_finished=True, + finish_reason="stop", + usage=ChatCompletionUsageBlock( + completion_tokens=5, + prompt_tokens=1234, + total_tokens=1239, + completion_tokens_details=None, + prompt_tokens_details=PromptTokensDetails( + audio_tokens=None, cached_tokens=543 + ).model_dump(), + ), + index=2, + ) + + # Use dictionaries directly instead of ModelResponseStream + chunks = [chunk1, chunk2] + processor = ChunkProcessor(chunks=chunks) + + usage = processor.calculate_usage( + chunks=chunks, + model="gpt-5.5", + completion_output="", + ) + + assert usage.prompt_tokens_details.cached_tokens == 543 def test_stream_chunk_builder_litellm_usage_chunks(): """ diff --git a/tests/test_litellm/llms/base_llm/test_base_model_iterator.py b/tests/test_litellm/llms/base_llm/test_base_model_iterator.py index b7f12a92ccc..f54e71c7c3c 100644 --- a/tests/test_litellm/llms/base_llm/test_base_model_iterator.py +++ b/tests/test_litellm/llms/base_llm/test_base_model_iterator.py @@ -221,3 +221,38 @@ async def test_pydantic_basemodel_chunk_passes_through_async(): assert len(chunks) == 1 assert "response.created" in chunks[0]["text"] + + +@pytest.mark.asyncio +async def test_aclose_closes_attached_http_response(): + """Regression for BerriAI/litellm#30244: CustomStreamWrapper.aclose() can + only release the upstream provider connection if the iterator exposes + aclose() and it reaches the underlying HTTP response. Without this, a + client disconnect leaves backends like vLLM generating into a dead pipe.""" + from unittest.mock import AsyncMock, MagicMock + + async def async_gen(): + yield "data: {}" + + iterator = BaseModelResponseIterator( + streaming_response=async_gen(), sync_stream=False + ) + http_response = MagicMock() + http_response.aclose = AsyncMock() + iterator.http_response = http_response + + await iterator.aclose() + + http_response.aclose.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_aclose_is_noop_without_http_response(): + async def async_gen(): + yield "data: {}" + + iterator = BaseModelResponseIterator( + streaming_response=async_gen(), sync_stream=False + ) + + await iterator.aclose() diff --git a/tests/test_litellm/llms/fastcrw/search/test_transformation.py b/tests/test_litellm/llms/fastcrw/search/test_transformation.py new file mode 100644 index 00000000000..adf8fec087c --- /dev/null +++ b/tests/test_litellm/llms/fastcrw/search/test_transformation.py @@ -0,0 +1,182 @@ +import os +from unittest.mock import Mock, patch + +import pytest + +import litellm +from litellm.llms.fastcrw.search.transformation import FastCRWSearchConfig + + +def _config() -> FastCRWSearchConfig: + return FastCRWSearchConfig() + + +def test_fastcrw_search_request_body(): + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "success": True, + "data": [ + { + "title": "Test Title", + "url": "https://example.com", + "description": "Test description", + "markdown": "Test content", + } + ], + } + + with ( + patch.dict(os.environ, {"CRW_API_KEY": "test-api-key"}), + patch( + "litellm.llms.custom_httpx.http_handler.HTTPHandler.post", + return_value=mock_response, + ) as mock_post, + ): + response = litellm.search( + query="test query", + search_provider="fastcrw", + max_results=10, + ) + + assert mock_post.called + call_kwargs = mock_post.call_args.kwargs + assert call_kwargs.get("url", "").endswith("/search") + + request_body = call_kwargs.get("json") + assert request_body is not None + assert request_body["query"] == "test query" + assert request_body["limit"] == 10 + + assert len(response.results) == 1 + result = response.results[0] + assert result.title == "Test Title" + assert result.url == "https://example.com" + assert result.snippet == "Test content" + + +def test_ui_friendly_name(): + assert _config().ui_friendly_name() == "fastCRW" + + +def test_validate_environment_with_explicit_key(): + headers = _config().validate_environment({}, api_key="explicit-key") + assert headers["Authorization"] == "Bearer explicit-key" + assert headers["Content-Type"] == "application/json" + + +def test_validate_environment_reads_env_key(): + with patch.dict(os.environ, {"CRW_API_KEY": "env-key"}, clear=False): + headers = _config().validate_environment({}) + assert headers["Authorization"] == "Bearer env-key" + + +def test_validate_environment_missing_key_raises(): + with patch.dict(os.environ, {}, clear=True): + with pytest.raises(ValueError, match="CRW_API_KEY"): + _config().validate_environment({}) + + +def test_get_complete_url_default_base(): + with patch.dict(os.environ, {}, clear=True): + assert _config().get_complete_url(None, {}) == "https://fastcrw.com/api/v1/search" + + +def test_get_complete_url_appends_search(): + assert ( + _config().get_complete_url("https://self-hosted.local/api/v1", {}) + == "https://self-hosted.local/api/v1/search" + ) + + +def test_get_complete_url_does_not_double_append(): + assert ( + _config().get_complete_url("https://self-hosted.local/api/v1/search", {}) + == "https://self-hosted.local/api/v1/search" + ) + + +def test_get_complete_url_reads_env_base(): + with patch.dict( + os.environ, {"CRW_API_BASE": "https://env-base.local/v1"}, clear=True + ): + assert _config().get_complete_url(None, {}) == "https://env-base.local/v1/search" + + +def test_transform_search_request_basic(): + data = _config().transform_search_request("hello", {"max_results": 5}) + assert data["query"] == "hello" + assert data["limit"] == 5 + assert data["scrapeOptions"]["formats"] == ["markdown"] + assert data["scrapeOptions"]["onlyMainContent"] is True + + +def test_transform_search_request_joins_list_query(): + assert _config().transform_search_request(["foo", "bar"], {})["query"] == "foo bar" + + +def test_transform_search_request_passes_through_extra_params(): + data = _config().transform_search_request("q", {"sources": ["web", "images"]}) + assert data["sources"] == ["web", "images"] + + +def test_transform_search_request_preserves_explicit_scrape_options(): + custom = {"formats": ["html"]} + data = _config().transform_search_request("q", {"scrapeOptions": custom}) + assert data["scrapeOptions"] == custom + + +def _resp(payload): + r = Mock() + r.json.return_value = payload + return r + + +def test_transform_search_response_prefers_markdown(): + resp = _config().transform_search_response( + _resp( + { + "success": True, + "data": [ + { + "title": "T", + "url": "https://e.com", + "description": "d", + "markdown": "md", + } + ], + } + ), + logging_obj=Mock(), + ) + assert len(resp.results) == 1 + assert resp.results[0].snippet == "md" + + +def test_transform_search_response_falls_back_to_description(): + resp = _config().transform_search_response( + _resp( + { + "success": True, + "data": [ + {"title": "T", "url": "https://e.com", "description": "only-desc"} + ], + } + ), + logging_obj=Mock(), + ) + assert resp.results[0].snippet == "only-desc" + + +def test_transform_search_response_empty_data(): + resp = _config().transform_search_response( + _resp({"success": True, "data": []}), logging_obj=Mock() + ) + assert resp.results == [] + + +def test_transform_search_response_non_list_data(): + resp = _config().transform_search_response( + _resp({"success": True, "data": {"unexpected": "shape"}}), logging_obj=Mock() + ) + assert resp.results == [] diff --git a/tests/test_litellm/llms/modelscope/chat/test_modelscope_chat_transformation.py b/tests/test_litellm/llms/modelscope/chat/test_modelscope_chat_transformation.py new file mode 100644 index 00000000000..2767deae176 --- /dev/null +++ b/tests/test_litellm/llms/modelscope/chat/test_modelscope_chat_transformation.py @@ -0,0 +1,394 @@ +""" +Unit tests for ModelScope configuration. + +These tests validate the ModelScopeChatConfig class which extends OpenAIGPTConfig. +ModelScope is an OpenAI-compatible provider with minor customizations. +""" + +import json +import os +import sys + +sys.path.insert( + 0, os.path.abspath("../../../../..") +) # Adds the parent directory to the system path + +from unittest.mock import patch + +import httpx +import pytest +import respx + +import litellm +from litellm import completion +from litellm.llms.modelscope.chat.transformation import ModelScopeChatConfig + +DEFAULT_MODEL = "Qwen/Qwen3.5-35B-A3B" + + +class TestModelScopeConfig: + """Test class for ModelScope functionality""" + + def test_default_api_base(self): + """Test that default API base is used when none is provided""" + config = ModelScopeChatConfig() + headers = {} + api_key = "fake-modelscope-key" + + result = config.validate_environment( + headers=headers, + model=DEFAULT_MODEL, + messages=[{"role": "user", "content": "Hey"}], + optional_params={}, + litellm_params={}, + api_key=api_key, + api_base=None, + ) + + assert result["Authorization"] == f"Bearer {api_key}" + assert result["Content-Type"] == "application/json" + + @pytest.mark.respx() + def test_modelscope_completion_mock(self, respx_mock): + """Mock test for basic ModelScope completion.""" + + litellm.disable_aiohttp_transport = True + + api_key = "fake-modelscope-key" + api_base = "https://api-inference.modelscope.cn/v1" + + respx_mock.post(f"{api_base}/chat/completions").respond( + json={ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": DEFAULT_MODEL, + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": '```python\nprint("Hey from LiteLLM!")\n```', + }, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 9, + "completion_tokens": 12, + "total_tokens": 21, + }, + }, + status_code=200, + ) + + response = completion( + model=f"modelscope/{DEFAULT_MODEL}", + messages=[ + {"role": "user", "content": "write code for saying hey from LiteLLM"} + ], + api_key=api_key, + api_base=api_base, + ) + + assert response is not None + assert response.choices[0].message.content is not None + assert "```python" in response.choices[0].message.content + + # ── _transform_messages tests ────────────────────────────────────── + + def test_transform_messages_flattens_text_content_list(self): + """Content lists containing only text items should be flattened to a string.""" + config = ModelScopeChatConfig() + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Hello"}, + {"type": "text", "text": " world"}, + ], + } + ] + + result = config._transform_messages(messages=messages, model=DEFAULT_MODEL) + + assert result[0]["content"] == "Hello world" + + def test_transform_messages_preserves_multimodal_content_list(self): + """Content lists with image_url should be preserved as lists for vision models.""" + config = ModelScopeChatConfig() + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is this?"}, + {"type": "image_url", "image_url": {"url": "https://example.com/img.png"}}, + ], + } + ] + + result = config._transform_messages(messages=messages, model=DEFAULT_MODEL) + + assert isinstance(result[0]["content"], list) + assert len(result[0]["content"]) == 2 + assert result[0]["content"][0]["type"] == "text" + assert result[0]["content"][1]["type"] == "image_url" + + def test_transform_messages_string_content_unchanged(self): + """Messages with string content should pass through unchanged.""" + config = ModelScopeChatConfig() + messages = [{"role": "user", "content": "Hello"}] + + result = config._transform_messages(messages=messages, model=DEFAULT_MODEL) + + assert result[0]["content"] == "Hello" + + def test_transform_messages_multi_turn(self): + """Multi-turn conversations should be handled correctly.""" + config = ModelScopeChatConfig() + messages = [ + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hello!"}, + { + "role": "user", + "content": [ + {"type": "text", "text": "Tell me more"}, + ], + }, + ] + + result = config._transform_messages(messages=messages, model=DEFAULT_MODEL) + + assert result[0]["content"] == "Hi" + assert result[1]["content"] == "Hello!" + assert result[2]["content"] == "Tell me more" + + def test_transform_messages_multimodal_multi_turn(self): + """Multi-turn with mixed text-only and multimodal messages.""" + config = ModelScopeChatConfig() + messages = [ + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hello!"}, + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this image"}, + {"type": "image_url", "image_url": {"url": "https://example.com/photo.jpg"}}, + ], + }, + ] + + result = config._transform_messages(messages=messages, model=DEFAULT_MODEL) + + assert result[0]["content"] == "Hi" + assert result[1]["content"] == "Hello!" + # Multimodal message should keep list format + assert isinstance(result[2]["content"], list) + assert result[2]["content"][1]["type"] == "image_url" + + # ── get_complete_url tests ───────────────────────────────────────── + + def test_get_complete_url_default(self): + """Default api_base should append /chat/completions.""" + config = ModelScopeChatConfig() + + url = config.get_complete_url( + api_base=None, + api_key="fake-key", + model=DEFAULT_MODEL, + optional_params={}, + litellm_params={}, + ) + + assert url == "https://api-inference.modelscope.cn/v1/chat/completions" + + def test_get_complete_url_custom_base(self): + """Custom api_base should append /chat/completions.""" + config = ModelScopeChatConfig() + + url = config.get_complete_url( + api_base="https://custom.modelscope.cn/v1", + api_key="fake-key", + model=DEFAULT_MODEL, + optional_params={}, + litellm_params={}, + ) + + assert url == "https://custom.modelscope.cn/v1/chat/completions" + + def test_get_complete_url_already_has_endpoint(self): + """api_base already ending in /chat/completions should not be doubled.""" + config = ModelScopeChatConfig() + + url = config.get_complete_url( + api_base="https://api-inference.modelscope.cn/v1/chat/completions", + api_key="fake-key", + model=DEFAULT_MODEL, + optional_params={}, + litellm_params={}, + ) + + assert url == "https://api-inference.modelscope.cn/v1/chat/completions" + assert url.count("/chat/completions") == 1 + + # ── _get_openai_compatible_provider_info tests ───────────────────── + + def test_get_provider_info_with_explicit_api_base(self): + """Explicit api_base and api_key should be returned as-is.""" + config = ModelScopeChatConfig() + + api_base, api_key = config._get_openai_compatible_provider_info( + api_base="https://custom.example.com/v1", + api_key="my-key", + ) + + assert api_base == "https://custom.example.com/v1" + assert api_key == "my-key" + + def test_get_provider_info_default_fallback(self): + """When no api_base or env var is set, DEFAULT_BASE_URL should be used.""" + config = ModelScopeChatConfig() + + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("MODELSCOPE_API_BASE", None) + os.environ.pop("MODELSCOPE_API_KEY", None) + + api_base, api_key = config._get_openai_compatible_provider_info( + api_base=None, + api_key=None, + ) + + assert api_base == "https://api-inference.modelscope.cn/v1" + assert api_key is None + + def test_get_provider_info_env_var_fallback(self): + """MODELSCOPE_API_BASE env var should be used when api_base is not provided.""" + config = ModelScopeChatConfig() + + with patch.dict( + os.environ, + {"MODELSCOPE_API_BASE": "https://env.modelscope.cn/v1"}, + ): + api_base, _ = config._get_openai_compatible_provider_info( + api_base=None, + api_key=None, + ) + + assert api_base == "https://env.modelscope.cn/v1" + + # ── Mock HTTP tests ──────────────────────────────────────────────── + + @pytest.mark.respx() + def test_completion_with_text_content_list(self, respx_mock): + """Verify that text-only content list messages are flattened before sending.""" + litellm.disable_aiohttp_transport = True + + api_key = "fake-modelscope-key" + api_base = "https://api-inference.modelscope.cn/v1" + captured_request = {} + + def capture_request(request): + captured_request["body"] = request.content + return httpx.Response( + 200, + json={ + "id": "chatcmpl-456", + "object": "chat.completion", + "created": 1677652288, + "model": DEFAULT_MODEL, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Sure!"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6}, + }, + ) + + respx_mock.post(f"{api_base}/chat/completions").mock(side_effect=capture_request) + + response = completion( + model=f"modelscope/{DEFAULT_MODEL}", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "Hello"}, + {"type": "text", "text": " world"}, + ], + } + ], + api_key=api_key, + api_base=api_base, + ) + + assert response.choices[0].message.content == "Sure!" + + body = json.loads(captured_request["body"]) + assert isinstance(body["messages"][0]["content"], str) + assert body["messages"][0]["content"] == "Hello world" + + @pytest.mark.respx() + def test_completion_with_multimodal_messages(self, respx_mock): + """Verify that multimodal messages (text + image_url) are sent as content lists.""" + litellm.disable_aiohttp_transport = True + + api_key = "fake-modelscope-key" + api_base = "https://api-inference.modelscope.cn/v1" + captured_request = {} + + def capture_request(request): + captured_request["body"] = request.content + return httpx.Response( + 200, + json={ + "id": "chatcmpl-789", + "object": "chat.completion", + "created": 1677652288, + "model": DEFAULT_MODEL, + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "A cat sitting on a couch.", + }, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 100, "completion_tokens": 8, "total_tokens": 108}, + }, + ) + + respx_mock.post(f"{api_base}/chat/completions").mock(side_effect=capture_request) + + response = completion( + model=f"modelscope/{DEFAULT_MODEL}", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + { + "type": "image_url", + "image_url": {"url": "https://example.com/cat.jpg"}, + }, + ], + } + ], + api_key=api_key, + api_base=api_base, + ) + + assert response.choices[0].message.content == "A cat sitting on a couch." + + body = json.loads(captured_request["body"]) + msg = body["messages"][0] + # Multimodal content should remain as a list + assert isinstance(msg["content"], list) + assert len(msg["content"]) == 2 + assert msg["content"][0] == {"type": "text", "text": "What is in this image?"} + assert msg["content"][1]["type"] == "image_url" + assert msg["content"][1]["image_url"]["url"] == "https://example.com/cat.jpg" diff --git a/tests/test_litellm/llms/modelscope/image_generation/test_modelscope_image_gen_transformation.py b/tests/test_litellm/llms/modelscope/image_generation/test_modelscope_image_gen_transformation.py new file mode 100644 index 00000000000..7f00f53c451 --- /dev/null +++ b/tests/test_litellm/llms/modelscope/image_generation/test_modelscope_image_gen_transformation.py @@ -0,0 +1,456 @@ +""" +Unit tests for ModelScope image generation configuration. + +These tests validate the ModelScopeImageGenerationConfig class which handles +transformation between OpenAI-compatible format and ModelScope API format. +""" + +import os +import sys +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.insert( + 0, os.path.abspath("../../../../..") +) # Adds the parent directory to the system path + +from litellm.llms.modelscope.image_generation.transformation import ( + ModelScopeImageGenerationConfig, +) +from litellm.types.utils import ImageResponse + + +class TestModelScopeImageGenerationTransformation: + def setup_method(self): + """Set up test fixtures before each test method.""" + self.config = ModelScopeImageGenerationConfig() + self.model = "modelscope/Qwen/Qwen-Image-Edit" + self.logging_obj = MagicMock() + + def test_get_supported_openai_params(self): + """Test that get_supported_openai_params returns correct parameters.""" + supported_params = self.config.get_supported_openai_params(self.model) + + assert "n" in supported_params + assert "size" in supported_params + assert "response_format" in supported_params + assert "user" in supported_params + + def test_map_openai_params(self): + """Test that map_openai_params correctly passes through parameters.""" + non_default_params = { + "n": 2, + "size": "1024x1024", + "response_format": "url", + } + optional_params = {} + + result = self.config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=self.model, + drop_params=False, + ) + + assert result["n"] == 2 + assert result["size"] == "1024x1024" + assert result["response_format"] == "url" + + def test_map_openai_params_with_user(self): + """Test that map_openai_params correctly passes through user parameter.""" + non_default_params = {"user": "test-user-123"} + optional_params = {} + + result = self.config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=self.model, + drop_params=False, + ) + + assert result["user"] == "test-user-123" + + def test_get_complete_url_default(self): + """Test that get_complete_url returns default ModelScope URL.""" + result = self.config.get_complete_url( + api_base=None, + api_key="test_key", + model=self.model, + optional_params={}, + litellm_params={}, + ) + + assert result == "https://api-inference.modelscope.cn/v1/images/generations" + + def test_get_complete_url_with_custom_base(self): + """Test that get_complete_url uses custom api_base.""" + custom_base = "https://custom.modelscope.cn/v1" + + result = self.config.get_complete_url( + api_base=custom_base, + api_key="test_key", + model=self.model, + optional_params={}, + litellm_params={}, + ) + + assert result == f"{custom_base}/images/generations" + + def test_get_complete_url_with_trailing_slash(self): + """Test that get_complete_url strips trailing slashes from base.""" + custom_base = "https://custom.modelscope.cn/v1/" + + result = self.config.get_complete_url( + api_base=custom_base, + api_key="test_key", + model=self.model, + optional_params={}, + litellm_params={}, + ) + + assert result == "https://custom.modelscope.cn/v1/images/generations" + + @patch("litellm.llms.modelscope.image_generation.transformation.get_secret_str") + def test_validate_environment_with_api_key(self, mock_get_secret): + """Test that validate_environment correctly sets authorization header.""" + headers = {} + api_key = "test_api_key" + + result = self.config.validate_environment( + headers=headers, + model=self.model, + messages=[], + optional_params={}, + litellm_params={}, + api_key=api_key, + ) + + assert result["Authorization"] == f"Bearer {api_key}" + assert result["Content-Type"] == "application/json" + mock_get_secret.assert_not_called() + + @patch("litellm.llms.modelscope.image_generation.transformation.get_secret_str") + def test_validate_environment_with_secret_key(self, mock_get_secret): + """Test that validate_environment uses secret API key when api_key is None.""" + mock_get_secret.return_value = "secret_api_key" + headers = {} + + result = self.config.validate_environment( + headers=headers, + model=self.model, + messages=[], + optional_params={}, + litellm_params={}, + api_key=None, + ) + + assert result["Authorization"] == "Bearer secret_api_key" + mock_get_secret.assert_called_once_with("MODELSCOPE_API_KEY") + + @patch("litellm.llms.modelscope.image_generation.transformation.get_secret_str") + def test_validate_environment_no_api_key(self, mock_get_secret): + """Test that validate_environment raises error when no API key is available.""" + mock_get_secret.return_value = None + headers = {} + + with pytest.raises(ValueError) as exc_info: + self.config.validate_environment( + headers=headers, + model=self.model, + messages=[], + optional_params={}, + litellm_params={}, + api_key=None, + ) + + assert "MODELSCOPE_API_KEY is not set" in str(exc_info.value) + + def test_transform_image_generation_request_basic(self): + """Test that transform_image_generation_request creates correct request body.""" + prompt = "A beautiful sunset over mountains" + optional_params = {} + + result = self.config.transform_image_generation_request( + model=self.model, + prompt=prompt, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert result["model"] == self.model + assert result["prompt"] == prompt + + def test_transform_image_generation_request_with_optional_params(self): + """Test that transform_image_generation_request includes optional params.""" + prompt = "A beautiful sunset" + optional_params = { + "n": 2, + "size": "1024x1024", + "response_format": "b64_json", + } + + result = self.config.transform_image_generation_request( + model=self.model, + prompt=prompt, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert result["model"] == self.model + assert result["prompt"] == prompt + assert result["n"] == 2 + assert result["size"] == "1024x1024" + assert result["response_format"] == "b64_json" + + def test_transform_image_generation_request_ignores_internal_params(self): + """Test that transform_image_generation_request ignores params starting with _.""" + prompt = "A beautiful sunset" + optional_params = { + "n": 2, + "_internal_param": "should_be_ignored", + } + + result = self.config.transform_image_generation_request( + model=self.model, + prompt=prompt, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert result["model"] == self.model + assert result["n"] == 2 + assert "_internal_param" not in result + + def test_transform_image_generation_response_with_url_images(self): + """Test that transform_image_generation_response correctly extracts URL images.""" + response_data = { + "created": 1234567890, + "data": [ + {"url": "https://example.com/image1.png"}, + {"url": "https://example.com/image2.png"}, + ], + } + + mock_response = MagicMock() + mock_response.json.return_value = response_data + mock_response.status_code = 200 + mock_response.headers = {} + + model_response = ImageResponse(data=[]) + + result = self.config.transform_image_generation_response( + model=self.model, + raw_response=mock_response, + model_response=model_response, + logging_obj=self.logging_obj, + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert len(result.data) == 2 + assert result.data[0].url == "https://example.com/image1.png" + assert result.data[1].url == "https://example.com/image2.png" + + def test_transform_image_generation_response_with_b64_json(self): + """Test that transform_image_generation_response correctly extracts base64 images.""" + response_data = { + "created": 1234567890, + "data": [ + {"b64_json": "iVBORw0KGgoAAAANS"}, + ], + } + + mock_response = MagicMock() + mock_response.json.return_value = response_data + mock_response.status_code = 200 + mock_response.headers = {} + + model_response = ImageResponse(data=[]) + + result = self.config.transform_image_generation_response( + model=self.model, + raw_response=mock_response, + model_response=model_response, + logging_obj=self.logging_obj, + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert len(result.data) == 1 + assert result.data[0].b64_json == "iVBORw0KGgoAAAANS" + assert result.data[0].url is None + + def test_transform_image_generation_response_with_revised_prompt(self): + """Test that transform_image_generation_response extracts revised_prompt.""" + response_data = { + "created": 1234567890, + "data": [ + { + "url": "https://example.com/image.png", + "revised_prompt": "A detailed description of a beautiful sunset", + }, + ], + } + + mock_response = MagicMock() + mock_response.json.return_value = response_data + mock_response.status_code = 200 + mock_response.headers = {} + + model_response = ImageResponse(data=[]) + + result = self.config.transform_image_generation_response( + model=self.model, + raw_response=mock_response, + model_response=model_response, + logging_obj=self.logging_obj, + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert len(result.data) == 1 + assert ( + result.data[0].revised_prompt + == "A detailed description of a beautiful sunset" + ) + + def test_transform_image_generation_response_empty_data(self): + """Test that transform_image_generation_response handles empty data array.""" + response_data = { + "created": 1234567890, + "data": [], + } + + mock_response = MagicMock() + mock_response.json.return_value = response_data + mock_response.status_code = 200 + mock_response.headers = {} + + model_response = ImageResponse(data=[]) + + result = self.config.transform_image_generation_response( + model=self.model, + raw_response=mock_response, + model_response=model_response, + logging_obj=self.logging_obj, + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert len(result.data) == 0 + + def test_transform_image_generation_response_error_handling(self): + """Test that transform_image_generation_response raises error on API error.""" + response_data = { + "error": { + "message": "Invalid prompt provided", + "type": "invalid_request_error", + } + } + + mock_response = MagicMock() + mock_response.json.return_value = response_data + mock_response.status_code = 400 + mock_response.headers = {} + + model_response = ImageResponse(data=[]) + + with pytest.raises(Exception) as exc_info: + self.config.transform_image_generation_response( + model=self.model, + raw_response=mock_response, + model_response=model_response, + logging_obj=self.logging_obj, + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert "ModelScope error" in str(exc_info.value) + assert "Invalid prompt provided" in str(exc_info.value) + + def test_transform_image_generation_response_json_error(self): + """Test that transform_image_generation_response raises error on invalid JSON.""" + import json + + mock_response = MagicMock() + mock_response.json.side_effect = json.JSONDecodeError("Invalid JSON", "", 0) + mock_response.status_code = 500 + mock_response.headers = {} + + model_response = ImageResponse(data=[]) + + with pytest.raises(Exception) as exc_info: + self.config.transform_image_generation_response( + model=self.model, + raw_response=mock_response, + model_response=model_response, + logging_obj=self.logging_obj, + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert "Error parsing ModelScope response" in str(exc_info.value) + + def test_get_error_class_bad_request(self): + """Test that get_error_class returns BadRequestError for 400 status.""" + from litellm.exceptions import BadRequestError + + error = self.config.get_error_class( + error_message="Bad request", + status_code=400, + headers={"Content-Type": "application/json"}, + ) + + assert isinstance(error, BadRequestError) + + def test_get_error_class_authentication_error(self): + """Test that get_error_class returns AuthenticationError for 401 status.""" + from litellm.exceptions import AuthenticationError + + error = self.config.get_error_class( + error_message="Invalid API key", + status_code=401, + headers={"Content-Type": "application/json"}, + ) + + assert isinstance(error, AuthenticationError) + + def test_get_error_class_internal_server_error(self): + """Test that get_error_class returns InternalServerError for 500+ status.""" + from litellm.exceptions import InternalServerError + + error = self.config.get_error_class( + error_message="Internal server error", + status_code=500, + headers={"Content-Type": "application/json"}, + ) + + assert isinstance(error, InternalServerError) + + def test_get_error_class_default(self): + """Test that get_error_class returns BadRequestError for other status codes.""" + from litellm.exceptions import BadRequestError + + error = self.config.get_error_class( + error_message="Some error", + status_code=404, + headers={"Content-Type": "application/json"}, + ) + + assert isinstance(error, BadRequestError) diff --git a/tests/test_litellm/llms/openai_like/test_libertai_provider.py b/tests/test_litellm/llms/openai_like/test_libertai_provider.py new file mode 100644 index 00000000000..fdbe3046e9b --- /dev/null +++ b/tests/test_litellm/llms/openai_like/test_libertai_provider.py @@ -0,0 +1,131 @@ +""" +Tests for LibertAI provider configuration and integration. +""" + +import litellm + + +class TestLibertAIProviderConfig: + """Test LibertAI provider configuration""" + + def test_libertai_in_provider_list(self): + """Test that libertai is in the provider list""" + from litellm import LlmProviders + + assert hasattr(LlmProviders, "LIBERTAI") + assert LlmProviders.LIBERTAI.value == "libertai" + assert "libertai" in litellm.provider_list + + def test_libertai_json_config_exists(self): + """Test that libertai is configured in providers.json""" + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + assert JSONProviderRegistry.exists("libertai") + + libertai = JSONProviderRegistry.get("libertai") + assert libertai is not None + assert libertai.base_url == "https://api.libertai.io/v1" + assert libertai.api_key_env == "LIBERTAI_API_KEY" + assert libertai.api_base_env == "LIBERTAI_API_BASE" + assert libertai.param_mappings.get("max_completion_tokens") == "max_tokens" + + def test_libertai_provider_resolution(self): + """Test that provider resolution finds libertai and the default base URL""" + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + model, provider, api_key, api_base = get_llm_provider( + model="libertai/qwen3.6-27b", + custom_llm_provider=None, + api_base=None, + api_key=None, + ) + + assert model == "qwen3.6-27b" + assert provider == "libertai" + assert api_base == "https://api.libertai.io/v1" + + def test_libertai_api_base_override(self): + """Test that an explicit api_base / api_key overrides the default""" + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + model, provider, api_key, api_base = get_llm_provider( + model="libertai/qwen3.6-27b", + custom_llm_provider=None, + api_base="https://custom.example.com/v1", + api_key="sk-test", + ) + + assert provider == "libertai" + assert api_base == "https://custom.example.com/v1" + assert api_key == "sk-test" + + def test_libertai_model_cost_map(self): + """Test that libertai models are present in the model cost map""" + model_cost = litellm.model_cost + + assert "libertai/qwen3.6-27b" in model_cost + info = model_cost["libertai/qwen3.6-27b"] + assert info["litellm_provider"] == "libertai" + assert info["mode"] == "chat" + assert info["max_input_tokens"] == 262144 + assert info["max_output_tokens"] == 262144 + + # thinking variants are marked as reasoning models + assert ( + model_cost["libertai/qwen3.6-27b-thinking"].get("supports_reasoning") + is True + ) + + def test_libertai_router_config(self): + """Test that libertai can be used in Router configuration""" + from litellm import Router + + router = Router( + model_list=[ + { + "model_name": "libertai-chat", + "litellm_params": { + "model": "libertai/qwen3.6-27b", + "api_key": "test-key", + }, + } + ] + ) + + assert len(router.model_list) == 1 + assert router.model_list[0]["model_name"] == "libertai-chat" + + def test_libertai_model_modes(self): + """Chat models carry mode 'chat'; the embedding model carries mode 'embedding'.""" + model_cost = litellm.model_cost + + # chat model + assert model_cost["libertai/qwen3.6-27b"]["mode"] == "chat" + + # embedding model (bge-m3) must be normalized to mode 'embedding' so + # /embeddings routing and the supported-endpoints matrix stay consistent + assert "libertai/bge-m3" in model_cost + bge = model_cost["libertai/bge-m3"] + assert bge["litellm_provider"] == "libertai" + assert bge["mode"] == "embedding" + + def test_libertai_supported_endpoints_matrix(self): + """The runtime-served backup matrix (GET /public/supported_endpoints) lists libertai.""" + import json + from pathlib import Path + + import litellm as _litellm + + backup_path = ( + Path(_litellm.__file__).parent / "provider_endpoints_support_backup.json" + ) + matrix = json.loads(backup_path.read_text()) + + assert "libertai" in matrix["providers"] + endpoints = matrix["providers"]["libertai"]["endpoints"] + assert endpoints["chat_completions"] is True + # embeddings is advertised false: the JSON-configured-provider path only + # wires chat routing (the OpenAILike embedding handler is reached solely + # for the literal openai_like/llamafile/lm_studio providers), matching + # the llamagate precedent. bge-m3 stays in the cost map for metadata. + assert endpoints["embeddings"] is False diff --git a/tests/test_litellm/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py b/tests/test_litellm/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py index cc8b14e5514..cf75964ddb7 100644 --- a/tests/test_litellm/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py +++ b/tests/test_litellm/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py @@ -1917,3 +1917,57 @@ class TestVertexAIGlobalLocation: assert "generativelanguage.googleapis.com" in url assert "cachedContents" in url + + +class TestContextCachingMultiRegionUrls: + """Regression coverage for #29571: multi-region vertex_location values + (`eu`, `us`) must resolve to the REP host (`aiplatform.{geo}.rep.googleapis.com`) + on the cachedContents endpoint, matching the inference path (already + fixed in #27293). Previously the URL was hardcoded to + `{location}-aiplatform.googleapis.com`, which doesn't exist for + multi-region locations and 404'd.""" + + def setup_method(self): + self.caching = ContextCachingEndpoints() + + @pytest.mark.parametrize("location", ["eu", "us"]) + def test_vertex_ai_multi_region_uses_rep_host(self, location): + _, url = self.caching._get_token_and_url_context_caching( + gemini_api_key=None, + custom_llm_provider="vertex_ai", + api_base=None, + vertex_project="my-project", + vertex_location=location, + vertex_auth_header="Bearer token", + ) + + assert url.startswith(f"https://aiplatform.{location}.rep.googleapis.com/") + assert f"/locations/{location}/cachedContents" in url + # Old broken host must no longer appear. + assert f"{location}-aiplatform.googleapis.com" not in url + + def test_vertex_ai_regional_still_uses_regional_host(self): + _, url = self.caching._get_token_and_url_context_caching( + gemini_api_key=None, + custom_llm_provider="vertex_ai", + api_base=None, + vertex_project="my-project", + vertex_location="us-central1", + vertex_auth_header="Bearer token", + ) + + assert url.startswith("https://us-central1-aiplatform.googleapis.com/") + assert "/locations/us-central1/cachedContents" in url + + def test_vertex_ai_global_still_uses_global_host(self): + _, url = self.caching._get_token_and_url_context_caching( + gemini_api_key=None, + custom_llm_provider="vertex_ai", + api_base=None, + vertex_project="my-project", + vertex_location="global", + vertex_auth_header="Bearer token", + ) + + assert url.startswith("https://aiplatform.googleapis.com/") + assert "/locations/global/cachedContents" in url diff --git a/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py index d8a674c2681..6c5ccd3562f 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py @@ -409,3 +409,101 @@ async def test_get_customer_daily_activity_with_end_user_aliases(monkeypatch): "end-user-1": {"alias": "Customer One"}, "end-user-2": {"alias": "Customer Two"}, } + + +@pytest.mark.asyncio +async def test_get_customer_daily_activity_non_admin_is_rejected(monkeypatch): + """ + Security regression: any non-admin caller must receive 401 from + /customer/daily/activity and /end_user/daily/activity. + + Before this fix, the endpoint performed no role check. A caller with + user_role=INTERNAL_USER could omit end_user_ids, causing entity_id=None + to flow into get_daily_activity where the SQL builder treats it as no + filter — returning every tenant's spend across the full + LiteLLM_DailyEndUserSpend table. + + LiteLLM_EndUserTable has no per-tenant ownership column, so non-admin + scoping is not possible. The correct fix is admin-only, matching the + existing /customer/list gate. + """ + from litellm.proxy.management_endpoints import customer_endpoints + from litellm.proxy.management_endpoints.customer_endpoints import ( + get_customer_daily_activity, + ) + + mock_prisma_client = MagicMock() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + + get_daily_activity_mock = AsyncMock() + monkeypatch.setattr( + customer_endpoints, "get_daily_activity", get_daily_activity_mock + ) + + non_admin_key = UserAPIKeyAuth( + user_id="regular-user-abc", + user_role=LitellmUserRoles.INTERNAL_USER, + ) + + with pytest.raises(HTTPException) as exc_info: + await get_customer_daily_activity( + end_user_ids=None, + start_date="2025-01-01", + end_date="2025-01-31", + model=None, + api_key=None, + page=1, + page_size=10, + exclude_end_user_ids=None, + user_api_key_dict=non_admin_key, + ) + + assert exc_info.value.status_code == 401 + assert "Admin-only endpoint" in str(exc_info.value.detail) + get_daily_activity_mock.assert_not_called() + + +@pytest.mark.asyncio +async def test_get_customer_daily_activity_service_account_key_is_rejected(monkeypatch): + """ + Security regression: service-account keys (user_id=None, role=INTERNAL_USER) + must be rejected at the admin gate before reaching get_daily_activity. + + A service-account key with end_user_ids omitted is the worst-case caller: + entity_id=None and no user identity to scope by — the SQL builder would + return the full LiteLLM_DailyEndUserSpend table with no WHERE clause. + """ + from litellm.proxy.management_endpoints import customer_endpoints + from litellm.proxy.management_endpoints.customer_endpoints import ( + get_customer_daily_activity, + ) + + mock_prisma_client = MagicMock() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + + get_daily_activity_mock = AsyncMock() + monkeypatch.setattr( + customer_endpoints, "get_daily_activity", get_daily_activity_mock + ) + + service_account_key = UserAPIKeyAuth( + user_id=None, + user_role=LitellmUserRoles.INTERNAL_USER, + ) + + with pytest.raises(HTTPException) as exc_info: + await get_customer_daily_activity( + end_user_ids=None, + start_date="2025-01-01", + end_date="2025-01-31", + model=None, + api_key=None, + page=1, + page_size=10, + exclude_end_user_ids=None, + user_api_key_dict=service_account_key, + ) + + assert exc_info.value.status_code == 401 + assert "Admin-only endpoint" in str(exc_info.value.detail) + get_daily_activity_mock.assert_not_called() diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index f0198320f22..b81807ee19e 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -3064,6 +3064,106 @@ async def test_list_team_v2_org_admin_sees_org_teams(): assert where["organization_id"] == {"in": ["org_A"]} +@pytest.mark.asyncio +async def test_list_team_v2_org_admin_own_user_id_sees_all_org_teams(): + """ + Test that an org admin whose own user_id is sent (as the UI does for + non-Admin roles) still sees all teams in their organization, not just + teams they are a direct member of. + + Regression test for https://github.com/BerriAI/litellm/issues/30215 + """ + from datetime import datetime + from unittest.mock import AsyncMock, Mock, patch + + from fastapi import Request + + from litellm.proxy._types import ( + LiteLLM_OrganizationMembershipTable, + LiteLLM_UserTable, + LitellmUserRoles, + UserAPIKeyAuth, + ) + from litellm.proxy.management_endpoints.team_endpoints import list_team_v2 + + mock_request = Mock(spec=Request) + mock_user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="org_admin_user", + ) + + mock_user = LiteLLM_UserTable( + user_id="org_admin_user", + teams=["team_1"], # direct member of only 1 team + organization_memberships=[ + LiteLLM_OrganizationMembershipTable( + user_id="org_admin_user", + organization_id="org_A", + user_role="org_admin", + spend=0.0, + created_at=datetime.now(), + updated_at=datetime.now(), + ), + ], + ) + + with ( + patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma, + patch("litellm.proxy.proxy_server.user_api_key_cache"), + patch("litellm.proxy.proxy_server.proxy_logging_obj"), + patch( + "litellm.proxy.management_endpoints.team_endpoints.get_user_object", + new_callable=AsyncMock, + return_value=mock_user, + ), + ): + mock_db = Mock() + mock_prisma.db = mock_db + + mock_team_1 = Mock() + mock_team_1.model_dump.return_value = { + "team_id": "team_1", + "team_alias": "Team One", + "organization_id": "org_A", + "members_with_roles": [{"user_id": "org_admin_user", "role": "admin"}], + } + mock_team_2 = Mock() + mock_team_2.model_dump.return_value = { + "team_id": "team_2", + "team_alias": "Team Two", + "organization_id": "org_A", + "members_with_roles": [{"user_id": "other_user", "role": "user"}], + } + mock_db.litellm_teamtable.find_many = AsyncMock( + return_value=[mock_team_1, mock_team_2] + ) + mock_db.litellm_teamtable.count = AsyncMock(return_value=2) + mock_db.litellm_verificationtoken.group_by = AsyncMock(return_value=[]) + + # UI sends the caller's own user_id for non-Admin roles + result = await list_team_v2( + http_request=mock_request, + user_id="org_admin_user", # same as caller — UI sends this + organization_id=None, + team_id=None, + team_alias=None, + user_api_key_dict=mock_user_api_key_dict, + page=1, + page_size=10, + sort_by=None, + sort_order="asc", + status=None, + ) + + assert result["total"] == 2 + assert len(result["teams"]) == 2 + + # Verify the where clause scopes by org only — no team_id filter + where = mock_db.litellm_teamtable.find_many.call_args.kwargs["where"] + assert where["organization_id"] == {"in": ["org_A"]} + assert "team_id" not in where + + @pytest.mark.asyncio async def test_list_team_v2_org_admin_cannot_view_other_orgs(): """ diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 3a1d15ef79c..9e77c6ecc9b 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -359,6 +359,7 @@ ignored_keys = [ "metadata.additional_usage_values.cache_read_input_tokens", "metadata.additional_usage_values.inference_geo", "metadata.additional_usage_values.speed", + "metadata.additional_usage_values.iterations", "metadata.litellm_overhead_time_ms", "metadata.cost_breakdown", "metadata.user_api_key", diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index ec186ffa795..8c28749b1cb 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -24,6 +24,7 @@ from litellm.proxy.common_request_processing import ( _is_azure_model_router_request, _override_openai_response_model, _parse_event_data_for_error, + _UpstreamClosingStreamingResponse, create_response, ) from litellm.proxy.dd_span_tagger import DDSpanTagger @@ -2415,6 +2416,186 @@ class TestHandleLLMApiExceptionDictDetail: assert proxy_exc.code == "500" +class TestStreamCloseOnDisconnect: + """ + Coverage for closing the upstream LLM stream when the client disconnects + mid-stream. Starlette abandons the response body iterator without calling + aclose(), so without these hooks the proxy->backend connection stays open + and the backend (e.g. vLLM) keeps generating into a dead pipe. + """ + + async def test_response_closes_body_iterator_when_task_cancelled(self): + """Cancellation landing in send() leaves the generator suspended at a + yield; only the response-level finally can close it.""" + closed = asyncio.Event() + + async def body(): + try: + while True: + yield "data: x\n\n" + finally: + closed.set() + + response = _UpstreamClosingStreamingResponse( + body(), media_type="text/event-stream" + ) + + async def receive(): + await asyncio.Event().wait() + + async def send(message): + if message["type"] == "http.response.body": + await asyncio.Event().wait() + + task = asyncio.create_task(response({"type": "http"}, receive, send)) + await asyncio.sleep(0.05) + assert not closed.is_set() + + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + assert closed.is_set() + + async def test_response_closes_body_iterator_on_http_disconnect(self): + closed = asyncio.Event() + disconnected = asyncio.Event() + body_sends = 0 + + async def body(): + try: + for i in range(1000): + yield f"data: {i}\n\n" + finally: + closed.set() + + response = _UpstreamClosingStreamingResponse( + body(), media_type="text/event-stream" + ) + + async def receive(): + await disconnected.wait() + return {"type": "http.disconnect"} + + async def send(message): + nonlocal body_sends + if message["type"] == "http.response.body": + body_sends += 1 + if body_sends == 3: + disconnected.set() + await asyncio.sleep(0.05) + + await response({"type": "http"}, receive, send) + + assert closed.is_set() + assert body_sends < 1000 + + async def test_upstream_closed_even_if_body_iterator_aclose_raises(self): + """A BaseException from body_iterator.aclose() (e.g. CancelledError) + must not prevent the upstream generator from being closed.""" + upstream_closed = asyncio.Event() + + class ExplodingIterator: + def __aiter__(self): + return self + + async def __anext__(self): + raise StopAsyncIteration + + async def aclose(self): + raise asyncio.CancelledError() + + async def upstream(): + try: + yield "data: a\n\n" + finally: + upstream_closed.set() + + upstream_gen = upstream() + await upstream_gen.__anext__() + response = _UpstreamClosingStreamingResponse( + ExplodingIterator(), + media_type="text/event-stream", + upstream_generator=upstream_gen, + ) + + async def receive(): + await asyncio.Event().wait() + + async def send(message): + pass + + await response({"type": "http"}, receive, send) + + assert upstream_closed.is_set() + + async def test_create_response_closes_wrapped_generator_on_cancellation(self): + """End to end through create_response: the upstream-facing generator + must be closed even when the body iterator was never started (client + gone before the first chunk could be sent).""" + inner_closed = asyncio.Event() + + async def wrapped(): + try: + while True: + yield "data: a\n\n" + finally: + inner_closed.set() + + response = await create_response( + generator=wrapped(), media_type="text/event-stream", headers={} + ) + + async def receive(): + await asyncio.Event().wait() + + async def send(message): + await asyncio.Event().wait() + + task = asyncio.create_task(response({"type": "http"}, receive, send)) + await asyncio.sleep(0.05) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + assert inner_closed.is_set() + + async def test_async_streaming_data_generator_closes_upstream_on_early_close( + self, + ): + class FakeUpstream: + def __init__(self): + self.aclosed = False + + def __aiter__(self): + return self + + async def __anext__(self): + return {"type": "chunk"} + + async def aclose(self): + self.aclosed = True + + ProxyLogging._callback_capabilities_cache.clear() + upstream = FakeUpstream() + gen = ProxyBaseLLMRequestProcessing.async_streaming_data_generator( + response=upstream, + user_api_key_dict=MagicMock(spec=UserAPIKeyAuth), + request_data={"model": "mock-model"}, + proxy_logging_obj=ProxyLogging(user_api_key_cache=MagicMock()), + serialize_chunk=lambda c: "data: x\n\n", + serialize_error=lambda e: "data: error\n\n", + ) + + await gen.__anext__() + await gen.__anext__() + assert not upstream.aclosed + + await gen.aclose() + + assert upstream.aclosed + + class TestHandleLLMApiExceptionRetryAfter: """RouterRateLimitError cooldown_time must surface as a retry-after header.""" diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 960fca205ce..a0b1676068b 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -2275,3 +2275,28 @@ class TestCacheControlPreservation: assert isinstance(result, list) assert len(result) == 1 assert result[0]["cache_control"] == {"type": "ephemeral"} + + +def test_function_call_tool_id_falls_back_to_unique_id_for_degenerate_call_id(): + """Bedrock Mantle returns a non-unique, index-based ``call_id`` (``call_0`` that + resets every response) alongside a unique ``id`` (``fc_...``). For that degenerate + form the converter must expose the unique ``id``; otherwise every tool call across + an agent's turns collapses to the same id, the agent cannot correlate its tool + results, and it loops re-issuing the same call. A normal (unique) ``call_id`` must + be preserved, since it is the canonical Responses API correlation key. Regression + for the bedrock-mantle gpt-5.5 non-streaming path.""" + from types import SimpleNamespace + + convert = ( + LiteLLMCompletionResponsesConfig.convert_response_function_tool_call_to_chat_completion_tool_call + ) + + mantle = SimpleNamespace( + id="fc_unique_abc123", call_id="call_0", name="get_weather", arguments="{}" + ) + assert convert(mantle)["id"] == "fc_unique_abc123" + + openai = SimpleNamespace( + id="fc_2", call_id="call_tokyo", name="get_weather", arguments="{}" + ) + assert convert(openai)["id"] == "call_tokyo" diff --git a/tests/test_litellm/test_azure_ad_token_credential_resolution.py b/tests/test_litellm/test_azure_ad_token_credential_resolution.py new file mode 100644 index 00000000000..958b236c9b3 --- /dev/null +++ b/tests/test_litellm/test_azure_ad_token_credential_resolution.py @@ -0,0 +1,145 @@ +""" +Regression for #30235. + +``Router.get_deployment_credentials_with_provider`` (router.py:8954) is +used by the proxy's ``/v1/files``, ``/v1/batches`` and passthrough +routing code paths to resolve the upstream credentials for a deployment +by model_id:: + + return CredentialLiteLLMParams( + **deployment.litellm_params.model_dump(exclude_none=True) + ).model_dump(exclude_none=True) + +That re-validation is strict. Any field NOT declared on +``CredentialLiteLLMParams`` gets dropped on the way through, even when +it was present on the original ``litellm_params``. + +Pre-fix, ``azure_ad_token`` was undeclared, so Azure deployments +configured with OAuth/M2M (``azure_ad_token`` in place of ``api_key``) +silently lost their token on every file upload and the proxy returned:: + + Missing credentials. Please pass one of api_key, azure_ad_token, + azure_ad_token_provider, ... + +Tests below pin two things: +1. ``CredentialLiteLLMParams`` directly accepts and round-trips + ``azure_ad_token``. +2. ``Router.get_deployment_credentials_with_provider`` preserves + ``azure_ad_token`` from a deployment's ``litellm_params``. +""" + +from unittest.mock import MagicMock, patch + +import pytest + + +class TestCredentialLiteLLMParamsAzureAdToken: + def test_azure_ad_token_round_trips_through_model_dump(self): + from litellm.types.router import CredentialLiteLLMParams + + params = CredentialLiteLLMParams( + api_base="https://my.openai.azure.com", + api_version="2024-08-01-preview", + azure_ad_token="oauth-bearer-token-xyz", + ) + dumped = params.model_dump(exclude_none=True) + assert dumped["azure_ad_token"] == "oauth-bearer-token-xyz", ( + "azure_ad_token dropped from CredentialLiteLLMParams.model_dump() — " + "every callsite that round-trips litellm_params through this class " + "will lose the token (#30235)" + ) + + def test_azure_ad_token_is_optional(self): + """Adding the field must not break deployments that don't use it + — confirm the default is None and it's excluded by + ``exclude_none``.""" + from litellm.types.router import CredentialLiteLLMParams + + params = CredentialLiteLLMParams(api_key="sk-static") + dumped = params.model_dump(exclude_none=True) + assert "azure_ad_token" not in dumped + assert dumped["api_key"] == "sk-static" + + def test_round_trip_preserves_full_credential_shape(self): + """The Router's get_deployment_credentials_with_provider pattern: + construct from a dict that has azure_ad_token alongside other + fields, dump, expect azure_ad_token to ride through alongside + the other declared fields.""" + from litellm.types.router import CredentialLiteLLMParams + + source = { + "api_base": "https://my.openai.azure.com", + "api_version": "2024-08-01-preview", + "azure_ad_token": "tok-123", + "api_key": None, # M2M deployment has no static key + } + rebuilt = CredentialLiteLLMParams( + **{k: v for k, v in source.items() if v is not None} + ).model_dump(exclude_none=True) + assert rebuilt.get("azure_ad_token") == "tok-123" + assert rebuilt.get("api_base") == "https://my.openai.azure.com" + assert "api_key" not in rebuilt + + +class TestRouterCredentialResolution: + """The actual fix surface: Router.get_deployment_credentials_with_provider + must preserve azure_ad_token on the resolved credentials dict so the + files endpoint can forward it to the Azure files client.""" + + def test_credentials_preserve_azure_ad_token(self): + from litellm import Router + + deployment_id = "azure-m2m-deployment-fixed-uuid" + router = Router( + model_list=[ + { + "model_name": "gpt-4o-azure-m2m", + "litellm_params": { + "model": "azure/gpt-4o", + "api_base": "https://my.openai.azure.com", + "api_version": "2024-08-01-preview", + "azure_ad_token": "tok-azure-m2m-xyz", + }, + "model_info": {"id": deployment_id}, + } + ] + ) + + credentials = router.get_deployment_credentials_with_provider( + model_id=deployment_id + ) + assert credentials is not None + assert credentials.get("azure_ad_token") == "tok-azure-m2m-xyz", ( + "Router credential resolution dropped azure_ad_token; the " + "files / batches / passthrough callers will not be able to " + "authenticate against Azure (#30235)" + ) + + def test_credentials_static_api_key_unaffected(self): + """Don't break the pre-fix happy path: a deployment with a + static api_key (no azure_ad_token) keeps its api_key and + azure_ad_token doesn't appear in the dump.""" + from litellm import Router + + deployment_id = "azure-static-key-deployment-fixed-uuid" + router = Router( + model_list=[ + { + "model_name": "gpt-4o-azure-static", + "litellm_params": { + "model": "azure/gpt-4o", + "api_base": "https://my.openai.azure.com", + "api_version": "2024-08-01-preview", + "api_key": "sk-static-key", + }, + "model_info": {"id": deployment_id}, + } + ] + ) + + credentials = router.get_deployment_credentials_with_provider( + model_id=deployment_id + ) + assert credentials is not None + assert credentials.get("api_key") == "sk-static-key" + assert "azure_ad_token" not in credentials diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index ad08029c2c4..6d9185ffcf2 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -180,6 +180,43 @@ def test_openrouter_qwen36_plus_model_info(): assert model_info["supports_vision"] is True +@pytest.mark.parametrize( + "model", + [ + "github_copilot/mai-code-1-flash", + "github_copilot/mai-code-1-flash-internal", + ], +) +def test_github_copilot_mai_code_1_flash_pricing(model): + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model_info = litellm.model_cost.get(model) + + assert model_info is not None, f"Missing model pricing entry: {model}" + assert model_info["litellm_provider"] == "github_copilot" + assert model_info["mode"] == "chat" + assert model_info["input_cost_per_token"] == 7.5e-07 + assert model_info["cache_read_input_token_cost"] == 7.5e-08 + assert model_info["output_cost_per_token"] == 4.5e-06 + assert model_info["supported_endpoints"] == ["/v1/chat/completions"] + + prompt_usd, completion_usd = cost_per_token( + model=model, + prompt_tokens=1000, + completion_tokens=500, + custom_llm_provider="github_copilot", + usage_object=Usage( + prompt_tokens=1000, + completion_tokens=500, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=200), + ), + ) + + assert prompt_usd == pytest.approx((800 * 7.5e-07) + (200 * 7.5e-08)) + assert completion_usd == pytest.approx(500 * 4.5e-06) + + def test_cost_calculator_with_usage(monkeypatch): os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") @@ -385,7 +422,7 @@ def test_handle_realtime_stream_cost_calculation(): ) assert cost == 0.0 # No usage, no cost - + def test_realtime_stream_combines_text_and_audio_token_details(): """Realtime response.done usage with input_token_details / output_token_details.""" from litellm.cost_calculator import RealtimeAPITokenUsageProcessor diff --git a/tests/test_litellm/test_register_model_zero_cost_persistence.py b/tests/test_litellm/test_register_model_zero_cost_persistence.py new file mode 100644 index 00000000000..15c8721a7f1 --- /dev/null +++ b/tests/test_litellm/test_register_model_zero_cost_persistence.py @@ -0,0 +1,187 @@ +""" +Regression for #30198. + +``register_model`` calls ``get_model_info(key)`` to fetch the existing +entry, then ``_update_dictionary`` merges its own ``value`` over it and +the result is written back into ``litellm.model_cost``. + +``_get_model_info_helper`` synthesizes ``input_cost_per_token`` and +``output_cost_per_token`` as 0 when the cost keys are missing from the +raw entry (the "price unknown" and "free" cases share the same +representation). So on the SECOND ``register_model`` call against an +already-present sparse entry (e.g. router model id with only +``{"id": ..., "db_model": True}``), the synthesized zeros get written +back, and the entry flips from "no cost keys" → "cost keys = 0". + +That defeats ``_is_cost_explicitly_configured`` (added in #24949), which +checks whether the cost keys are present in the raw entry — after the +write-back they are. ``_is_model_cost_zero`` then returns ``True`` and +``common_checks`` skips every tag / key / team / user / org budget check +for the group. Spend keeps recording (cost calc resolves by model name), +so the symptom is silent: requests that should 429 keep returning 200. + +Tests below replicate the Router-built-twice scenario from the report +and confirm the sparse entry stays sparse. +""" + +import importlib +import os +import sys +from typing import Any, Dict + +import pytest + + +@pytest.fixture(autouse=True) +def _restore_model_cost(): + import litellm + + original = dict(litellm.model_cost) + try: + yield + finally: + litellm.model_cost.clear() + litellm.model_cost.update(original) + + +def _sparse_router_value(model_cost_key: str) -> Dict[str, Any]: + # Mirrors what Router builds for a db_model deployment with no custom + # pricing (litellm/router.py:_create_deployment). + return { + "model_name": "gpt-4o-mini", + "litellm_params": { + "model": "gpt-4o-mini", + "custom_llm_provider": "openai", + "api_key": "sk-test", + }, + "model_info": {"id": model_cost_key, "db_model": True}, + } + + +def test_first_registration_leaves_sparse_entry_without_cost_keys(): + """First ``register_model`` call against an unknown key must NOT add + cost keys to the entry — otherwise the very first registration would + already poison the map.""" + import litellm + + key = "fixed-uuid-30198-first" + litellm.model_cost.pop(key, None) + + litellm.register_model({key: {"litellm_provider": "openai"}}) + + entry = litellm.model_cost.get(key, {}) + assert "input_cost_per_token" not in entry, entry + assert "output_cost_per_token" not in entry, entry + + +def test_second_registration_does_not_persist_synthesized_zero_costs(): + """The #30198 bug: re-registering the same sparse entry made + ``get_model_info`` synthesize cost = 0 and write it back. Verify the + entry stays clean after a second pass.""" + import litellm + + key = "fixed-uuid-30198-double-register" + litellm.model_cost.pop(key, None) + + payload = {key: {"litellm_provider": "openai"}} + litellm.register_model(payload) + litellm.register_model(payload) + + entry = litellm.model_cost.get(key, {}) + assert "input_cost_per_token" not in entry, ( + "second register_model() persisted a synthesized zero " + "input_cost_per_token; this disables budget enforcement" + ) + assert "output_cost_per_token" not in entry, ( + "second register_model() persisted a synthesized zero " + "output_cost_per_token; this disables budget enforcement" + ) + + +def test_explicit_zero_cost_in_value_is_preserved(): + """If the caller actually wants the model marked free, the explicit + zero must survive the dedup. The fix must only strip SYNTHESIZED + zeros, not caller-provided ones.""" + import litellm + + key = "fixed-uuid-30198-explicit-zero" + litellm.model_cost.pop(key, None) + + litellm.register_model( + { + key: { + "litellm_provider": "openai", + "input_cost_per_token": 0, + "output_cost_per_token": 0, + } + } + ) + + entry = litellm.model_cost[key] + assert entry["input_cost_per_token"] == 0 + assert entry["output_cost_per_token"] == 0 + + # Re-registering with the same explicit zeros must keep them. + litellm.register_model( + { + key: { + "litellm_provider": "openai", + "input_cost_per_token": 0, + "output_cost_per_token": 0, + } + } + ) + entry = litellm.model_cost[key] + assert entry["input_cost_per_token"] == 0 + assert entry["output_cost_per_token"] == 0 + + +def test_real_pricing_for_known_model_survives_re_registration(): + """A model with built-in pricing (e.g. gpt-4o-mini) must keep its + real per-token rates across repeated registrations of an empty + payload that names the same key.""" + import litellm + + base_in = litellm.model_cost["gpt-4o-mini"]["input_cost_per_token"] + base_out = litellm.model_cost["gpt-4o-mini"]["output_cost_per_token"] + assert base_in > 0 and base_out > 0 + + litellm.register_model({"gpt-4o-mini": {"litellm_provider": "openai"}}) + litellm.register_model({"gpt-4o-mini": {"litellm_provider": "openai"}}) + + assert litellm.model_cost["gpt-4o-mini"]["input_cost_per_token"] == base_in + assert litellm.model_cost["gpt-4o-mini"]["output_cost_per_token"] == base_out + + +def test_router_double_init_keeps_db_model_entry_sparse(): + """End-to-end repro from the issue body: building Router twice on + the same model_list must not flip the per-deployment entry to + cost=0. This is the exact production symptom (#30198).""" + import litellm + from litellm import Router + + deployment_id = "fixed-uuid-30198-router-init" + litellm.model_cost.pop(deployment_id, None) + + model_list = [_sparse_router_value(deployment_id)] + + Router(model_list=model_list) + after_first = dict(litellm.model_cost.get(deployment_id, {})) + + Router(model_list=model_list) + after_second = dict(litellm.model_cost.get(deployment_id, {})) + + # Cost keys must not appear AT ALL on a sparse db_model deployment + # (matches the pre-bug shape) — the bug rewrites them as 0. + for snapshot, label in ( + (after_first, "first Router()"), + (after_second, "second Router()"), + ): + assert "input_cost_per_token" not in snapshot, ( + f"{label} persisted input_cost_per_token={snapshot.get('input_cost_per_token')!r} " + f"on a sparse db_model entry; this disables budget enforcement" + ) + assert "output_cost_per_token" not in snapshot, ( + f"{label} persisted output_cost_per_token={snapshot.get('output_cost_per_token')!r} " + f"on a sparse db_model entry; this disables budget enforcement" + ) diff --git a/tests/test_litellm/test_responses_streaming_container_ownership.py b/tests/test_litellm/test_responses_streaming_container_ownership.py new file mode 100644 index 00000000000..07cc309798b --- /dev/null +++ b/tests/test_litellm/test_responses_streaming_container_ownership.py @@ -0,0 +1,261 @@ +""" +Regression for #30210. + +When streaming /v1/responses goes through the proxy + Router, the +streaming iterator is wrapped by ``Router._aresponses_streaming_iterator`` +which returns ``FallbackResponsesStreamWrapper``. That wrapper set +``self.completed_response = None`` in __init__ and never updated it, +so the proxy's container-ownership hook (which reads +``getattr(stream_response, "completed_response", None)`` via +``ProxyBaseLLMRequestProcessing._extract_completed_responses_response``) +saw None on every streaming call and silently recorded nothing — +follow-up ``GET /v1/containers//files`` then 403'd for the very +key that created the container. + +Tests below construct the wrapper from a fake async generator that +yields one terminal ``response.completed`` chunk and assert the +wrapper now carries that chunk on ``completed_response`` so the +proxy hook can walk it. +""" + +import asyncio +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest + + +def _make_wrapper_class(): + """Pull ``FallbackResponsesStreamWrapper`` out by running + ``Router._aresponses_streaming_iterator`` long enough to construct + the class then return it. Mirrors how the wrapper is actually + instantiated in production.""" + from litellm.router import Router + from litellm.responses.streaming_iterator import ( + BaseResponsesAPIStreamingIterator, + ) + + # Minimal source iterator stub with every attribute the wrapper + # copies in __init__ (see router.py:2552-2583). + source = SimpleNamespace( + response=None, + model="openai/gpt-5.5", + logging_obj=None, + responses_api_provider_config=None, + start_time=None, + litellm_metadata=None, + custom_llm_provider="openai", + request_data={}, + call_type="aresponses", + _hidden_params={}, + ) + + # The class is defined inside _aresponses_streaming_iterator; capture + # it by patching FallbackResponsesStreamWrapper into a sentinel on + # construction. + captured = {} + + real_router_module = __import__("litellm.router", fromlist=["Router"]) + + async def _drive(): + async def empty_gen(): + if False: + yield # pragma: no cover + return + + router = Router( + model_list=[ + { + "model_name": "openai/gpt-5.5", + "litellm_params": {"model": "openai/gpt-5.5", "api_key": "sk-test"}, + } + ] + ) + wrapped = await router._aresponses_streaming_iterator( + response=source, # type: ignore[arg-type] + initial_kwargs={}, + ) + captured["wrapper_cls"] = type(wrapped) + captured["instance"] = wrapped + + asyncio.run(_drive()) + return captured["wrapper_cls"], captured["instance"] + + +def _terminal_chunk(event_type: str): + """A SimpleNamespace shaped like the openai responses-api terminal + event chunks the wrapper inspects (.type attribute).""" + return SimpleNamespace( + type=event_type, + response=SimpleNamespace( + id="resp_test", + output=[], + container={"id": "cntr_test", "type": "code_interpreter"}, + ), + ) + + +def _non_terminal_chunk(event_type: str = "response.output_text.delta"): + return SimpleNamespace(type=event_type, delta="hello") + + +class TestStreamWrapperCapturesTerminalEvent: + def test_terminal_completed_event_is_recorded_on_wrapper(self): + """The #30210 bug: a forwarded ``response.completed`` chunk used + to leave ``completed_response`` at None on the wrapper. Verify + it now carries the chunk.""" + wrapper_cls, _ = _make_wrapper_class() + + async def gen(): + yield _non_terminal_chunk() + yield _terminal_chunk("response.completed") + + wrapper = wrapper_cls(gen()) + # Drain the wrapper. + out = asyncio.run(_drain(wrapper)) + assert len(out) == 2 + assert wrapper.completed_response is not None, ( + "FallbackResponsesStreamWrapper.completed_response is still None " + "after a response.completed chunk passed through — the proxy " + "container-ownership hook will 403 follow-up file lookups (#30210)" + ) + assert wrapper.completed_response.type == "response.completed" + + def test_terminal_incomplete_event_is_recorded(self): + wrapper_cls, _ = _make_wrapper_class() + + async def gen(): + yield _terminal_chunk("response.incomplete") + + wrapper = wrapper_cls(gen()) + asyncio.run(_drain(wrapper)) + assert wrapper.completed_response is not None + assert wrapper.completed_response.type == "response.incomplete" + + def test_terminal_failed_event_is_recorded(self): + wrapper_cls, _ = _make_wrapper_class() + + async def gen(): + yield _terminal_chunk("response.failed") + + wrapper = wrapper_cls(gen()) + asyncio.run(_drain(wrapper)) + assert wrapper.completed_response is not None + assert wrapper.completed_response.type == "response.failed" + + def test_non_terminal_chunks_do_not_set_completed_response(self): + wrapper_cls, _ = _make_wrapper_class() + + async def gen(): + yield _non_terminal_chunk("response.output_text.delta") + yield _non_terminal_chunk("response.code_interpreter.in_progress") + + wrapper = wrapper_cls(gen()) + asyncio.run(_drain(wrapper)) + assert ( + wrapper.completed_response is None + ), "non-terminal chunks must not set completed_response" + + def test_first_terminal_event_wins(self): + """Real streams only emit one terminal event, but defend against + future producers emitting more: keep the first one (the inner + source iterator behaves the same way).""" + wrapper_cls, _ = _make_wrapper_class() + + first = _terminal_chunk("response.completed") + first.response.id = "resp_first" + second = _terminal_chunk("response.completed") + second.response.id = "resp_second" + + async def gen(): + yield first + yield second + + wrapper = wrapper_cls(gen()) + asyncio.run(_drain(wrapper)) + assert wrapper.completed_response.response.id == "resp_first" + + +class TestProxyOwnershipHookReadsCompletedResponse: + """End-to-end: the proxy hook reads exactly the attribute the + wrapper now populates. Pin that the helper still extracts the + response correctly so the ownership recording path doesn't break.""" + + def test_extract_returns_inner_response_object(self): + from litellm.proxy.common_request_processing import ( + ProxyBaseLLMRequestProcessing, + ) + + wrapper_cls, _ = _make_wrapper_class() + + async def gen(): + yield _terminal_chunk("response.completed") + + wrapper = wrapper_cls(gen()) + asyncio.run(_drain(wrapper)) + + extracted = ProxyBaseLLMRequestProcessing._extract_completed_responses_response( + wrapper + ) + assert extracted is not None + assert extracted.id == "resp_test" + assert extracted.container["id"] == "cntr_test" + + +class TestSilentSkipNowLogged: + """Reporter's secondary ask: when completed_response is None, the + ownership hook silently dropped on the floor. Make sure the new + warning fires so operators see a hint instead of a mute 403.""" + + def test_warning_logged_when_completed_response_missing(self, caplog): + import logging + from litellm.proxy.common_request_processing import ( + ProxyBaseLLMRequestProcessing, + ) + + # Wrap a generator that produces NO terminal event so the + # wrapper stays at completed_response=None — same shape as the + # pre-fix bug. + wrapper_cls, _ = _make_wrapper_class() + + async def gen(): + yield _non_terminal_chunk() + + wrapper = wrapper_cls(gen()) + + async def driver(): + async def inner_gen(): + async for c in wrapper: + yield c + + # Patch _record_container_owners_from_responses_if_needed to + # a noop async so the warning branch is exercised in + # isolation. + with patch.object( + ProxyBaseLLMRequestProcessing, + "_record_container_owners_from_responses_if_needed", + new=MagicMock(), + ): + wrapped = ProxyBaseLLMRequestProcessing._wrap_responses_stream_for_container_ownership( + original_stream_response=wrapper, + wrapped_generator=inner_gen(), + user_api_key_dict=MagicMock(), + ) + async for _ in wrapped: + pass + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + asyncio.run(driver()) + + assert any( + "Container ownership recording skipped on streaming /v1/responses" + in r.message + for r in caplog.records + ), "silent-skip warning never fired despite completed_response=None" + + +async def _drain(it): + out = [] + async for chunk in it: + out.append(chunk) + return out diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index d95a918a3b0..ef1b6e669f8 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -24993,6 +24993,8 @@ export interface components { aws_region_name?: string | null; /** Aws Secret Access Key */ aws_secret_access_key?: string | null; + /** Azure Ad Token */ + azure_ad_token?: string | null; /** Budget Duration */ budget_duration?: string | null; /** Cache Creation Input Audio Token Cost */ @@ -32632,6 +32634,8 @@ export interface components { aws_region_name?: string | null; /** Aws Secret Access Key */ aws_secret_access_key?: string | null; + /** Azure Ad Token */ + azure_ad_token?: string | null; /** Budget Duration */ budget_duration?: string | null; /** Cache Creation Input Audio Token Cost */