diff --git a/enterprise/enterprise_hooks/google_text_moderation.py b/enterprise/enterprise_hooks/google_text_moderation.py index 7dbbc9ed6ce..ddf092a143e 100644 --- a/enterprise/enterprise_hooks/google_text_moderation.py +++ b/enterprise/enterprise_hooks/google_text_moderation.py @@ -15,7 +15,10 @@ import litellm from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import UserAPIKeyAuth -from litellm.proxy.guardrails._content_utils import iter_message_text +from litellm.proxy.guardrails._content_utils import ( + is_non_conversational_request, + iter_message_text, +) from litellm.types.utils import CallTypesLiteral _THRESHOLD_VALUE: Final = TypeAdapter(object) @@ -94,7 +97,7 @@ class ENTERPRISE_GoogleTextModeration(CustomLogger): async def async_moderation_hook( self, - data: dict, + data: dict[str, object], user_api_key_dict: UserAPIKeyAuth, call_type: CallTypesLiteral, ): @@ -102,6 +105,9 @@ class ENTERPRISE_GoogleTextModeration(CustomLogger): - Calls Google's Text Moderation API - Rejects request if it fails safety check """ + if is_non_conversational_request(call_type, data): + verbose_proxy_logger.debug("Google text moderation: skipping non-conversational call type %s", call_type) + return # Covers multimodal list content + Responses-API input. text = "".join(iter_message_text(data)) if text: diff --git a/enterprise/litellm_enterprise/enterprise_callbacks/callback_controls.py b/enterprise/litellm_enterprise/enterprise_callbacks/callback_controls.py index bf99e1a72ca..6a97a9de414 100644 --- a/enterprise/litellm_enterprise/enterprise_callbacks/callback_controls.py +++ b/enterprise/litellm_enterprise/enterprise_callbacks/callback_controls.py @@ -33,9 +33,8 @@ class EnterpriseCallbackControls: try: disabled_callbacks = EnterpriseCallbackControls.get_disabled_callbacks(litellm_params, standard_callback_dynamic_params) - verbose_logger.debug(f"Dynamically disabled callbacks from {X_LITELLM_DISABLE_CALLBACKS}: {disabled_callbacks}") - verbose_logger.debug(f"Checking if {callback} is disabled via headers. Disable callbacks from headers: {disabled_callbacks}") if disabled_callbacks is not None: + verbose_logger.debug(f"Checking if {callback} is in dynamically disabled callbacks: {disabled_callbacks}") ######################################################### # premium user check ######################################################### diff --git a/litellm-rust/crates/llms-types/src/formats/audio_transcription/response.rs b/litellm-rust/crates/llms-types/src/formats/audio_transcription/response.rs index 624eb7be4e4..211d4d0f648 100644 --- a/litellm-rust/crates/llms-types/src/formats/audio_transcription/response.rs +++ b/litellm-rust/crates/llms-types/src/formats/audio_transcription/response.rs @@ -1,14 +1,31 @@ use serde_json::Value; +#[macro_rules_attribute::apply(crate::wire_type)] +pub struct AudioTranscriptionUsage { + pub input_tokens: u64, + pub output_tokens: u64, + pub total_tokens: u64, +} + #[macro_rules_attribute::apply(crate::wire_type)] pub struct AudioTranscriptionResponseData { pub text: String, + pub usage: Option, } impl AudioTranscriptionResponseData { pub fn into_json(self) -> Value { - serde_json::json!({ - "text": self.text, - }) + match self.usage { + None => serde_json::json!({"text": self.text}), + Some(usage) => serde_json::json!({ + "text": self.text, + "usage": { + "type": "tokens", + "input_tokens": usage.input_tokens, + "output_tokens": usage.output_tokens, + "total_tokens": usage.total_tokens, + }, + }), + } } } diff --git a/litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs b/litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs index 0f8703f223b..1c976c1f73d 100644 --- a/litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs +++ b/litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs @@ -6,8 +6,8 @@ use litellm_auth_aws::{ }; use litellm_core_utils::core_helpers::json_type_name; use litellm_llms_types::{ - formats::audio_transcription::AudioTranscriptionResponseData, - providers::bedrock::ConverseResponse, + formats::audio_transcription::{AudioTranscriptionResponseData, AudioTranscriptionUsage}, + providers::bedrock::{ConverseResponse, ConverseUsage}, }; use serde::Deserialize; use serde_json::{Map, Value, json}; @@ -101,10 +101,10 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig { if let Some(prompt) = optional_string(&optional_params, "prompt") { instruction.push_str(&format!(" Additional context: {prompt}")); } - let mut inference_config = Map::from_iter([("maxTokens".to_string(), json!(4096))]); - if let Some(temperature) = optional_params.get("temperature") { - inference_config.insert("temperature".to_string(), temperature.clone()); - } + let temperature = optional_params + .get("temperature") + .cloned() + .unwrap_or(json!(0)); Ok(AudioTranscriptionRequestData { body: json!({ "messages": [{ @@ -114,8 +114,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig { {"text": instruction} ] }], - "system": [{"text": "You are a transcription assistant."}], - "inferenceConfig": inference_config, + "inferenceConfig": {"maxTokens": 4096, "temperature": temperature}, }), }) } @@ -125,6 +124,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig { _model: &str, response_json: Value, ) -> Result { + let reports_usage = response_json.get("usage").is_some(); let response: ConverseResponse = serde_json::from_value(response_json).map_err(|error| { Error::InvalidResponse(ErrorDetail::invalid("Converse response", error)) @@ -137,6 +137,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig { } Ok(AudioTranscriptionResponseData { text: response.content_text(), + usage: reports_usage.then(|| transcription_usage(&response.usage)), }) } @@ -191,22 +192,41 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig { } } +fn transcription_usage(usage: &ConverseUsage) -> AudioTranscriptionUsage { + AudioTranscriptionUsage { + input_tokens: usage.input_tokens, + output_tokens: usage.output_tokens, + total_tokens: usage + .total_tokens + .unwrap_or(usage.input_tokens + usage.output_tokens), + } +} + #[cfg(test)] mod tests { use super::*; + use rstest::rstest; fn no_env(_: &str) -> Option { None } - #[test] - fn request_matches_python_shape() { - let params = Map::from_iter([ - ("language".to_string(), json!("en")), - ("prompt".to_string(), json!("Speaker names")), - ("temperature".to_string(), json!(0)), - ("timestamp_granularities".to_string(), json!(["word"])), - ]); + #[rstest] + #[case::caller_temperature(Some(json!(0.4)), json!(0.4))] + #[case::default_temperature(None, json!(0))] + fn request_sends_audio_and_instruction_without_system_block( + #[case] temperature: Option, + #[case] expected_temperature: Value, + ) { + let params = Map::from_iter( + [ + ("language".to_string(), json!("en")), + ("prompt".to_string(), json!("Speaker names")), + ("timestamp_granularities".to_string(), json!(["word"])), + ] + .into_iter() + .chain(temperature.map(|value| ("temperature".to_string(), value))), + ); let params = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG.map_transcription_params(¶ms); let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG .transform_audio_transcription_request( @@ -225,22 +245,39 @@ mod tests { {"text": "Transcribe the audio. Respond with only the transcript. The audio language is en. Additional context: Speaker names"} ] }], - "system": [{"text": "You are a transcription assistant."}], - "inferenceConfig": {"maxTokens": 4096, "temperature": 0} + "inferenceConfig": {"maxTokens": 4096, "temperature": expected_temperature} }) ); } - #[test] - fn response_concatenates_content_blocks() { + #[rstest] + #[case::reported_total( + json!({"inputTokens": 390, "outputTokens": 60, "totalTokens": 450}), + json!({"text": "hello world", "usage": {"type": "tokens", "input_tokens": 390, "output_tokens": 60, "total_tokens": 450}}) + )] + #[case::derived_total( + json!({"inputTokens": 1, "outputTokens": 2}), + json!({"text": "hello world", "usage": {"type": "tokens", "input_tokens": 1, "output_tokens": 2, "total_tokens": 3}}) + )] + fn response_carries_text_and_converse_usage(#[case] usage: Value, #[case] expected: Value) { let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG .transform_audio_transcription_response( "model", - json!({"output": {"message": {"content": [{"text": "hello "}, {"text": "world"}]}}, "usage": {"inputTokens": 1, "outputTokens": 2}}), + json!({"output": {"message": {"content": [{"text": "hello "}, {"text": "world"}]}}, "usage": usage}), ) .expect("response"); - assert_eq!(result.text, "hello world"); - assert_eq!(result.into_json(), json!({"text": "hello world"})); + assert_eq!(result.into_json(), expected); + } + + #[rstest] + fn response_without_usage_reports_none() { + let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG + .transform_audio_transcription_response( + "model", + json!({"output": {"message": {"content": [{"text": "hello"}]}}}), + ) + .expect("response"); + assert_eq!(result.into_json(), json!({"text": "hello"})); } #[test] diff --git a/litellm/__init__.py b/litellm/__init__.py index 44d5fcdeb68..b22240b6c04 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -681,6 +681,7 @@ moonshot_models: Set = set() publicai_models: Set = set() darkbloom_models: Set = set() openinfer_models: Final[Set[str]] = set() +streamlake_models: Final[Set[str]] = set() zerogpu_models: Final[Set[str]] = set() v0_models: Set = set() vispark_models: Set[str] = set() @@ -971,6 +972,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje darkbloom_models.add(key) elif value.get("litellm_provider") == "openinfer": openinfer_models.add(key) + elif value.get("litellm_provider") == "streamlake": + streamlake_models.add(key) elif value.get("litellm_provider") == "zerogpu": zerogpu_models.add(key) elif value.get("litellm_provider") == "v0": @@ -1148,6 +1151,7 @@ model_list = list( | publicai_models | darkbloom_models | openinfer_models + | streamlake_models | zerogpu_models | v0_models | vispark_models @@ -1269,6 +1273,7 @@ def _build_models_by_provider() -> dict: "publicai": publicai_models, "darkbloom": darkbloom_models, "openinfer": openinfer_models, + "streamlake": streamlake_models, "zerogpu": zerogpu_models, "v0": v0_models, "vispark": vispark_models, diff --git a/litellm/constants.py b/litellm/constants.py index a79028494f6..65bd7a7b5a7 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1065,6 +1065,7 @@ openai_compatible_endpoints: Final[list[str]] = [ "https://api.reka.ai/v1", "https://gigachat.devices.sberbank.ru/api/v1", "https://api.openinfer.ai/v1", + "https://vanchin.streamlake.ai/api/gateway/v1/endpoints", ] @@ -1147,6 +1148,7 @@ openai_compatible_providers: Final[list[str]] = [ "prism", "sail", "reka", + "streamlake", ] OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS: Final = frozenset({"openai"} | frozenset(openai_compatible_providers)) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 49e75620c4e..ce872d7a085 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -693,7 +693,7 @@ def cost_per_token( elif call_type == "atranscription" or call_type == "transcription": if _transcription_usage_has_token_details(usage_block): return generic_cost_per_token( - model=model_without_prefix, + model=model, usage=usage_block, custom_llm_provider=custom_llm_provider, service_tier=service_tier, diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index 8178b7e98b8..8792fd0f58a 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -77,6 +77,7 @@ OPTIONAL_KWARGS_KEYS: Final = ( "otpm", "tpd", "rpd", + "daily_reset_time", "use_xai_oauth", "chatgpt_auth_file", "github_copilot_auth_type", diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 8b66c27b930..620c8413f29 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1130,9 +1130,6 @@ class Logging(LiteLLMLoggingBaseClass): } self.litellm_request_debug = litellm_params.get("litellm_request_debug", False) self.logger_fn = litellm_params.get("logger_fn", None) - if is_debugging_on() or self.litellm_request_debug: - verbose_logger.debug("self.optional_params: %s", self.optional_params) - self.model_call_details.update( { "model": self.model, diff --git a/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py b/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py index a02c40b7611..78a8cb51bac 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py +++ b/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py @@ -27,13 +27,18 @@ class TranscriptionUsageObjectTransformation: if isinstance(usage_object, TranscriptionUsageDurationObject): return None elif isinstance(usage_object, TranscriptionUsageTokensObject): + input_token_details: Final = usage_object.input_token_details return Usage( prompt_tokens=usage_object.input_tokens, completion_tokens=usage_object.output_tokens, total_tokens=usage_object.total_tokens, - prompt_tokens_details=PromptTokensDetailsWrapper( - text_tokens=usage_object.input_token_details.text_tokens, - audio_tokens=usage_object.input_token_details.audio_tokens, + prompt_tokens_details=( + None + if input_token_details is None + else PromptTokensDetailsWrapper( + text_tokens=input_token_details.text_tokens, + audio_tokens=input_token_details.audio_tokens, + ) ), ) return None diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 2348d8e57d6..f2f06d801a8 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -92,6 +92,18 @@ def _next_sync_or_exhausted(it: Iterator[object]) -> object: return _SYNC_ITER_EXHAUSTED +def _stamp_usage_cost( + usage_chunk: ModelResponse | TextCompletionResponse | ModelResponseStream, + complete_response: ModelResponse | TextCompletionResponse | ModelResponseStream, +) -> None: + bill: Final = (get_hidden_params(complete_response) or {}).get("response_cost") + usage: Final[object] = getattr(usage_chunk, "usage", None) + if usage is None or isinstance(bill, bool) or not isinstance(bill, (int, float)): + return + if bill > 0 or isinstance(getattr(usage, "cost", None), (int, float)): + setattr(usage, "cost", float(bill)) # noqa: B010 # cost is an optional extra on the usage model + + def _stamp_served_service_tier(response: ModelResponseStream, complete_streaming_response: ModelResponse) -> None: served_tier: Final = complete_streaming_response.model_dump().get("service_tier") if isinstance(served_tier, str) and served_tier: @@ -2141,6 +2153,7 @@ class CustomStreamWrapper: getattr(complete_streaming_response, "usage"), ) _stamp_served_service_tier(response, complete_streaming_response) + _stamp_usage_cost(response, complete_streaming_response) try: _cache_copy = complete_streaming_response.model_copy(deep=True) _log_copy = complete_streaming_response.model_copy(deep=True) @@ -2361,6 +2374,7 @@ class CustomStreamWrapper: getattr(complete_streaming_response, "usage"), ) _stamp_served_service_tier(response, complete_streaming_response) + _stamp_usage_cost(response, complete_streaming_response) try: _copy = complete_streaming_response.model_copy(deep=True) except RuntimeError: diff --git a/litellm/llms/azure/common_utils.py b/litellm/llms/azure/common_utils.py index 4bd19889bcc..6bb6d584385 100644 --- a/litellm/llms/azure/common_utils.py +++ b/litellm/llms/azure/common_utils.py @@ -621,7 +621,7 @@ class BaseAzureLLM(BaseOpenAILLM): # For Azure v1 API, use standard OpenAI client instead of AzureOpenAI # See: https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#api-specs - if self._is_azure_v1_api_version(api_version): + if self.is_azure_v1_api_version(api_version): # Extract only params that OpenAI client accepts # Always use /openai/v1/ regardless of whether user passed "v1", "latest", or "preview" # The OpenAI client accepts a callable for `api_key` and re-invokes it @@ -922,7 +922,7 @@ class BaseAzureLLM(BaseOpenAILLM): else: new_url = api_base - if BaseAzureLLM._is_azure_v1_api_version(api_version): + if BaseAzureLLM.is_azure_v1_api_version(api_version): # ensure the request go to /openai/v1 and not just /openai if "/openai/v1" not in new_url: parsed_url: Final = httpx.URL(new_url) @@ -952,7 +952,7 @@ class BaseAzureLLM(BaseOpenAILLM): Returns None when ``api_version`` is a dated one, which still uses the deployment route. """ - if not BaseAzureLLM._is_azure_v1_api_version(api_version): + if not BaseAzureLLM.is_azure_v1_api_version(api_version): return None base_url: Final = httpx.URL(api_base) @@ -970,7 +970,7 @@ class BaseAzureLLM(BaseOpenAILLM): ) @staticmethod - def _is_azure_v1_api_version(api_version: str | None) -> bool: + def is_azure_v1_api_version(api_version: str | None) -> bool: if api_version is None: return False return api_version in {"preview", "latest", "v1"} diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index 647ad97e02c..cdeeef1d96c 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -54,6 +54,16 @@ def _resolve_responses_api_version(api_base: str | None, litellm_params: Mapping return api_version +def _v1_responses_url(api_base: str) -> httpx.URL: + parsed_url: Final = httpx.URL(api_base) + path: Final = parsed_url.path.rstrip("/") + root: Final = next( + (path.removesuffix(route) for route in ("/openai/v1/responses", "/openai/responses") if path.endswith(route)), + path, + ) + return parsed_url.copy_with(path=f"{root}/openai/v1/responses", query=None) + + def _with_responses_api_version(api_base: str | None, litellm_params: Mapping[str, object]) -> Mapping[str, object]: return MappingProxyType({**litellm_params, "api_version": _resolve_responses_api_version(api_base, litellm_params)}) @@ -247,20 +257,20 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): if api_base is None: raise ValueError("api_base is required for Azure WebSocket") - parsed_url: Final = httpx.URL(api_base) - path = parsed_url.path.rstrip("/") - # Strip existing /openai/responses path if the api_base already contains it - for suffix in ("/openai/v1/responses", "/openai/responses"): - if path.endswith(suffix): - path = path[: -len(suffix)] - break - scheme: Final = "wss" if parsed_url.scheme == "https" else "ws" - return str(parsed_url.copy_with(scheme=scheme, path=f"{path}/openai/v1/responses", query=None)) + v1_url: Final = _v1_responses_url(api_base) + return str(v1_url.copy_with(scheme="wss" if v1_url.scheme == "https" else "ws")) def model_in_websocket_url(self) -> bool: # Azure sends the model in the response.create body, not the URL return False + def compact_url(self, api_base: str) -> str: + params: Final = httpx.URL(api_base).params + v1_params: Final = ( + params if BaseAzureLLM.is_azure_v1_api_version(params.get("api-version")) else params.remove("api-version") + ) + return super().compact_url(str(_v1_responses_url(api_base).copy_with(params=v1_params))) + ######################################################### ########## DELETE RESPONSE API TRANSFORMATION ############## ######################################################### diff --git a/litellm/llms/bedrock/audio_transcription/__init__.py b/litellm/llms/bedrock/audio_transcription/__init__.py index 7f7a138398b..bc74e9c38ab 100644 --- a/litellm/llms/bedrock/audio_transcription/__init__.py +++ b/litellm/llms/bedrock/audio_transcription/__init__.py @@ -1,7 +1,11 @@ import base64 +import math +from collections.abc import Mapping +from dataclasses import dataclass from typing import Final import httpx +from pydantic import BaseModel, ConfigDict from litellm.litellm_core_utils.audio_utils.utils import process_audio_file from litellm.rust_bridge import runtime @@ -14,7 +18,72 @@ from litellm.rust_bridge.transcription.native import ( RustAtranscription, RustTranscription, ) -from litellm.types.utils import FileTypes, TranscriptionResponse +from litellm.secret_managers.main import get_secret_str +from litellm.types.utils import FileTypes, TranscriptionResponse, TranscriptionUsageTokensObject + +SUPPORTED_OPENAI_TRANSCRIPTION_PARAMS: Final = ("language", "prompt", "temperature", "response_format") +TEXT_RESPONSE_FORMATS: Final = frozenset({"json", "text"}) + + +def _as_float(value: object) -> float | None: + if isinstance(value, bool) or not isinstance(value, (int, float, str)): + return None + try: + number: Final = float(value) + except ValueError: + return None + return number if math.isfinite(number) else None + + +def _rejection(key: str, value: object) -> str | None: + if key not in SUPPORTED_OPENAI_TRANSCRIPTION_PARAMS: + return f"bedrock transcription does not support {key}." + if key == "response_format" and value not in TEXT_RESPONSE_FORMATS: + return f"bedrock transcription returns only the transcript text, so response_format={value!r} is not supported." + if key == "temperature" and _as_float(value) is None: + return f"bedrock transcription needs a number for temperature, got {value!r}." + return None + + +@dataclass(frozen=True, slots=True) +class UnsupportedTranscriptionParams: + message: str + + +def map_openai_transcription_params( + non_default_params: Mapping[str, object], drop_params: bool +) -> Mapping[str, object] | UnsupportedTranscriptionParams: + rejections: Final = { + key: reason for key, value in non_default_params.items() if (reason := _rejection(key, value)) is not None + } + if rejections and not drop_params: + return UnsupportedTranscriptionParams( + message=f"{' '.join(rejections.values())} To drop it from the call, set `litellm.drop_params = True`." + ) + return { + key: _as_float(value) if key == "temperature" else value + for key, value in non_default_params.items() + if key not in rejections + } + + +class _RustTranscriptionResult(BaseModel): + model_config = ConfigDict(frozen=True) + + text: str + usage: TranscriptionUsageTokensObject | None = None + + +def _transcription_response( + result: Mapping[str, object], optional_params: Mapping[str, object] +) -> TranscriptionResponse: + parsed: Final = _RustTranscriptionResult.model_validate(result) + response: Final = TranscriptionResponse(text=parsed.text).model_copy(update={"usage": parsed.usage}) + region_name: Final = ( + optional_params.get("aws_region_name") or get_secret_str("AWS_REGION_NAME") or get_secret_str("AWS_REGION") + ) + response.hidden_params = {**response.hidden_params, "region_name": region_name} + return response class BedrockAudioTranscriptionRustDispatch: @@ -64,7 +133,7 @@ class BedrockAudioTranscriptionRustDispatch: "timeout_seconds": timeout_to_seconds(timeout), } call: Final = NativeCall(args=(), kwargs=fields, base={}) - return TranscriptionResponse(**rust(call)) + return _transcription_response(rust(call), optional_params) return runtime.run( RouteContext(Route.TRANSCRIPTION, provider=custom_llm_provider, model=model), @@ -97,7 +166,7 @@ class BedrockAudioTranscriptionRustDispatch: "timeout_seconds": timeout_to_seconds(timeout), } call: Final = NativeCall(args=(), kwargs=fields, base={}) - return TranscriptionResponse(**await rust(call)) + return _transcription_response(await rust(call), optional_params) return await runtime.arun( RouteContext(Route.TRANSCRIPTION, provider=custom_llm_provider, model=model), diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index d37a12a91d6..559daf767c2 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -32,11 +32,11 @@ import copy import json import time import uuid -from collections.abc import Mapping, MutableSequence, Sequence +from collections.abc import Iterable, Mapping, MutableSequence, Sequence from dataclasses import dataclass -from itertools import accumulate, chain, repeat +from itertools import accumulate, chain, groupby, repeat from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, Union, cast +from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, TypeVar, Union, cast from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict @@ -55,7 +55,9 @@ from litellm.llms.base_llm.guardrail_translation.base_translation import ( ) from litellm.llms.base_llm.guardrail_translation.utils import ( blocked_responses_stream_usage, + effective_scan_only_tool_results_for_guardrail, effective_skip_system_message_for_guardrail, + effective_skip_tool_message_for_guardrail, list_field, merge_guardrailed_scoped_messages, role_out_of_guardrail_scope, @@ -65,7 +67,10 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( stream_item_items, unappliable_request_rewrite, ) -from litellm.llms.openai.responses.guardrail_translation.tool_merge import merge_guardrailed_tools +from litellm.llms.openai.responses.guardrail_translation.tool_merge import ( + append_returned_tools, + merge_guardrailed_tools, +) from litellm.responses.litellm_completion_transformation.compaction_items import ( compaction_summary, decode_compaction_item, @@ -74,7 +79,9 @@ from litellm.responses.litellm_completion_transformation.compaction_items import from litellm.responses.litellm_completion_transformation.custom_tools import without_malformed_tool_flags from litellm.responses.litellm_completion_transformation.tool_search import is_server_tool_search_tool from litellm.responses.litellm_completion_transformation.transformation import ( + TOOL_CALL_OUTPUT_ITEM_TYPES, LiteLLMCompletionResponsesConfig, + tool_output_to_chat_content, ) from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import ( @@ -270,6 +277,7 @@ _TOOL_CALL_PAYLOAD_EVENT_TYPES: Final = _TOOL_CALL_PAYLOAD_DELTA_EVENT_TYPES | f ) _OUTPUT_ITEM_EVENT_TYPES: Final = frozenset({"response.output_item.added", "response.output_item.done"}) _OUTPUT_TEXT_EVENT_TYPES: Final = frozenset({"response.output_text.delta", "response.output_text.done"}) +_TOOL_OUTPUT_TEXT_PART_TYPES: Final = frozenset({"input_text", "output_text", "text"}) _PATCHABLE_ITEM_FIELDS: Final[Mapping[str, str]] = MappingProxyType( {"function_call_output": "output", "custom_tool_call_output": "output", "message": "content"} ) @@ -519,10 +527,109 @@ def scannable_instructions(data: Mapping[str, object], *, skip_system: bool = Fa def _input_item_role(item: object) -> str: + if stream_item_field(item, "type") in TOOL_CALL_OUTPUT_ITEM_TYPES: + return "tool" role: Final = item.get("role") if isinstance(item, Mapping) else None return role.lower() if isinstance(role, str) else "" +def _scanned_text_field(item: Mapping[str, object]) -> str: + return "output" if item.get("type") in TOOL_CALL_OUTPUT_ITEM_TYPES else "content" + + +def _tool_output_part_text(part: object) -> str | None: + if not isinstance(part, dict): + return None + fields: Final = _item_fields_adapter.validate_python(part) + text: Final = fields.get("text") + return text if fields.get("type") in _TOOL_OUTPUT_TEXT_PART_TYPES and isinstance(text, str) else None + + +def tool_output_texts(item: object) -> tuple[tuple[int | None, str], ...]: + if not stream_item_field(item, "call_id") or stream_item_field(item, "type") == "computer_call_output": + return () + parts: Final = stream_item_items(item, "output") + part_texts: Final = tuple( + (index, text) for index, part in enumerate(parts) if (text := _tool_output_part_text(part)) + ) + if part_texts: + return part_texts + tool_content: Final = tool_output_to_chat_content(stream_item_field(item, "output")) + return ((None, tool_content),) if isinstance(tool_content, str) and tool_content else () + + +_Grouped: Final = TypeVar("_Grouped") + + +def _pair_call_id(pair: tuple[str, object]) -> str: + return pair[0] + + +def _grouped_by_call_id(pairs: Iterable[tuple[str, _Grouped]]) -> Mapping[str, tuple[_Grouped, ...]]: + ordered: Final = sorted(pairs, key=_pair_call_id) + return MappingProxyType( + {call_id: tuple(value for _, value in group) for call_id, group in groupby(ordered, key=_pair_call_id)} + ) + + +def _texts_by_tool_row( + messages: Sequence[Mapping[str, object]], input_data: "str | ResponseInputParam" +) -> Mapping[int, tuple[str, ...]]: + if isinstance(input_data, str): + return MappingProxyType({}) + output_texts_by_call_id: Final = _grouped_by_call_id( + (str(stream_item_field(item, "call_id")), tuple(text for _, text in tool_output_texts(item))) + for item in input_data + if stream_item_field(item, "type") in TOOL_CALL_OUTPUT_ITEM_TYPES + ) + row_indices_by_call_id: Final = _grouped_by_call_id( + (str(message.get("tool_call_id")), index) + for index, message in enumerate(messages) + if message.get("role") == "tool" + ) + return MappingProxyType( + dict( + chain.from_iterable( + zip(indices, output_texts_by_call_id[call_id], strict=True) + for call_id, indices in row_indices_by_call_id.items() + if len(indices) == len(output_texts_by_call_id.get(call_id, ())) + ) + ) + ) + + +def _is_string_tool_row(message: Mapping[str, object]) -> bool: + return message.get("role") == "tool" and isinstance(message.get("content"), str) + + +def _split_into_text_parts(message: AllMessageValues, texts: tuple[str, ...]) -> AllMessageValues: + if not _is_string_tool_row(message) or len(texts) < 2: + return message + return cast( # cast-ok: a tool row whose content is its text parts as chat text blocks + "AllMessageValues", {**message, "content": [{"type": "text", "text": text} for text in texts]} + ) + + +def _with_split_rows_restored( + rows: Sequence[AllMessageValues], split_rows: Sequence[tuple[AllMessageValues, AllMessageValues]] +) -> Sequence[AllMessageValues]: + if not isinstance(rows, list): + return rows + return [next((original for split, original in split_rows if row == split), row) for row in rows] + + +def _is_textless_tool_row(message: Mapping[str, object], texts: tuple[str, ...] | None) -> bool: + return _is_string_tool_row(message) and (not message.get("content") or texts == ()) + + +def _rewritten_whole_text(item: object, field: str, guardrail_response: str) -> object | None: + if isinstance(stream_item_field(item, field), str): + return guardrail_response + if tool_output_texts(item) == ((None, guardrail_response),): + return None + return guardrail_response + + def _patched_request_fields( raw_input: object, instructions: object, @@ -636,17 +743,28 @@ class OpenAIResponsesHandler(BaseTranslation): Handles both string input and list of message objects. """ - input_data: Final[str | ResponseInputParam | None] = data.get("input") - if not isinstance(input_data, (str, list)): + raw_input: Final[str | ResponseInputParam | None] = data.get("input") + if not isinstance(raw_input, (str, list)): return data + input_data: Final = cast("str | ResponseInputParam", raw_input) # cast-ok: fields read via stream_item_field skip_system: Final = effective_skip_system_message_for_guardrail(guardrail_to_apply) + skip_tool: Final = effective_skip_tool_message_for_guardrail(guardrail_to_apply) + scan_only: Final = effective_scan_only_tool_results_for_guardrail(guardrail_to_apply) structured_messages: Final = self.get_structured_messages(data) - scoped_indices: Final = scoped_structured_message_indices( - structured_messages or [], scan_only_tool_results=False, skip_system=skip_system, skip_tool=False + all_messages: Final = structured_messages or [] + texts_by_tool_row: Final = _texts_by_tool_row(all_messages, input_data) + scoped_indices: Final = tuple( + index + for index in scoped_structured_message_indices( + all_messages, scan_only_tool_results=scan_only, skip_system=skip_system, skip_tool=skip_tool + ) + if not _is_textless_tool_row(all_messages[index], texts_by_tool_row.get(index)) ) - scoped_structured_messages: Final = ( - [structured_messages[index] for index in scoped_indices] if structured_messages else None + split_rows: Final = tuple( + (_split_into_text_parts(all_messages[index], texts_by_tool_row.get(index, ())), all_messages[index]) + for index in scoped_indices ) + scoped_structured_messages: Final = [split for split, _ in split_rows] if structured_messages else None raw_tools: Final = data.get("tools") original_tools: Final[tuple[Mapping[str, object], ...]] = ( tuple(raw_tools) if isinstance(raw_tools, list) else () @@ -661,7 +779,12 @@ class OpenAIResponsesHandler(BaseTranslation): ) ) extracted: Final = self._extract_guardrail_inputs( - data, input_data, flattened_tool_groups, skip_system=skip_system + data, + input_data, + () if scan_only else flattened_tool_groups, + skip_system=skip_system, + skip_tool=skip_tool, + scan_only=scan_only, ) if not extracted.inputs.get("texts"): return data @@ -674,13 +797,14 @@ class OpenAIResponsesHandler(BaseTranslation): logging_obj=litellm_logging_obj, ) self._apply_guardrailed_tools_to_data( - data, original_tools, flattened_tool_groups, guardrailed_inputs.get("tools") + data, original_tools, flattened_tool_groups, guardrailed_inputs.get("tools"), scan_only=scan_only ) written_back: Final = self._written_back_request_fields( data, structured_messages or (), scoped_indices, scoped_structured_messages, + tuple((split, original) for split, original in split_rows if split is not original), guardrail_to_apply, guardrailed_inputs, ) @@ -730,8 +854,10 @@ class OpenAIResponsesHandler(BaseTranslation): flattened_tool_groups: Sequence[Sequence[Mapping[str, object]]], *, skip_system: bool = False, + skip_tool: bool = False, + scan_only: bool = False, ) -> _ExtractedInputs: - instructions: Final = scannable_instructions(data, skip_system=skip_system) + instructions: Final = scannable_instructions(data, skip_system=skip_system or scan_only) texts_to_check: Final[list[str]] = [] if instructions is None else [instructions] images_to_check: Final[list[str]] = [] task_mappings: Final[list[tuple[int, int | None]]] = [] @@ -745,11 +871,15 @@ class OpenAIResponsesHandler(BaseTranslation): ) ) if isinstance(input_data, str): - texts_to_check.append(input_data) + if not scan_only: + texts_to_check.append(input_data) else: for msg_idx, message in enumerate(input_data): if role_out_of_guardrail_scope( - _input_item_role(message), skip_system_message=skip_system, skip_tool_message=False + _input_item_role(message), + skip_system_message=skip_system, + skip_tool_message=skip_tool, + scan_only_tool_results=scan_only, ): continue self._extract_input_text_and_images( @@ -775,12 +905,14 @@ class OpenAIResponsesHandler(BaseTranslation): structured_messages: Sequence[AllMessageValues], scoped_indices: Sequence[int], scoped_structured_messages: Sequence[AllMessageValues] | None, + split_rows: Sequence[tuple[AllMessageValues, AllMessageValues]], guardrail_to_apply: "CustomGuardrail", guardrailed_inputs: GenericGuardrailAPIInputs, ) -> _RequestFields | None: - guardrailed: Final = guardrailed_inputs.get("structured_messages") - if guardrailed is None or guardrailed is scoped_structured_messages: + returned: Final = guardrailed_inputs.get("structured_messages") + if returned is None or returned is scoped_structured_messages: return None + guardrailed: Final = _with_split_rows_restored(returned, split_rows) covers_full_request: Final = len(scoped_indices) == len(structured_messages) or ( guardrail_to_apply.structured_messages_cover_full_request() and len(guardrailed) == len(structured_messages) ) @@ -814,11 +946,14 @@ class OpenAIResponsesHandler(BaseTranslation): original_tools: Sequence[Mapping[str, object]], flattened_tool_groups: Sequence[Sequence[Mapping[str, object]]], guardrailed_tools: Sequence[ChatCompletionToolParam] | None, + *, + scan_only: bool, ) -> None: if guardrailed_tools is None: return + merge: Final = append_returned_tools if scan_only else merge_guardrailed_tools data["tools"] = list( # rebind-ok: in-place request rewrite - merge_guardrailed_tools(original_tools, flattened_tool_groups, guardrailed_tools) + merge(original_tools, flattened_tool_groups, guardrailed_tools) ) def _extract_input_text_and_images( @@ -839,6 +974,11 @@ class OpenAIResponsesHandler(BaseTranslation): texts_to_check.append(summary) task_mappings.append((msg_idx, None)) return + if message.get("type") in TOOL_CALL_OUTPUT_ITEM_TYPES: + for content_idx, text in tool_output_texts(message): + texts_to_check.append(text) + task_mappings.append((msg_idx, content_idx)) + return content: Final = message.get("content", None) if content is None: return @@ -890,18 +1030,20 @@ class OpenAIResponsesHandler(BaseTranslation): ) continue - content = messages[msg_idx].get("content", None) + field = _scanned_text_field(messages[msg_idx]) + content = messages[msg_idx].get(field, None) if content is None: continue - if isinstance(content, str) and content_idx_optional is None: - # Replace string content with guardrail response - messages[msg_idx]["content"] = guardrail_response + if content_idx_optional is None: + rewritten = _rewritten_whole_text(messages[msg_idx], field, guardrail_response) + if rewritten is not None: + messages[msg_idx][field] = rewritten - elif isinstance(content, list) and content_idx_optional is not None: + elif isinstance(content, list): # Replace specific text item in list content - if isinstance(messages[msg_idx]["content"][content_idx_optional], dict): - messages[msg_idx]["content"][content_idx_optional]["text"] = guardrail_response + if isinstance(messages[msg_idx][field][content_idx_optional], dict): + messages[msg_idx][field][content_idx_optional]["text"] = guardrail_response async def process_output_response( self, diff --git a/litellm/llms/openai/responses/guardrail_translation/tool_merge.py b/litellm/llms/openai/responses/guardrail_translation/tool_merge.py index f295a864b71..e529cba4edf 100644 --- a/litellm/llms/openai/responses/guardrail_translation/tool_merge.py +++ b/litellm/llms/openai/responses/guardrail_translation/tool_merge.py @@ -6,6 +6,7 @@ from typing import Final, TypeAlias from pydantic import BaseModel, TypeAdapter, ValidationError from litellm._logging import verbose_logger +from litellm.llms.base_llm.guardrail_translation.utils import merge_returned_tools_into_request_tools, openai_tool_name from litellm.responses.litellm_completion_transformation.custom_tools import custom_tool_grammar_suffix from litellm.responses.litellm_completion_transformation.transformation import ( NAMESPACE_DESCRIPTION_SEPARATOR, @@ -183,3 +184,18 @@ def merge_guardrailed_tools( tuple(tool for key, tool in zip(guardrailed_keys, guardrailed) if key not in owned_keys) ) return tuple(chain(merged_originals, appended)) + + +def append_returned_tools( + original_tools: Sequence[Tool], + flattened_groups: Sequence[Sequence[Tool]], + returned_tools: Iterable[object], +) -> tuple[Tool, ...]: + request_chat_tools: Final = tuple(chain.from_iterable(flattened_groups)) + additions: Final = merge_returned_tools_into_request_tools( + request_tools=request_chat_tools, returned_tools=_validated_tools(returned_tools), tool_name=openai_tool_name + )[len(request_chat_tools) :] + return ( + *original_tools, + *LiteLLMCompletionResponsesConfig.transform_chat_completion_tool_params_to_responses_api_tools(additions), + ) diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 9cb074a177e..033326e653f 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -66,6 +66,29 @@ _PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS: Final = frozenset({LlmProviders.AZURE, _PROVIDERS_REPLAYING_ONLY_THEIR_OWN_REASONING: Final = _PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS _ROLES_READING_PROMPT_CACHE_BREAKPOINTS: Final = frozenset({"system", "developer", "user"}) _PART_TYPES_READING_PROMPT_CACHE_BREAKPOINTS: Final = frozenset({"input_text", "input_image", "input_file"}) +_COMPACT_TRACKING_PARAMS: Final = frozenset({"metadata", "user"}) +_COMPACT_REJECTED_PARAMS: Final = frozenset( + { + "background", + "context_management", + "max_output_tokens", + "max_tool_calls", + "moderation", + "partial_images", + "prompt", + "safety_identifier", + "store", + "stream_options", + "temperature", + "tool_choice", + "top_logprobs", + "top_p", + "truncation", + } +) +_PROVIDERS_WITH_STRICT_COMPACT_PARAMS: Final = frozenset( + {LlmProviders.OPENAI, LlmProviders.AZURE, LlmProviders.AZURE_AI} +) _INPUT_ITEM: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) _CONTENT_PARTS: Final[TypeAdapter[Sequence[Mapping[str, object]]]] = TypeAdapter(Sequence[Mapping[str, object]]) @@ -1182,6 +1205,17 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): ######################################################### ########## COMPACT RESPONSE API TRANSFORMATION ########## ######################################################### + def compact_url(self, api_base: str) -> str: + parsed_url: Final = httpx.URL(api_base) + return str(parsed_url.copy_with(path=parsed_url.path.rstrip("/") + "/compact")) + + def compact_dropped_params(self, litellm_params: GenericLiteLLMParams) -> frozenset[str]: + if self.custom_llm_provider not in _PROVIDERS_WITH_STRICT_COMPACT_PARAMS: + return frozenset() + if _drops_params(litellm_params): + return _COMPACT_TRACKING_PARAMS | _COMPACT_REJECTED_PARAMS + return _COMPACT_TRACKING_PARAMS + def transform_compact_response_api_request( self, model: str, @@ -1197,10 +1231,8 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): OpenAI API expects the following request - POST /v1/responses/compact """ - # Preserve query params (e.g., api-version) while appending /compact. - parsed_url: Final = httpx.URL(api_base) - compact_path: Final = parsed_url.path.rstrip("/") + "/compact" - url: Final = str(parsed_url.copy_with(path=compact_path)) + url: Final = self.compact_url(api_base) + dropped_params: Final = self.compact_dropped_params(litellm_params) request_params: Final = cast( # cast-ok: the base signature leaves the params bag untyped "Mapping[str, object]", response_api_optional_request_params @@ -1214,14 +1246,11 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): ) if sanitized_tools is not None: response_api_optional_request_params["tools"] = sanitized_tools - data: Final = cast( # cast-ok: typed dict spread is inferred as Unknown by basedpyright - dict[str, object], - { - "model": model, - "input": replay_safe_input, - **response_api_optional_request_params, - }, - ) + data: Final = { + "model": model, + "input": replay_safe_input, + **{key: value for key, value in request_params.items() if key not in dropped_params}, + } return url, data diff --git a/litellm/llms/openai_like/dynamic_config.py b/litellm/llms/openai_like/dynamic_config.py index 04b93a94df6..85f25573987 100644 --- a/litellm/llms/openai_like/dynamic_config.py +++ b/litellm/llms/openai_like/dynamic_config.py @@ -110,6 +110,15 @@ def create_config_class(provider: SimpleProviderConfig): return super()._transform_messages(messages=transformed, model=model, is_async=True) return super()._transform_messages(messages=transformed, model=model, is_async=False) + def _should_preserve_cache_control_for_endpoint( + self, + custom_llm_provider: str | None, + api_base: str | None, + ) -> bool: + return provider.special_handling.get( + "preserve_cache_control", False + ) or super()._should_preserve_cache_control_for_endpoint(custom_llm_provider, api_base) + def _get_openai_compatible_provider_info( self, api_base: str | None, api_key: str | None ) -> tuple[str | None, str | None]: @@ -243,7 +252,7 @@ def create_config_class(provider: SimpleProviderConfig): return JSONProviderConfig -_responses_config_cache: Final[dict] = {} +_responses_config_cache: Final[dict[str, "type[OpenAILikeResponsesConfig]"]] = {} def create_responses_config_class(provider: SimpleProviderConfig) -> "type[OpenAILikeResponsesConfig]": diff --git a/litellm/llms/openai_like/json_loader.py b/litellm/llms/openai_like/json_loader.py index d3474c931a9..6e99b9e7fec 100644 --- a/litellm/llms/openai_like/json_loader.py +++ b/litellm/llms/openai_like/json_loader.py @@ -3,6 +3,7 @@ JSON-based provider configuration loader for OpenAI-compatible providers. """ import json +from collections.abc import Mapping from pathlib import Path from typing import Final @@ -12,7 +13,7 @@ from litellm._logging import verbose_logger from litellm.types.utils import ProviderSpecificModelInfo _PROVIDER_INFO_ADAPTER: Final = TypeAdapter(ProviderSpecificModelInfo) -_SPECIAL_HANDLING_ADAPTER: Final = TypeAdapter(dict[str, bool]) +_SPECIAL_HANDLING_ADAPTER: Final = TypeAdapter(Mapping[str, bool]) class SimpleProviderConfig: @@ -21,12 +22,14 @@ class SimpleProviderConfig: def __init__(self, slug: str, data: dict): self.slug = slug self.base_url = data["base_url"] - self.api_key_env = data["api_key_env"] - self.api_base_env = data.get("api_base_env") + self.api_key_env: str = data["api_key_env"] + self.api_base_env: str | None = data.get("api_base_env") self.base_class = data.get("base_class", "openai_gpt") self.param_mappings = data.get("param_mappings", {}) self.constraints = data.get("constraints", {}) - self.special_handling: Final = _SPECIAL_HANDLING_ADAPTER.validate_python(data.get("special_handling", {})) + self.special_handling: Mapping[str, bool] = _SPECIAL_HANDLING_ADAPTER.validate_python( + data.get("special_handling", {}) + ) self.supported_endpoints = data.get("supported_endpoints", []) self.provider_info: ProviderSpecificModelInfo = ( _PROVIDER_INFO_ADAPTER.validate_python(data["provider_info"]) if "provider_info" in data else {} diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index 4ce070985f2..1b59c5ed78b 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -270,6 +270,22 @@ "/v1/responses" ] }, + "streamlake": { + "base_url": "https://vanchin.streamlake.ai/api/gateway/v1/endpoints", + "api_key_env": "STREAMLAKE_API_KEY", + "api_base_env": "STREAMLAKE_API_BASE", + "param_mappings": { + "max_tokens": "max_completion_tokens" + }, + "provider_info": { + "supports_function_calling": true, + "supports_reasoning": true + }, + "special_handling": { + "preserve_cache_control": true + }, + "supported_endpoints": ["/v1/chat/completions"] + }, "vispark": { "base_url": "https://api.lab.vispark.in/v1", "api_key_env": "VISPARK_LAB_API_KEY", diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 525514161d3..9fa54a8d335 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -41939,6 +41939,120 @@ "supports_system_messages": true, "supports_native_structured_output": true }, + "bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 4.12e-08, + "input_cost_per_token_batches": 2.06e-08, + "input_cost_per_token_flex": 2.06e-08, + "input_cost_per_token_priority": 7.21e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 4.12e-08, + "output_cost_per_token_batches": 2.06e-08, + "output_cost_per_token_flex": 2.06e-08, + "output_cost_per_token_priority": 7.21e-08, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/ap-southeast-2/mistral.voxtral-small-24b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 1.03e-07, + "input_cost_per_token_batches": 5.15e-08, + "input_cost_per_token_flex": 5.15e-08, + "input_cost_per_token_priority": 1.8025e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 3.09e-07, + "output_cost_per_token_batches": 1.545e-07, + "output_cost_per_token_flex": 1.545e-07, + "output_cost_per_token_priority": 5.4075e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/ap-northeast-1/mistral.voxtral-mini-3b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 5e-08, + "input_cost_per_token_batches": 3e-08, + "input_cost_per_token_flex": 3e-08, + "input_cost_per_token_priority": 9e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 5e-08, + "output_cost_per_token_batches": 3e-08, + "output_cost_per_token_flex": 3e-08, + "output_cost_per_token_priority": 9e-08, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-northeast-1/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/ap-northeast-1/mistral.voxtral-small-24b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 1.2e-07, + "input_cost_per_token_batches": 6e-08, + "input_cost_per_token_flex": 6e-08, + "input_cost_per_token_priority": 2.1e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 3.6e-07, + "output_cost_per_token_batches": 1.8e-07, + "output_cost_per_token_flex": 1.8e-07, + "output_cost_per_token_priority": 6.3e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-northeast-1/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/eu-west-2/mistral.voxtral-mini-3b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 6e-08, + "input_cost_per_token_batches": 3e-08, + "input_cost_per_token_flex": 3e-08, + "input_cost_per_token_priority": 1.1e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 6e-08, + "output_cost_per_token_batches": 3e-08, + "output_cost_per_token_flex": 3e-08, + "output_cost_per_token_priority": 1.1e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/eu-west-2/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/eu-west-2/mistral.voxtral-small-24b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 1.6e-07, + "input_cost_per_token_batches": 8e-08, + "input_cost_per_token_flex": 8e-08, + "input_cost_per_token_priority": 2.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 4.7e-07, + "output_cost_per_token_batches": 2.3e-07, + "output_cost_per_token_flex": 2.3e-07, + "output_cost_per_token_priority": 8.2e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/eu-west-2/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, "mistral/codestral-2508": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 3e-07, @@ -61374,6 +61488,359 @@ "supports_tool_choice": true, "supports_vision": true }, + "streamlake/deepseek-v4-flash-0731": { + "cache_read_input_token_cost": 1.4e-08, + "input_cost_per_token": 4.4e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 1.32e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/DeepSeek-V4-Pro-0813": { + "cache_read_input_token_cost": 4.4e-08, + "input_cost_per_token": 1.32e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 1048566, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 3.96e-06, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/DeepSeek-V3": { + "cache_read_input_token_cost": 1.14e-07, + "input_cost_per_token": 2.86e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 56000, + "max_output_tokens": 8000, + "mode": "chat", + "output_cost_per_token": 1.143e-06, + "supports_function_calling": true, + "supports_reasoning": false, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/GLM-5.3": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_assistant_prefill": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/GLM-5.3-Flash": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_assistant_prefill": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/Kimi-K2-Instruct-0905": { + "cache_read_input_token_cost": 1.43e-07, + "input_cost_per_token": 5.71e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 224000, + "max_output_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 2.286e-06, + "supports_function_calling": true, + "supports_reasoning": false, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-r1": { + "cache_read_input_token_cost": 2.29e-07, + "input_cost_per_token": 5.71e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 98304, + "max_output_tokens": 16000, + "mode": "chat", + "output_cost_per_token": 2.286e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-v3.1": { + "cache_read_input_token_cost": 2.29e-07, + "input_cost_per_token": 5.71e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 98304, + "max_output_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 1.714e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-v4-flash": { + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 1.4e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 2.8e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-v4-pro": { + "cache_read_input_token_cost": 1.45e-07, + "input_cost_per_token": 1.74e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 3.48e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-v4.1-flash-1017": { + "cache_read_input_token_cost": 6e-09, + "input_cost_per_token": 3e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-v4.1-flash-260910": { + "cache_read_input_token_cost": 6e-09, + "input_cost_per_token": 3e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 1048568, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/glm-5": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 202752, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3.2e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/glm-5.1": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 202752, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/glm-5.2": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 1048566, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/kimi-k2.6": { + "cache_read_input_token_cost": 1.6e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 262144, + "max_output_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 4e-06, + "supports_function_calling": true, + "supports_vision": true, + "supports_reasoning": false, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/kimi-k2.7-code": { + "cache_read_input_token_cost": 1.9e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 262144, + "max_output_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 4e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/mimo-v2.6-flash": { + "cache_read_input_token_cost": 2.8e-09, + "input_cost_per_token": 1.4e-07, + "litellm_provider": "streamlake", + "mode": "chat", + "output_cost_per_token": 2.8e-07, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/mimo-v2.6-pro": { + "cache_read_input_token_cost": 3.6e-09, + "input_cost_per_token": 4.35e-07, + "litellm_provider": "streamlake", + "mode": "chat", + "output_cost_per_token": 8.7e-07, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/minimax-m2.5": { + "cache_creation_input_token_cost": 3.75e-07, + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 196608, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/minimax-m3": { + "cache_read_input_token_cost": 1.2e-07, + "cache_read_input_token_cost_above_512k_tokens": 2.4e-07, + "input_cost_per_token": 6e-07, + "input_cost_per_token_above_512k_tokens": 1.2e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 1000000, + "max_output_tokens": 512000, + "mode": "chat", + "output_cost_per_token": 2.4e-06, + "output_cost_per_token_above_512k_tokens": 4.8e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/qwen3.5-397b-a17b": { + "cache_read_input_token_cost": 1.2e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 258048, + "max_output_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/qwen3.6-27b": { + "input_cost_per_token": 6e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 229376, + "max_output_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "supports_function_calling": true, + "supports_vision": true, + "supports_reasoning": false, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/qwen3.6-35b-a3b": { + "input_cost_per_token": 3.75e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 229376, + "max_output_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.25e-06, + "supports_function_calling": true, + "supports_vision": true, + "supports_reasoning": false, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/qwen3.8-2.4t-a95b": { + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 983616, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 6e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "cache_creation_input_token_cost": 2.5e-06, + "explicit_cache_read_input_token_cost": 1.7e-07, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/qwen3.8-27b": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 5e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 983616, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "cache_creation_input_token_cost": 6.25e-07, + "explicit_cache_read_input_token_cost": 5e-08, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, "deepseek-v4-flash": { "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 6e-09, @@ -62689,6 +63156,14 @@ "litellm_provider": "anthropic" } }, + { + "name": "typesafe-jev-ids", + "pattern": "^jev-(?:latest|preview|1\\.13\\.0|1\\.14\\.0)$", + "description": "A bare Jev id that has a typesafe/ price entry (jev-latest, jev-preview, jev-1.13.0, jev-1.14.0), the name the /typesafe pass-through forwards to TypeSafe. Listed by name, matched case-insensitively like every rule here, so foo/jev-latest or a model group named like a Jev id TypeSafe does not serve, such as jev-1.0 or jev-internal-router, never resolves to typesafe. A new Jev version goes here together with its price entry. Provider inference feeds the proxy's provider/* wildcard access checks, so this rule is what lets typesafe/* on a key, team or project allowlist cover a jev-latest request.", + "model_info": { + "litellm_provider": "typesafe" + } + }, { "name": "claude-family-baseline", "pattern": "claude-[a-z]+-\\d+(?:[-.]\\d+)?(?:-\\d{8})?", diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index a42ce69a75f..819aca98338 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -3785,6 +3785,32 @@ "generateContent": true } }, + "streamlake": { + "display_name": "StreamLake (`streamlake`)", + "url": "https://docs.litellm.ai/docs/providers/openai_compatible", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, "tencent": { "display_name": "Tencent TokenHub (`tencent`)", "url": "https://docs.litellm.ai/docs/providers/tencent", diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 448643ef59b..1ead470bbd2 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -8075,7 +8075,6 @@ async def vector_store_access_check( rag_ids: Final = (rag_vector_store_id,) if rag_vector_store_id is not None else () vector_store_ids_to_run: Final = tuple(dict.fromkeys((*registry_ids, *rag_ids))) if not vector_store_ids_to_run: - verbose_proxy_logger.debug("Vector store to run not found, skipping vector store access check") return True ######################################################### diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 89886c43525..792efa480ec 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -2581,7 +2581,6 @@ class ProxyBaseLLMRequestProcessing: # have mutated `self.data` in place, and the audit-trail snapshot taken in # add_litellm_data_to_request predates that mutation. refresh_proxy_server_request_body_snapshot(self.data, guardrails_applied=True) - verbose_proxy_logger.debug("receiving data: %s", self.data) if "messages" in self.data and self.data["messages"]: logging_obj.update_messages(self.data["messages"]) diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index c4c3f0ad634..56cd14d987d 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -845,5 +845,3 @@ def _add_vector_store_id_from_path(request_data: dict, request: Request) -> None "populate_request_with_path_params: Updated request_data with vector_store_ids=%s", request_data.get("vector_store_ids"), ) - else: - verbose_proxy_logger.debug("populate_request_with_path_params: No vector_store_id present in path=%s", path) diff --git a/litellm/proxy/guardrails/_content_utils.py b/litellm/proxy/guardrails/_content_utils.py index 9bea22acd71..51934c99db6 100644 --- a/litellm/proxy/guardrails/_content_utils.py +++ b/litellm/proxy/guardrails/_content_utils.py @@ -62,6 +62,20 @@ def is_non_conversational_call_type(call_type: str) -> bool: return call_type in NON_CONVERSATIONAL_CALL_TYPES +_PASSTHROUGH_EMBEDDINGS_ENDPOINT: Final = re.compile(r"/?(?:[\w-]+/)*embeddings/?") + + +def is_non_conversational_request(call_type: str, data: Mapping[str, object]) -> bool: + if is_non_conversational_call_type(call_type): + return True + endpoint: Final = data.get("endpoint") + return ( + call_type == "allm_passthrough_route" + and isinstance(endpoint, str) + and _PASSTHROUGH_EMBEDDINGS_ENDPOINT.fullmatch(endpoint) is not None + ) + + TEXT_PART_TYPES: Final[frozenset[str]] = frozenset( {"text", "input_text", "output_text", "summary_text", "reasoning_text"} ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index adda465ad95..9fa37e1827e 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -49,6 +49,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, bedrock_bearer_token, run_aws_signing from litellm.llms.bedrock.guardrail_attachments import find_request_attachments from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, get_async_httpx_client, httpxSpecialProvider, ) @@ -286,9 +287,12 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): streaming_sampling_rate: int | None = None, streaming_end_of_stream_only: bool | None = None, streaming_buffer_release_on_scan: bool | None = None, + async_handler: AsyncHTTPHandler | None = None, **kwargs, ): - self.async_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.GuardrailCallback) + self.async_handler = async_handler or get_async_httpx_client( + llm_provider=httpxSpecialProvider.GuardrailCallback + ) self._set_streaming_params( BedrockGuardrailStreamingParams.from_extras( MappingProxyType( diff --git a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/example_config.yaml b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/example_config.yaml index 9b5c4e557f3..665d8cde4cc 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/example_config.yaml +++ b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/example_config.yaml @@ -6,20 +6,19 @@ model_list: model: gpt-4 api_key: os.environ/OPENAI_API_KEY -litellm_settings: - guardrails: - - guardrail_name: "my-generic-guardrail" - litellm_params: - guardrail: generic_guardrail_api - mode: pre_call # Options: pre_call, post_call, during_call, [pre_call, post_call] - api_key: os.environ/GENERIC_GUARDRAIL_API_KEY # Optional if using Bearer auth - api_base: http://localhost:8080 # Required. Endpoint /beta/litellm_basic_guardrail_api is automatically appended - unreachable_fallback: fail_closed # Options: fail_closed (default, raise), fail_open (proceed if endpoint unreachable or upstream returns 502/503/504) - default_on: false # Set to true to apply to all requests by default - additional_provider_specific_params: - # Any additional parameters your guardrail API needs - api_version: "v1" - custom_param: "value" +guardrails: + - guardrail_name: "my-generic-guardrail" + litellm_params: + guardrail: generic_guardrail_api + mode: pre_call # Options: pre_call, post_call, during_call, [pre_call, post_call] + api_key: os.environ/GENERIC_GUARDRAIL_API_KEY # Optional if using Bearer auth + api_base: http://localhost:8080 # Required. Endpoint /beta/litellm_basic_guardrail_api is automatically appended + unreachable_fallback: fail_closed # Options: fail_closed (default, raise), fail_open (proceed if endpoint unreachable or upstream returns 502/503/504) + default_on: false # Set to true to apply to all requests by default + additional_provider_specific_params: + # Any additional parameters your guardrail API needs + api_version: "v1" + custom_param: "value" # Usage examples: diff --git a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py index d50001ba636..3940a320fb8 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py +++ b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py @@ -30,6 +30,7 @@ from litellm.integrations.custom_guardrail import ( from litellm.llms.base_llm.guardrail_translation.utils import ( effective_scan_only_tool_results_for_guardrail, effective_skip_system_message_for_guardrail, + effective_skip_tool_message_for_guardrail, role_out_of_guardrail_scope, ) from litellm.llms.custom_httpx.http_handler import ( @@ -37,12 +38,16 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, ) -from litellm.llms.openai.responses.guardrail_translation.handler import scannable_instructions +from litellm.llms.openai.responses.guardrail_translation.handler import ( + scannable_instructions, + tool_output_texts, +) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.callback_utils import ( add_guardrail_scan_id, add_guardrail_to_applied_guardrails_header, ) +from litellm.responses.litellm_completion_transformation.transformation import TOOL_CALL_OUTPUT_ITEM_TYPES from litellm.types.guardrails import GuardrailEventHooks from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import AllMessageValues @@ -112,12 +117,16 @@ class _ResponsesInputItem(LiteLLMBaseModel): type: str | None = None role: str | None = None content: str | tuple[_ResponsesContentPart, ...] | None = None + output: object = None + call_id: object = None - def text_count(self, *, skip_system: bool) -> int: - if role_out_of_guardrail_scope( - (self.role or "").lower(), skip_system_message=skip_system, skip_tool_message=False - ): + def text_count(self, *, skip_system: bool, skip_tool: bool) -> int: + is_tool_output: Final = self.type in TOOL_CALL_OUTPUT_ITEM_TYPES + role: Final = "tool" if is_tool_output else (self.role or "").lower() + if role_out_of_guardrail_scope(role, skip_system_message=skip_system, skip_tool_message=skip_tool): return 0 + if is_tool_output: + return len(tool_output_texts(self)) if isinstance(self.content, str): return 1 if self.content is None: @@ -1692,13 +1701,14 @@ class PanwPrismaAirsHandler(CustomGuardrail): request_data: Mapping[str, object], *, skip_system: bool, + skip_tool: bool, ) -> frozenset[int] | None: """Return the ``texts`` indices flattened from Responses ``reasoning`` input items. The Responses translation handler gives those model-authored items the default ``user`` role, so the latest-turn selection must not mistake one for a human turn. Empty for requests without a Responses ``input`` item list; None when the raw items - (after the leading ``instructions`` text, both minus whatever ``skip_system`` drops) + (after the leading ``instructions`` text, both minus whatever ``skip_system`` and ``skip_tool`` drop) do not account for every entry of ``texts``. """ try: @@ -1708,7 +1718,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): if not isinstance(raw_input, tuple): return frozenset() offset: Final = 0 if scannable_instructions(request_data, skip_system=skip_system) is None else 1 - counts: Final = tuple(item.text_count(skip_system=skip_system) for item in raw_input) + counts: Final = tuple(item.text_count(skip_system=skip_system, skip_tool=skip_tool) for item in raw_input) if offset + sum(counts) != len(texts): return None starts: Final = itertools.accumulate(counts, initial=offset) @@ -1738,7 +1748,10 @@ class PanwPrismaAirsHandler(CustomGuardrail): if sources is None: return None reasoning: Final = self._reasoning_item_text_indices( - texts, request_data, skip_system=effective_skip_system_message_for_guardrail(self) + texts, + request_data, + skip_system=effective_skip_system_message_for_guardrail(self), + skip_tool=effective_skip_tool_message_for_guardrail(self), ) if reasoning is None: return None diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 2c05f724374..14d941ff62c 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -3024,13 +3024,7 @@ async def add_litellm_data_to_request( getattr(request.state, "litellm_roi_estimator", False) is True ) - verbose_proxy_logger.debug( - "[PROXY] returned data from litellm_pre_call_utils: %s", - data, - ) - # Team/Project credential overrides from model_config - # Placed after the debug log to avoid leaking credential secrets in logs _apply_credential_overrides_from_model_config( data=data, user_api_key_dict=user_api_key_dict, diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index e478096f7ad..972ec18d487 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -50,6 +50,7 @@ from litellm.constants import ( from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException from litellm.integrations.custom_guardrail import guardrail_request_data_with_streaming, is_guardrail_intervention from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix +from litellm.litellm_core_utils.fallback_generalizations import match_routing_generalization from litellm.llms.anthropic.common_utils import AnthropicModelInfo, merge_anthropic_beta_headers from litellm.llms.azure.passthrough.transformation import ( foreign_azure_deployment, @@ -698,6 +699,20 @@ async def typesafe_proxy_route( return await endpoint_func(request, fastapi_response, user_api_key_dict) +_COST_MAP_ENTRIES: Final = TypeAdapter(Mapping[str, object]) + + +def typesafe_allowlist_model_names() -> tuple[str, ...]: + if not passthrough_endpoint_router.has_credentials(custom_llm_provider="typesafe"): + return () + cost_map: Final = _COST_MAP_ENTRIES.validate_python(litellm.model_cost) # pyright: ignore[reportUnknownMemberType] # legacy registry is validated at this boundary + priced_names: Final = (name.removeprefix("typesafe/") for name in cost_map if name.startswith("typesafe/")) + jev_models_typesafe_serves: Final = sorted( + name for name in priced_names if match_routing_generalization(name) == "typesafe" + ) + return ("typesafe/*", *jev_models_typesafe_serves) + + @router.post( "/laya/v1/systemone", tags=["Laya Pass-through", "pass-through"], diff --git a/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py b/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py index 270a1cc2f3b..1d0b3d00434 100644 --- a/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py +++ b/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py @@ -1,4 +1,5 @@ import json +import os from collections.abc import Callable from typing import TYPE_CHECKING, Final @@ -88,6 +89,14 @@ class PassthroughEndpointRouter: ) return _resolve_oidc_reference(get_secret_str(_env_variable_name)) + def has_credentials(self, custom_llm_provider: str) -> bool: + if self._get_deployment_api_key(custom_llm_provider=custom_llm_provider, region_name=None) is not None: + return True + env_variable_name: Final = self._get_default_env_variable_name_passthrough_endpoint( + custom_llm_provider=custom_llm_provider, + ) + return bool(os.environ.get(env_variable_name)) + def _get_deployment_api_key( self, custom_llm_provider: str, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 42816e7db61..d6ec62b25c2 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -12390,6 +12390,12 @@ async def model_list( hidden_names: Final = blocked_names | unhealthy_names + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import typesafe_allowlist_model_names + + asks_for_allowlist_names: Final = ( + include_model_access_groups is True and return_wildcard_routes is True and only_model_access_groups is not True + ) + # If scope=expand and user has admin privileges, return all proxy models if should_expand_scope: # Get all proxy models as if user is a proxy admin @@ -12410,7 +12416,10 @@ async def model_list( all_models = get_complete_model_list( key_models=[], team_models=[], - proxy_model_list=proxy_model_list, + proxy_model_list=[ + *proxy_model_list, + *(typesafe_allowlist_model_names() if asks_for_allowlist_names else ()), + ], user_model=None, infer_model_from_keys=False, return_wildcard_routes=include_wildcard_routes, @@ -12471,7 +12480,7 @@ async def model_list( # Otherwise, use the normal behavior (current implementation) # Get available models for the user - all_models = await get_available_models_for_user( + available_models: Final = await get_available_models_for_user( user_api_key_dict=user_api_key_dict, llm_router=llm_router, general_settings=general_settings, @@ -12484,6 +12493,11 @@ async def model_list( return_wildcard_routes=include_wildcard_routes, user_api_key_cache=user_api_key_cache, ) + offers_typesafe_allowlist_names: Final = ( + asks_for_allowlist_names and team_id is None and user_api_key_has_admin_view(user_api_key_dict) + ) + typesafe_names_to_offer: Final = typesafe_allowlist_model_names() if offers_typesafe_allowlist_names else () + all_models = [*available_models, *(name for name in typesafe_names_to_offer if name not in available_models)] undiscoverable_names: Final = undiscoverable_model_names( all_models, llm_router, user_api_key_dict, team_id or user_api_key_dict.team_id @@ -18437,7 +18451,6 @@ async def async_queue_request( "body": copy.copy(data), # use copy instead of deepcopy } - verbose_proxy_logger.debug("receiving data: %s", data) data["model"] = ( general_settings.get("completion_model", None) # server default or user_model # model name passed via cli args diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index 528e9eb5b14..668c3f0043c 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -3519,6 +3519,34 @@ ], "default_model_placeholder": "inferenceapis/deepseek-ai/DeepSeek-V4-Flash" }, + { + "provider": "StreamLake", + "provider_display_name": "StreamLake", + "litellm_provider": "streamlake", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://vanchin.streamlake.ai/api/gateway/v1/endpoints", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "streamlake/" + }, { "provider": "Snowflake", "provider_display_name": "Snowflake", diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index bc92bc9cb3a..a342d704a9d 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -321,7 +321,7 @@ def _responses_file_part(item: Mapping[str, object]) -> _ResponsesFilePart: ) -def _decode_responses_media(item: Mapping[str, object]) -> _ResponsesMediaPart | None: +def decode_responses_media(item: Mapping[str, object]) -> _ResponsesMediaPart | None: kind: Final = item.get("type") if kind in ("input_image", "image_url", "computer_screenshot") and any(key in item for key in ("image_url", "url")): image: Final = _responses_image_part(item) @@ -383,6 +383,52 @@ def _attribute_fields(value: object) -> dict[str, object]: return dict(cast("Iterable[tuple[str, object]]", value)) # cast-ok: dict() raises on non-pair values, as before +def tool_output_to_chat_content( + output: object, +) -> str | list[ToolMessageContentPart]: + if output is None: + return "" + if isinstance(output, str): + return output + + single_media: Final = decode_responses_media(output) if isinstance(output, Mapping) else None + if single_media is not None: + return [single_media] + if isinstance(output, list): + normalized_blocks: Final[list[ToolMessageContentPart]] = [] + text_acc: Final[list[str]] = [] + for part in output: + if not isinstance(part, dict): + continue + part_type = part.get("type") + if part_type in ("input_text", "output_text", "text"): + txt = part.get("text") + if isinstance(txt, str) and txt: + text_acc.append(txt) + normalized_blocks.append({"type": "text", "text": txt}) + elif (media := decode_responses_media(part)) is not None: + normalized_blocks.append(media) + + if any(b.get("type") != "text" for b in normalized_blocks): + return normalized_blocks + if text_acc: + return "".join(text_acc) + try: + return json.dumps(output) + except Exception: + return str(output) + + try: + return json.dumps(output) + except Exception: + return str(output) + + +TOOL_CALL_OUTPUT_ITEM_TYPES: Final = frozenset( + {"function_call_output", "custom_tool_call_output", "computer_call_output", "tool_result"} +) + + def _input_item_role(input_item: Mapping[str, object]) -> str: return cast(str, input_item.get("role") or "user") # cast-ok: client-supplied role forwarded verbatim, unvalidated @@ -2371,12 +2417,7 @@ class LiteLLMCompletionResponsesConfig: """ Check if the input item is a tool call output """ - return input_item.get("type") in [ - "function_call_output", - "custom_tool_call_output", - "computer_call_output", - "tool_result", # Anthropic/MCP format - ] + return input_item.get("type") in TOOL_CALL_OUTPUT_ITEM_TYPES @staticmethod def _is_input_item_function_call(input_item: Mapping[str, object]) -> bool: @@ -2400,58 +2441,7 @@ class LiteLLMCompletionResponsesConfig: if not call_id: return [] - def _normalize_function_call_output_to_tool_content( - output: object, - ) -> str | list[ToolMessageContentPart]: - """ - Preserve structured media for downstream chat adapters and token estimation. - Text-only outputs retain their existing string representation. - """ - if output is None: - return "" - if isinstance(output, str): - return output - - single_media: Final = _decode_responses_media(output) if isinstance(output, Mapping) else None - if single_media is not None: - return [single_media] - # Some adapters represent tool output as a list of "input_*" parts - if isinstance(output, list): - normalized_blocks: Final[list[ToolMessageContentPart]] = [] - text_acc: Final[list[str]] = [] - for part in output: - if not isinstance(part, dict): - continue - part_type = part.get("type") - if part_type in ("input_text", "output_text", "text"): - txt = part.get("text") - if isinstance(txt, str) and txt: - text_acc.append(txt) - normalized_blocks.append({"type": "text", "text": txt}) - elif (media := _decode_responses_media(part)) is not None: - normalized_blocks.append(media) - - if any(b.get("type") != "text" for b in normalized_blocks): - return normalized_blocks - if text_acc: - return "".join(text_acc) - try: - # last resort: keep something meaningful for providers that require a string - import json as _json - - return _json.dumps(output) - except Exception: - return str(output) - - # Fallback for dict/number/etc. - try: - import json as _json - - return _json.dumps(output) - except Exception: - return str(output) - - tool_content: Final = _normalize_function_call_output_to_tool_content(tool_call_output.get("output")) + tool_content: Final = tool_output_to_chat_content(tool_call_output.get("output")) output_cache_control: Final = _input_item_cache_control(tool_call_output) tool_output_message: Final = ( ChatCompletionToolMessage(role="tool", content=tool_content, tool_call_id=str(call_id)) @@ -2606,7 +2596,7 @@ class LiteLLMCompletionResponsesConfig: content_list.append(item) elif isinstance(item, dict): if ( - media := _decode_responses_media( + media := decode_responses_media( fields := _validate_responses_media(_STR_KEY_DICT_ADAPTER, item) ) ) is not None: diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 4e418a91156..5060e03f040 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -39,7 +39,11 @@ from litellm.constants import ( ) from litellm.exceptions import MidStreamFallbackError, RateLimitError from litellm.litellm_core_utils.asyncify import run_async_function -from litellm.litellm_core_utils.core_helpers import process_response_headers +from litellm.litellm_core_utils.core_helpers import ( + process_response_headers, + provider_reported_usage_cost, + set_response_cost_in_hidden_params, +) from litellm.litellm_core_utils.exception_mapping_utils import with_deployment_api_base from litellm.litellm_core_utils.internal_call_metadata import ( is_background_response, @@ -1825,20 +1829,32 @@ def _estimate_usage_safely( def stamp_responses_usage_cost( response_obj: ResponsesAPIResponse | None, logging_obj: LiteLLMLoggingObj | None ) -> None: - if response_obj is None or logging_obj is None: + """The terminal response's ``usage.cost`` is the bill litellm writes to the spend row. It is priced on a copy + shaped the way the success handler shapes the spend row's result (the reported ``usage.cost`` in the + response-cost header, the usage in chat shape), so the client and the row read the same number.""" + if response_obj is None: return usage_obj: Final[ResponseAPIUsage | None] = _usage_as_model(getattr(response_obj, "usage", None)) if usage_obj is None: return - response_obj.usage = usage_obj # rebind-ok: the stamped cost has to ride on the response the client receives - if isinstance(getattr(usage_obj, "cost", None), (int, float)): + response_obj.usage = usage_obj # rebind-ok: the typed usage has to ride on the response the client receives + if logging_obj is None: return + priced: Final = response_obj.model_copy( + update={"usage": ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(usage_obj)} + ) + priced.hidden_params = {} + set_response_cost_in_hidden_params( + priced, provider_reported_usage_cost(usage_obj, logging_obj.model_call_details.get("custom_llm_provider")) + ) try: - cost: Final[float | None] = logging_obj.response_cost_calculator(result=response_obj) - except Exception: + bill: Final = logging_obj.response_cost_calculator(result=priced) + except Exception: # noqa: BLE001 # a pricing failure leaves the client usage as the upstream sent it return - if isinstance(cost, (int, float)) and cost > 0: - setattr(usage_obj, "cost", cost) + if isinstance(bill, bool) or not isinstance(bill, (int, float)): + return + if bill > 0 or isinstance(getattr(usage_obj, "cost", None), (int, float)): + setattr(usage_obj, "cost", float(bill)) def opening_item_payload(payload: Mapping[str, object]) -> Mapping[str, object]: diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index cf993c69814..05d6a36c1f6 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -766,6 +766,12 @@ class ResponsesAPIRequestUtils: model_id=model_id, response_id=decoded_response_id, ) + except ValueError: + return DecodedResponseId( + custom_llm_provider=None, + model_id=None, + response_id=response_id, + ) except Exception as e: verbose_logger.debug("Error decoding response_id '%s': %s", response_id, e) return DecodedResponseId( diff --git a/litellm/router.py b/litellm/router.py index 930d8e77b95..53e1d44abe7 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -4921,6 +4921,7 @@ class Router: deployment=deployment, forwarded_keys=kwargs.pop(_ALIAS_MARKER_FORWARDED_PARAMS_KWARG, ()) ): kwargs.pop(key, None) + kwargs.pop("daily_reset_time", None) # pyright: ignore[reportUnknownMemberType] # kwargs is an untyped dict defaults: Final = _LITELLM_PARAMS_ADAPTER.validate_python(self.default_litellm_params) self._merge_tools_from_deployment(deployment=deployment, kwargs=kwargs, defaults=defaults) @@ -5668,7 +5669,7 @@ class Router: def _image_generation(self, prompt: str, model: str, **kwargs): model_name: Final = "" try: - verbose_router_logger.debug("Inside _image_generation()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside _image_generation()- model: %s", model) deployment: Final = self.get_available_deployment( model=model, messages=[{"role": "user", "content": "prompt"}], @@ -5730,7 +5731,7 @@ class Router: async def _aimage_generation(self, prompt: str, model: str, **kwargs): model_name = model try: - verbose_router_logger.debug("Inside _image_generation()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside _image_generation()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) deployment: Final = await self.async_get_available_deployment( model=model, @@ -5815,7 +5816,7 @@ class Router: async def _atranscription(self, file: FileTypes, model: str, **kwargs): model_name: Final = model try: - verbose_router_logger.debug("Inside _atranscription()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside _atranscription()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) deployment: Final = await self.async_get_available_deployment( model=model, @@ -5908,7 +5909,7 @@ class Router: async def _aspeech(self, model: str, input: str, voice: str | None = None, **kwargs): model_name: Final = model try: - verbose_router_logger.debug("Inside _aspeech()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside _aspeech()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) deployment: Final = await self.async_get_available_deployment( model=model, @@ -5970,7 +5971,7 @@ class Router: async def _arerank(self, model: str, **kwargs): model_name = None try: - verbose_router_logger.debug("Inside _rerank()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside _rerank()- model: %s", model) deployment: Final = await self.async_get_available_deployment( model=model, specific_deployment=kwargs.pop("specific_deployment", None), @@ -6079,7 +6080,7 @@ class Router: async def _atext_completion(self, model: str, prompt: str, **kwargs): try: - verbose_router_logger.debug("Inside _atext_completion()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside _atext_completion()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) deployment: Final = await self.async_get_available_deployment( model=model, @@ -6149,7 +6150,7 @@ class Router: async def _aadapter_completion(self, adapter_id: str, model: str, **kwargs): try: - verbose_router_logger.debug("Inside _aadapter_completion()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside _aadapter_completion()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) deployment: Final = await self.async_get_available_deployment( model=model, @@ -6243,7 +6244,7 @@ class Router: kwargs=kwargs, metadata_variable_name="litellm_metadata", ) - verbose_router_logger.debug("Inside aguardrail() - guardrail_name: %s; kwargs: %s", guardrail_name, kwargs) + verbose_router_logger.debug("Inside aguardrail() - guardrail_name: %s", guardrail_name) response: Final = await self.async_function_with_fallbacks(**kwargs) return response @@ -6324,9 +6325,7 @@ class Router: controls: Final = per_request_fallback_controls(kwargs) kwargs[MID_STREAM_FALLBACK_CONTROLS_KEY] = controls # rebind-ok: forwarded to every hop self._update_kwargs_before_fallbacks(model=model, kwargs=kwargs, metadata_variable_name="litellm_metadata") - verbose_router_logger.debug( - "Inside ageneric_api_call_with_fallbacks() - model: %s; kwargs: %s", model, kwargs - ) + verbose_router_logger.debug("Inside ageneric_api_call_with_fallbacks() - model: %s", model) response: Final = await self.async_function_with_fallbacks(**kwargs) return response except Exception as e: @@ -7146,9 +7145,7 @@ class Router: handler_name: Final = original_function.__name__ metadata_variable_name: Final = get_router_metadata_variable_name(function_name="generic_api_call") try: - verbose_router_logger.debug( - "Inside _generic_api_call() - handler: %s, model: %s; kwargs: %s", handler_name, model, kwargs - ) + verbose_router_logger.debug("Inside _generic_api_call() - handler: %s, model: %s", handler_name, model) self._update_kwargs_before_fallbacks( model=model, kwargs=kwargs, @@ -7223,7 +7220,7 @@ class Router: def _embedding(self, input: str | list[object], model: str, **kwargs): model_name = None try: - verbose_router_logger.debug("Inside embedding()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside embedding()- model: %s", model) with embedding_deployment_selection(): deployment: Final = self.get_available_deployment( model=model, @@ -7312,7 +7309,7 @@ class Router: async def _aembedding(self, input: str | list[object], model: str, **kwargs): model_name = None try: - verbose_router_logger.debug("Inside _aembedding()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside _aembedding()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) with embedding_deployment_selection(): deployment: Final = await self.async_get_available_deployment( @@ -7396,7 +7393,7 @@ class Router: try: from litellm.router_utils.common_utils import add_model_file_id_mappings - verbose_router_logger.debug("Inside _atext_completion()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside _atext_completion()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) healthy_deployments: Final = await self.async_get_healthy_deployments( model=model, @@ -7634,7 +7631,7 @@ class Router: **kwargs, ) -> LiteLLMBatch: try: - verbose_router_logger.debug("Inside _acreate_batch()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside _acreate_batch()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) with batch_create_routing(): deployment: Final = await self.async_get_available_deployment( @@ -7833,7 +7830,7 @@ class Router: **kwargs, ) -> LiteLLMBatch: try: - verbose_router_logger.debug("Inside _acancel_batch()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside _acancel_batch()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) deployment: Final = await self.async_get_available_deployment( model=model, @@ -10470,7 +10467,7 @@ class Router: @staticmethod def _inherit_builtin_base_rates_for_scoped_pricing( - model_info: dict, # mutable-ok: cost-map entry filled in place + model_info: dict[str, object], # mutable-ok: cost-map entry filled in place backend_model: str, custom_llm_provider: str | None, ) -> None: diff --git a/litellm/router_utils/pre_call_checks/model_rate_limit_check.py b/litellm/router_utils/pre_call_checks/model_rate_limit_check.py index 9eebcf39515..f4b41211c6e 100644 --- a/litellm/router_utils/pre_call_checks/model_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/model_rate_limit_check.py @@ -12,7 +12,7 @@ is logged the first time such a deployment is seen. """ import contextlib -from collections.abc import Callable +from collections.abc import Callable, Mapping from datetime import datetime, timedelta from typing import TYPE_CHECKING, Any, Final, Literal @@ -38,7 +38,7 @@ from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import ( ) from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET from litellm.types.caching import RedisPipelineIncrementOperation -from litellm.types.router import RouterErrors +from litellm.types.router import RouterErrors, parse_daily_reset_time from litellm.types.utils import StandardLoggingPayload from litellm.utils import get_utc_datetime @@ -58,42 +58,49 @@ class RoutingArgs: DAY_FORMAT: Final = "%Y-%m-%d" -def seconds_until_utc_midnight(now: datetime) -> int: - next_midnight: Final = (now + timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0) - return max(1, int((next_midnight - now).total_seconds())) +def daily_window_day(now: datetime, reset_offset: timedelta) -> str: + return (now - reset_offset).strftime(DAY_FORMAT) -def _first_set(*values: int | None) -> int | None: - return next((value for value in values if value is not None), None) +def seconds_until_daily_reset(now: datetime, reset_offset: timedelta = timedelta(0)) -> int: + window_start: Final = (now - reset_offset).replace(hour=0, minute=0, second=0, microsecond=0) + next_reset: Final = window_start + timedelta(days=1) + reset_offset + return max(1, int((next_reset - now).total_seconds())) -class _DailyLimitFields(BaseModel): +class _DailyLimitParams(BaseModel): + """rpd, tpd and daily_reset_time live in litellm_params only.""" + model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True) + model: str | None = None tpd: int | None = None rpd: int | None = None + daily_reset_time: str | None = None + + def reset_offset(self) -> timedelta: + return parse_daily_reset_time(self.daily_reset_time) -class _DailyLimitParams(_DailyLimitFields): - model: str | None = None - - -class _DailyLimitModelInfo(_DailyLimitFields): +class _DailyLimitModelInfo(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True) id: str | None = None -class _DailyLimitDeployment(_DailyLimitFields): +class _DailyLimitDeployment(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True) model_name: str = "" litellm_params: _DailyLimitParams = _DailyLimitParams() model_info: _DailyLimitModelInfo = _DailyLimitModelInfo() def limits(self) -> tuple[int | None, int | None]: - return ( - _first_set(self.tpd, self.litellm_params.tpd, self.model_info.tpd), - _first_set(self.rpd, self.litellm_params.rpd, self.model_info.rpd), - ) + return self.litellm_params.tpd, self.litellm_params.rpd - def cache_keys(self, current_day: str) -> tuple[str, str]: + def reset_offset(self) -> timedelta: + return self.litellm_params.reset_offset() + + def cache_keys(self, now: datetime) -> tuple[str, str]: prefix: Final = f"{self.model_info.id}:{self.litellm_params.model}" + current_day: Final = daily_window_day(now, self.reset_offset()) return f"{prefix}:tpd:{current_day}", f"{prefix}:rpd:{current_day}" def limit_error(self, kind: Literal["tpd", "rpd"], limit: int, usage: int, now: datetime) -> litellm.RateLimitError: @@ -104,12 +111,19 @@ class _DailyLimitDeployment(_DailyLimitFields): response=httpx.Response( status_code=429, content=f"{RouterErrors.user_defined_ratelimit_error.value} {kind} limit={limit}. current usage={usage}. id={self.model_info.id}, model_group={self.model_name}", - headers={"retry-after": str(seconds_until_utc_midnight(now))}, + headers={"retry-after": str(seconds_until_daily_reset(now, self.reset_offset()))}, request=httpx.Request(method="model_rate_limit_check", url="https://github.com/BerriAI/litellm"), ), ) +def _reset_offset_from_kwargs(kwargs: Mapping[str, object]) -> timedelta: + litellm_params: Final = kwargs.get("litellm_params") + if litellm_params is None: + return timedelta(0) + return _DailyLimitParams.model_validate(litellm_params).reset_offset() + + class ModelRateLimitingCheck(CustomLogger): """ Pre-call check that enforces TPM/RPM or ITPM/OTPM limits on model deployments. @@ -242,7 +256,7 @@ class ModelRateLimitingCheck(CustomLogger): tpd_limit, rpd_limit = deployment.limits() if tpd_limit is None and rpd_limit is None: return - tpd_key, rpd_key = deployment.cache_keys(now.strftime(DAY_FORMAT)) + tpd_key, rpd_key = deployment.cache_keys(now) if tpd_limit is not None: current_tpd: Final = self._get_current_tpm(tpd_key, tpd_limit) if current_tpd is not None and current_tpd >= tpd_limit: @@ -394,7 +408,7 @@ class ModelRateLimitingCheck(CustomLogger): current_minute: Final = dt.strftime("%H-%M") tpm_key, rpm_key = self._get_cache_keys(deployment, current_minute) - tpd_key, rpd_key = daily.cache_keys(dt.strftime(DAY_FORMAT)) + tpd_key, rpd_key = daily.cache_keys(dt) model_id: Final = deployment.get("model_info", {}).get("id") model_name: Final = deployment.get("litellm_params", {}).get("model") @@ -510,7 +524,8 @@ class ModelRateLimitingCheck(CustomLogger): dt: Final = self.clock() current_minute: Final = dt.strftime("%H-%M") tpm_key: Final = f"{model_id}:{model}:tpm:{current_minute}" - tpd_key: Final = f"{model_id}:{model}:tpd:{dt.strftime(DAY_FORMAT)}" + reset_offset: Final = _reset_offset_from_kwargs(kwargs) # pyright: ignore[reportUnknownArgumentType] # CustomLogger passes an untyped kwargs dict + tpd_key: Final = f"{model_id}:{model}:tpd:{daily_window_day(dt, reset_offset)}" verbose_router_logger.debug("[TPM TRACKING] Incrementing %s by %s", tpm_key, total_tokens) @@ -570,7 +585,8 @@ class ModelRateLimitingCheck(CustomLogger): dt: Final = self.clock() current_minute: Final = dt.strftime("%H-%M") tpm_key: Final = f"{model_id}:{model}:tpm:{current_minute}" - tpd_key: Final = f"{model_id}:{model}:tpd:{dt.strftime(DAY_FORMAT)}" + reset_offset: Final = _reset_offset_from_kwargs(kwargs) # pyright: ignore[reportUnknownArgumentType] # CustomLogger passes an untyped kwargs dict + tpd_key: Final = f"{model_id}:{model}:tpd:{daily_window_day(dt, reset_offset)}" tokens: Final = int(total_tokens) self._increment(tpm_key, tokens, RoutingArgs.ttl) diff --git a/litellm/router_utils/search_api_router.py b/litellm/router_utils/search_api_router.py index 1cfb311d796..26326a7e6d4 100644 --- a/litellm/router_utils/search_api_router.py +++ b/litellm/router_utils/search_api_router.py @@ -155,10 +155,9 @@ class SearchAPIRouter: available_search_tool_names: Final = [tool.get("search_tool_name") for tool in router_instance.search_tools] verbose_router_logger.debug( - "Inside SearchAPIRouter.async_search_with_fallbacks() - search_tool_name: %s, Available Search Tools: %s, kwargs: %s", + "Inside SearchAPIRouter.async_search_with_fallbacks() - search_tool_name: %s, Available Search Tools: %s", search_tool_name, available_search_tool_names, - kwargs, ) # Use the existing retry/fallback infrastructure diff --git a/litellm/types/litellm_params.py b/litellm/types/litellm_params.py index 9196b7a226f..2065093f295 100644 --- a/litellm/types/litellm_params.py +++ b/litellm/types/litellm_params.py @@ -191,6 +191,7 @@ class DeploymentOptions: otpm: int | None = None rpd: int | None = None tpd: int | None = None + daily_reset_time: str | None = None kubernetes_pod_discovery: bool | None = None default_api_key_rpm_limit: int | None = None default_api_key_tpm_limit: int | None = None diff --git a/litellm/types/router.py b/litellm/types/router.py index f8de2128a80..60c9e596588 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -4,6 +4,7 @@ litellm.Router Types - includes RouterConfig, UpdateRouterConfig, ModelInfo etc import datetime import enum +import re from collections.abc import Container, Mapping, Sequence from dataclasses import dataclass, field from typing import ( @@ -518,6 +519,19 @@ def holds_secret_pointer(param_name: str) -> bool: _RESERVED_INIT_KEYS: Final = frozenset({"self", "params", "__class__"}) +_DAILY_RESET_TIME_PATTERN: Final = re.compile(r"\d{2}:\d{2}") + + +def parse_daily_reset_time(value: str | None) -> datetime.timedelta: + """`daily_reset_time` is "HH:MM" in UTC, when the rpd/tpd window starts over; unset means midnight UTC.""" + if value is None: + return datetime.timedelta(0) + if _DAILY_RESET_TIME_PATTERN.fullmatch(value) is None: + raise ValueError(f"daily_reset_time must be HH:MM in UTC, got {value!r}") + parsed: Final = datetime.time.fromisoformat(value) + return datetime.timedelta(hours=parsed.hour, minutes=parsed.minute) + + class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): """ LiteLLM Params without 'model' arg (used across completion / assistants api) @@ -535,6 +549,7 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): otpm: int | None = None tpd: int | None = None rpd: int | None = None + daily_reset_time: str | None = None timeout: float | str | httpx.Timeout | SDKTimeout | None = None # if str, pass in as os.environ/ # read timeout of a live stream, a faked stream takes the longer of this and timeout. If str, pass in as os.environ/ stream_timeout: float | str | None = None @@ -626,6 +641,13 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): valkey_text_field: str | None = None valkey_embedding_field: str | None = None + @field_validator("daily_reset_time") + @classmethod + def _daily_reset_time_is_utc_hh_mm(cls, value: str | None) -> str | None: + if value is not None: + parse_daily_reset_time(value) + return value + @field_validator("timeout") @classmethod def normalize_timeout( @@ -742,6 +764,7 @@ class LiteLLMParamsTypedDict(TypedDict, total=False): otpm: int | None tpd: ReadOnly[int | None] rpd: ReadOnly[int | None] + daily_reset_time: ReadOnly[str | None] order: int | None weight: int | None max_parallel_requests: int | None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 298ab0848c6..faa6fb6cf7f 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2879,7 +2879,7 @@ class TranscriptionUsageTokensObject(LiteLLMBaseModel): input_tokens: int output_tokens: int total_tokens: int - input_token_details: TranscriptionUsageInputTokenDetailsObject + input_token_details: TranscriptionUsageInputTokenDetailsObject | None = None class TranscriptionResponse(OpenAIObject): @@ -4595,6 +4595,7 @@ class LlmProviders(str, Enum): META = "meta" SAIL = "sail" REKA = "reka" + STREAMLAKE = "streamlake" LITELLM_AGENT = "litellm_agent" CURSOR = "cursor" BEDROCK_MANTLE = "bedrock_mantle" diff --git a/litellm/utils.py b/litellm/utils.py index a3dafecd08e..c4d221c5e90 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -304,7 +304,7 @@ except (ImportError, AttributeError, TypeError): # Convert to str (if necessary) claude_json_str = json.dumps(json_data) import importlib.metadata -from collections.abc import AsyncIterator, Callable, Collection, Iterable, Iterator, Mapping, Sequence +from collections.abc import AsyncIterator, Callable, Collection, Iterable, Iterator, Mapping, Sequence, Sized from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypeVar, cast, runtime_checkable from typing_extensions import assert_never @@ -606,9 +606,23 @@ def custom_llm_setup(): litellm.provider_list.append(custom_llm["provider"]) if custom_llm["provider"] not in litellm._custom_providers: + _warn_if_built_in_provider_shadows_custom_handler(custom_llm["provider"]) litellm._custom_providers.append(custom_llm["provider"]) +def _warn_if_built_in_provider_shadows_custom_handler(provider: str) -> None: + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + if provider not in LlmProvidersSet and not JSONProviderRegistry.exists(provider): + return + verbose_logger.warning( + "custom_provider_map provider '%s' has the same name as a built-in LiteLLM provider, which handles " + "'%s/' models instead of your custom_handler. Rename the provider and its models to use your handler", + provider, + provider, + ) + + def add_custom_logger_callback_to_specific_event(callback: str, logging_event: Literal["success", "failure"]) -> None: """ Add a custom logger callback to the specific event @@ -4067,7 +4081,9 @@ def get_optional_params_transcription( "timestamp_granularities": None, } - non_default_params = {k: v for k, v in passed_params.items() if (k in default_params and v != default_params[k])} + non_default_params: Final = { + k: v for k, v in passed_params.items() if (k in default_params and v != default_params[k]) + } optional_params = {} ## raise exception if non-default value passed for non-openai/azure embedding calls @@ -4104,6 +4120,19 @@ def get_optional_params_transcription( model=model, drop_params=drop_params if drop_params is not None else False, ) + elif custom_llm_provider == "bedrock": + from litellm.llms.bedrock.audio_transcription import ( + UnsupportedTranscriptionParams, + map_openai_transcription_params, + ) + + match map_openai_transcription_params( + non_default_params, drop_params=drop_params is True or litellm.drop_params is True + ): + case UnsupportedTranscriptionParams(message=message): + raise UnsupportedParamsError(message=message, llm_provider="bedrock") + case mapped_params: + optional_params = dict(mapped_params) elif provider_config is not None: # custom audio transcription config supported_params = provider_config.get_supported_openai_params(model=model) _check_valid_arg(supported_params=supported_params) @@ -4118,7 +4147,7 @@ def get_optional_params_transcription( optional_params=optional_params, passed_params=passed_params, custom_llm_provider=custom_llm_provider, - openai_params=OPENAI_TRANSCRIPTION_PARAMS, + openai_params=list(default_params) if custom_llm_provider == "bedrock" else OPENAI_TRANSCRIPTION_PARAMS, additional_drop_params=kwargs.get("additional_drop_params", None), ) @@ -5059,8 +5088,10 @@ def get_optional_params( supported_params: List[str] - supported params from the litellm config """ verbose_logger.info("\nLiteLLM completion() model= %s; provider = %s", model, custom_llm_provider) - verbose_logger.debug("\nLiteLLM: Params passed to completion() %s", passed_params) - verbose_logger.debug("\nLiteLLM: Non-Default params passed to completion() %s", non_default_params) + if verbose_logger.isEnabledFor(logging.DEBUG): + verbose_logger.debug( + "\nLiteLLM: Non-Default params passed to completion() %s", _debug_params_repr(non_default_params) + ) unsupported_params: Final = {} for k in non_default_params: if k not in supported_params: @@ -5601,7 +5632,9 @@ def get_optional_params( additional_drop_params=additional_drop_params, ) if _print_verbose_is_active(): - print_verbose(f"Final returned optional params: {redact_credentials_in_payload(optional_params)}") + print_verbose( + f"Final returned optional params: {_debug_params_repr(redact_credentials_in_payload(optional_params))}" + ) optional_params = apply_openai_param_overrides( optional_params=optional_params, non_default_params=non_default_params, @@ -6610,8 +6643,6 @@ def get_model_info_helper( model=model, custom_llm_provider=custom_llm_provider or declared_authenticating_provider(model) ) - verbose_logger.debug("checking potential_model_names in litellm.model_cost: %s", potential_model_names) - combined_model_name: Final = potential_model_names["combined_model_name"] stripped_model_name: Final = potential_model_names["stripped_model_name"] combined_stripped_model_name: Final = potential_model_names["combined_stripped_model_name"] @@ -6714,6 +6745,7 @@ def get_model_info_helper( key, _model_info = generalization if _model_info is None or key is None: + verbose_logger.debug("model not found in litellm.model_cost, tried: %s", potential_model_names) raise ModelNotMappedError(_model_not_mapped_message(model, custom_llm_provider)) _input_cost_per_token: float | None = _model_info.get("input_cost_per_token") if _input_cost_per_token is None: @@ -8534,6 +8566,23 @@ def get_valid_models( return [] # NON-Blocking +_MAX_INLINE_DEBUG_ARG_CHARS: Final = 500 + + +def _debug_arg_repr(value: object) -> str: + """Short form of a call argument, since the outgoing provider request log already prints the request body.""" + rendered: Final = repr(value) + if len(rendered) <= _MAX_INLINE_DEBUG_ARG_CHARS: + return rendered + if isinstance(value, Sized): + return f"<{type(value).__name__} len={len(value)}>" + return f"<{type(value).__name__}>" + + +def _debug_params_repr(params: Mapping[str, object]) -> str: + return "{" + ", ".join(f"{key!r}: {_debug_arg_repr(value)}" for key, value in params.items()) + "}" + + def print_args_passed_to_litellm(original_function, args, kwargs): if not is_debugging_on(): return @@ -8554,9 +8603,9 @@ def print_args_passed_to_litellm(original_function, args, kwargs): ): return - args_str: Final = ", ".join(map(repr, args)) + args_str: Final = ", ".join(map(_debug_arg_repr, args)) redacted_kwargs: Final = redact_credentials_in_payload(kwargs) - kwargs_str: Final = ", ".join(f"{key}={value!r}" for key, value in redacted_kwargs.items()) + kwargs_str: Final = ", ".join(f"{key}={_debug_arg_repr(value)}" for key, value in redacted_kwargs.items()) print_verbose( "\n", ) # new line before diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 525514161d3..9fa54a8d335 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -41939,6 +41939,120 @@ "supports_system_messages": true, "supports_native_structured_output": true }, + "bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 4.12e-08, + "input_cost_per_token_batches": 2.06e-08, + "input_cost_per_token_flex": 2.06e-08, + "input_cost_per_token_priority": 7.21e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 4.12e-08, + "output_cost_per_token_batches": 2.06e-08, + "output_cost_per_token_flex": 2.06e-08, + "output_cost_per_token_priority": 7.21e-08, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/ap-southeast-2/mistral.voxtral-small-24b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 1.03e-07, + "input_cost_per_token_batches": 5.15e-08, + "input_cost_per_token_flex": 5.15e-08, + "input_cost_per_token_priority": 1.8025e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 3.09e-07, + "output_cost_per_token_batches": 1.545e-07, + "output_cost_per_token_flex": 1.545e-07, + "output_cost_per_token_priority": 5.4075e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/ap-northeast-1/mistral.voxtral-mini-3b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 5e-08, + "input_cost_per_token_batches": 3e-08, + "input_cost_per_token_flex": 3e-08, + "input_cost_per_token_priority": 9e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 5e-08, + "output_cost_per_token_batches": 3e-08, + "output_cost_per_token_flex": 3e-08, + "output_cost_per_token_priority": 9e-08, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-northeast-1/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/ap-northeast-1/mistral.voxtral-small-24b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 1.2e-07, + "input_cost_per_token_batches": 6e-08, + "input_cost_per_token_flex": 6e-08, + "input_cost_per_token_priority": 2.1e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 3.6e-07, + "output_cost_per_token_batches": 1.8e-07, + "output_cost_per_token_flex": 1.8e-07, + "output_cost_per_token_priority": 6.3e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-northeast-1/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/eu-west-2/mistral.voxtral-mini-3b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 6e-08, + "input_cost_per_token_batches": 3e-08, + "input_cost_per_token_flex": 3e-08, + "input_cost_per_token_priority": 1.1e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 6e-08, + "output_cost_per_token_batches": 3e-08, + "output_cost_per_token_flex": 3e-08, + "output_cost_per_token_priority": 1.1e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/eu-west-2/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/eu-west-2/mistral.voxtral-small-24b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 1.6e-07, + "input_cost_per_token_batches": 8e-08, + "input_cost_per_token_flex": 8e-08, + "input_cost_per_token_priority": 2.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 4.7e-07, + "output_cost_per_token_batches": 2.3e-07, + "output_cost_per_token_flex": 2.3e-07, + "output_cost_per_token_priority": 8.2e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/eu-west-2/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, "mistral/codestral-2508": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 3e-07, @@ -61374,6 +61488,359 @@ "supports_tool_choice": true, "supports_vision": true }, + "streamlake/deepseek-v4-flash-0731": { + "cache_read_input_token_cost": 1.4e-08, + "input_cost_per_token": 4.4e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 1.32e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/DeepSeek-V4-Pro-0813": { + "cache_read_input_token_cost": 4.4e-08, + "input_cost_per_token": 1.32e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 1048566, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 3.96e-06, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/DeepSeek-V3": { + "cache_read_input_token_cost": 1.14e-07, + "input_cost_per_token": 2.86e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 56000, + "max_output_tokens": 8000, + "mode": "chat", + "output_cost_per_token": 1.143e-06, + "supports_function_calling": true, + "supports_reasoning": false, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/GLM-5.3": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_assistant_prefill": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/GLM-5.3-Flash": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_assistant_prefill": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/Kimi-K2-Instruct-0905": { + "cache_read_input_token_cost": 1.43e-07, + "input_cost_per_token": 5.71e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 224000, + "max_output_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 2.286e-06, + "supports_function_calling": true, + "supports_reasoning": false, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-r1": { + "cache_read_input_token_cost": 2.29e-07, + "input_cost_per_token": 5.71e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 98304, + "max_output_tokens": 16000, + "mode": "chat", + "output_cost_per_token": 2.286e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-v3.1": { + "cache_read_input_token_cost": 2.29e-07, + "input_cost_per_token": 5.71e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 98304, + "max_output_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 1.714e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-v4-flash": { + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 1.4e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 2.8e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-v4-pro": { + "cache_read_input_token_cost": 1.45e-07, + "input_cost_per_token": 1.74e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 3.48e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-v4.1-flash-1017": { + "cache_read_input_token_cost": 6e-09, + "input_cost_per_token": 3e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-v4.1-flash-260910": { + "cache_read_input_token_cost": 6e-09, + "input_cost_per_token": 3e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 1048568, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/glm-5": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 202752, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3.2e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/glm-5.1": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 202752, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/glm-5.2": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 1048566, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/kimi-k2.6": { + "cache_read_input_token_cost": 1.6e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 262144, + "max_output_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 4e-06, + "supports_function_calling": true, + "supports_vision": true, + "supports_reasoning": false, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/kimi-k2.7-code": { + "cache_read_input_token_cost": 1.9e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 262144, + "max_output_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 4e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/mimo-v2.6-flash": { + "cache_read_input_token_cost": 2.8e-09, + "input_cost_per_token": 1.4e-07, + "litellm_provider": "streamlake", + "mode": "chat", + "output_cost_per_token": 2.8e-07, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/mimo-v2.6-pro": { + "cache_read_input_token_cost": 3.6e-09, + "input_cost_per_token": 4.35e-07, + "litellm_provider": "streamlake", + "mode": "chat", + "output_cost_per_token": 8.7e-07, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/minimax-m2.5": { + "cache_creation_input_token_cost": 3.75e-07, + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 196608, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/minimax-m3": { + "cache_read_input_token_cost": 1.2e-07, + "cache_read_input_token_cost_above_512k_tokens": 2.4e-07, + "input_cost_per_token": 6e-07, + "input_cost_per_token_above_512k_tokens": 1.2e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 1000000, + "max_output_tokens": 512000, + "mode": "chat", + "output_cost_per_token": 2.4e-06, + "output_cost_per_token_above_512k_tokens": 4.8e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/qwen3.5-397b-a17b": { + "cache_read_input_token_cost": 1.2e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 258048, + "max_output_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/qwen3.6-27b": { + "input_cost_per_token": 6e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 229376, + "max_output_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "supports_function_calling": true, + "supports_vision": true, + "supports_reasoning": false, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/qwen3.6-35b-a3b": { + "input_cost_per_token": 3.75e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 229376, + "max_output_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.25e-06, + "supports_function_calling": true, + "supports_vision": true, + "supports_reasoning": false, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/qwen3.8-2.4t-a95b": { + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 983616, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 6e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "cache_creation_input_token_cost": 2.5e-06, + "explicit_cache_read_input_token_cost": 1.7e-07, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/qwen3.8-27b": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 5e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 983616, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "cache_creation_input_token_cost": 6.25e-07, + "explicit_cache_read_input_token_cost": 5e-08, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, "deepseek-v4-flash": { "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 6e-09, @@ -62689,6 +63156,14 @@ "litellm_provider": "anthropic" } }, + { + "name": "typesafe-jev-ids", + "pattern": "^jev-(?:latest|preview|1\\.13\\.0|1\\.14\\.0)$", + "description": "A bare Jev id that has a typesafe/ price entry (jev-latest, jev-preview, jev-1.13.0, jev-1.14.0), the name the /typesafe pass-through forwards to TypeSafe. Listed by name, matched case-insensitively like every rule here, so foo/jev-latest or a model group named like a Jev id TypeSafe does not serve, such as jev-1.0 or jev-internal-router, never resolves to typesafe. A new Jev version goes here together with its price entry. Provider inference feeds the proxy's provider/* wildcard access checks, so this rule is what lets typesafe/* on a key, team or project allowlist cover a jev-latest request.", + "model_info": { + "litellm_provider": "typesafe" + } + }, { "name": "claude-family-baseline", "pattern": "claude-[a-z]+-\\d+(?:[-.]\\d+)?(?:-\\d{8})?", diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index a42ce69a75f..819aca98338 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -3785,6 +3785,32 @@ "generateContent": true } }, + "streamlake": { + "display_name": "StreamLake (`streamlake`)", + "url": "https://docs.litellm.ai/docs/providers/openai_compatible", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, "tencent": { "display_name": "Tencent TokenHub (`tencent`)", "url": "https://docs.litellm.ai/docs/providers/tencent", diff --git a/tests/unit/AGENTS.md b/tests/unit/AGENTS.md index ca717e8efb0..ee6648c0a82 100644 --- a/tests/unit/AGENTS.md +++ b/tests/unit/AGENTS.md @@ -8,6 +8,10 @@ the same tests against Google's live spec Unit tests must not depend on the current time or execution speed. Use fixed timestamps for fixtures and controlled clocks for time-dependent behavior. Incidental clock reads inside production code are allowed +A unix socket goes under the `socket_dir` or `socket_path` fixture, never `tmp_path`, whose paths run past macOS's +104-byte AF_UNIX limit and fail only on a Mac. Code that opens a socket in its own `tempfile` directory runs under +`socket_tempdir` + ## What good looks like ```python diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index e64f5cca5b8..3070ebb0235 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -2,6 +2,8 @@ import asyncio import base64 import importlib import os +import shutil +import tempfile from collections.abc import Callable, Coroutine, Iterator from dataclasses import dataclass, field from pathlib import Path @@ -217,6 +219,38 @@ def _flush_client_caches() -> None: _reset_aws_auth_caches() +_MACOS_LONGEST_SOCKET_PATH: Final = 103 +_LONGEST_SOCKET_SUFFIX: Final = len("/sock12345678/d/litellm-pgbouncer-12345678/.s.PGSQL.65535") + + +def _socket_root() -> str: + default: Final = tempfile.gettempdir() + fits: Final = len(os.fsencode(default)) + _LONGEST_SOCKET_SUFFIX <= _MACOS_LONGEST_SOCKET_PATH + return default if fits else "/tmp" + + +@pytest.fixture +def socket_dir(tmp_path: Path) -> Iterator[Path]: + """``tmp_path`` behind a short link, so unix socket paths fit the sun_path limit that it exceeds on macOS.""" + link_dir: Final = Path(tempfile.mkdtemp(prefix="sock", dir=_socket_root())) + link: Final = link_dir / "d" + link.symlink_to(tmp_path, target_is_directory=True) + yield link + shutil.rmtree(link_dir, ignore_errors=True) + + +@pytest.fixture +def socket_path(socket_dir: Path) -> Path: + return socket_dir / "spend.sock" + + +@pytest.fixture +def socket_tempdir(socket_dir: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Point ``tempfile``'s default directory at ``socket_dir``, for code that picks its own socket directory.""" + monkeypatch.setattr(tempfile, "tempdir", str(socket_dir)) + return socket_dir + + @pytest.fixture(autouse=True, scope="session") def bundled_tiktoken_cache() -> None: importlib.import_module("litellm.litellm_core_utils.default_encoding") diff --git a/tests/unit/enterprise/enterprise_callbacks/test_callback_controls.py b/tests/unit/enterprise/enterprise_callbacks/test_callback_controls.py index 063e9e0ba7f..9648cb6ffc4 100644 --- a/tests/unit/enterprise/enterprise_callbacks/test_callback_controls.py +++ b/tests/unit/enterprise/enterprise_callbacks/test_callback_controls.py @@ -1,5 +1,5 @@ import unittest.mock as mock -from typing import cast +from typing import Final, cast from unittest.mock import MagicMock, patch import pytest @@ -197,6 +197,17 @@ class TestEnterpriseCallbackControls: ) assert result is False + def test_no_disable_request_logs_nothing(self, caplog: pytest.LogCaptureFixture) -> None: + litellm_params: Final = {"proxy_server_request": {"url": "test", "headers": {"user-agent": "codex"}}} + + with caplog.at_level("DEBUG", logger="LiteLLM"): + result: Final = EnterpriseCallbackControls.is_callback_disabled_dynamically( + DataDogLogger.__new__(DataDogLogger), litellm_params, StandardCallbackDynamicParams() + ) + + assert result is False + assert caplog.records == [] + def test_callback_not_disabled_when_header_none( self, mock_premium_user, mock_request_headers ): diff --git a/tests/unit/gateway/test_launch.py b/tests/unit/gateway/test_launch.py index f76aea030fe..4f5298e830b 100644 --- a/tests/unit/gateway/test_launch.py +++ b/tests/unit/gateway/test_launch.py @@ -140,7 +140,7 @@ class TestPoolDatabaseUrl: assert "DATABASE_URL" in outcome.reason def test_token_auth_hands_the_workers_the_pool_user_not_the_token( - self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path + self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, socket_tempdir: Path ): monkeypatch.setenv("IAM_TOKEN_DB_AUTH", "true") monkeypatch.setenv("AWS_REGION_NAME", "us-east-1") @@ -149,7 +149,7 @@ class TestPoolDatabaseUrl: with _minted_iam_token("MINTED_TOKEN"): outcome: Final = pool_database_url( DatabaseURLSettings.from_env(), - PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pooler(tmp_path))), + PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pooler(socket_tempdir))), environ, ) assert isinstance(outcome, str), outcome @@ -160,12 +160,12 @@ class TestPoolDatabaseUrl: class TestMain: def test_workers_inherit_the_loopback_url_the_supervisor_installed( - self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path + self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, socket_tempdir: Path ): port: Final = _free_port() monkeypatch.setenv("LITELLM_PGBOUNCER_ENABLED", "true") monkeypatch.setenv("LITELLM_PGBOUNCER_PORT", str(port)) - monkeypatch.setenv("LITELLM_PGBOUNCER_BINARY", str(_fake_pooler(tmp_path))) + monkeypatch.setenv("LITELLM_PGBOUNCER_BINARY", str(_fake_pooler(socket_tempdir))) monkeypatch.setenv("KEEPALIVE_TIMEOUT", "75") served: Final[list[tuple[str, ...]]] = [] main(("--workers", "4"), serve=lambda argv: served.append(tuple(argv))) @@ -183,7 +183,7 @@ class TestMain: assert _query(os.environ["DATABASE_URL"])["pgbouncer"] == "true" def test_iam_workers_keep_the_loopback_url_instead_of_minting_their_own( - self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path + self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, socket_tempdir: Path ): port: Final = _free_port() monkeypatch.delenv("DATABASE_PASSWORD") @@ -191,7 +191,7 @@ class TestMain: monkeypatch.setenv("AWS_REGION_NAME", "us-east-1") monkeypatch.setenv("LITELLM_PGBOUNCER_ENABLED", "true") monkeypatch.setenv("LITELLM_PGBOUNCER_PORT", str(port)) - monkeypatch.setenv("LITELLM_PGBOUNCER_BINARY", str(_fake_pooler(tmp_path))) + monkeypatch.setenv("LITELLM_PGBOUNCER_BINARY", str(_fake_pooler(socket_tempdir))) served: Final[list[tuple[str, ...]]] = [] with _minted_iam_token("SUPERVISOR_TOKEN"): main(("--workers", "4"), serve=lambda argv: served.append(tuple(argv))) diff --git a/tests/unit/integrations/datadog/test_datadog.py b/tests/unit/integrations/datadog/test_datadog.py index 14cd376cbc8..a9fd5ce27aa 100644 --- a/tests/unit/integrations/datadog/test_datadog.py +++ b/tests/unit/integrations/datadog/test_datadog.py @@ -905,10 +905,10 @@ async def test_a_failed_request_is_delivered_as_an_error_log_that_keeps_the_erro @pytest.mark.asyncio async def test_a_failing_redis_cache_is_delivered_to_datadog_as_redis_warnings( - delivery: tuple[DataDogLogger, respx.Route], monkeypatch: pytest.MonkeyPatch, tmp_path: Path + delivery: tuple[DataDogLogger, respx.Route], monkeypatch: pytest.MonkeyPatch, socket_dir: Path ) -> None: datadog_logger, intake = delivery - absent_socket: Final = str(tmp_path / "absent.sock") + absent_socket: Final = str(socket_dir / "absent.sock") redis_cache: Final = Cache(type="redis", url=f"unix://{absent_socket}") monkeypatch.setattr(redis_cache.cache.service_logger_obj, "dd_logger", datadog_logger, raising=False) monkeypatch.setattr(litellm, "service_callback", ["datadog"]) diff --git a/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py b/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py index e9e3f7b8174..25de5ef60d9 100644 --- a/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py +++ b/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py @@ -714,6 +714,48 @@ class TestClaudeModelPatternMatching: litellm.get_llm_provider(model="claude-opus-4-9") +@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") +class TestTypesafeJevPatternMatching: + def test_every_priced_jev_id_routes_to_typesafe( + self, shipped_generalizations: None + ) -> None: + from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap + + priced_ids: Final = tuple( + key.removeprefix("typesafe/") + for key in GetModelCostMap.load_local_model_cost_map() + if key.startswith("typesafe/") + ) + assert "jev-latest" in priced_ids + assert { + model: litellm.get_llm_provider(model=model)[1] for model in priced_ids + } == dict.fromkeys(priced_ids, "typesafe") + + @pytest.mark.parametrize( + "model", + [ + "foo/jev-latest", + "openjev-latest", + "jev-", + "jev-latest model", + "jev-internal-router", + "jev-2.0-internal", + "jev-1.0-acme.router", + "jev-1.0", + "jev-2.0", + "jev-1.13", + ], + ) + def test_namespaced_or_partial_jev_name_does_not_match_rule( + self, model: str, shipped_generalizations: None + ) -> None: + from litellm.litellm_core_utils.fallback_generalizations import ( + match_routing_generalization, + ) + + assert match_routing_generalization(model) is None + + @pytest.mark.parametrize("prefix", ["openai", "anthropic", "custom_openai", "openai_like"]) def test_opencode_affinity_resolution_preserves_existing_transport(prefix: str) -> None: api_base: Final = "https://opencode.ai/zen/go/v1" diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index 79978ecfe8e..b580f526a0f 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -6316,6 +6316,22 @@ def test_chunk_creator_preserves_hidden_provider_specific_fields_from_parsed_chu assert assembled._hidden_params["provider_specific_fields"] == {"traffic_type": "ON_DEMAND_FLEX"} +def test_the_usage_chunk_carries_the_bill_the_builder_priced_for_the_spend_row() -> None: + from litellm.litellm_core_utils.streaming_handler import ( + _stamp_usage_cost, # pyright: ignore[reportPrivateUsage] # the helper under test + ) + + complete: Final = ModelResponse(model="omni-model", usage=Usage(prompt_tokens=100, completion_tokens=20)) + complete.hidden_params = {"response_cost": 0.00032} + chunk_usage: Final = Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120) + setattr(chunk_usage, "cost", 0.037) + usage_chunk: Final = ModelResponseStream(id="chunk-1", model="omni-model", choices=[], usage=chunk_usage) + + _stamp_usage_cost(usage_chunk, complete) + + assert usage_chunk.usage is not None and usage_chunk.usage.cost == 0.00032, "the client sees what the row bills" + + def test_chunk_creator_keeps_provider_model_private_across_stream(): from litellm.router_utils.add_retry_fallback_headers import ( get_hidden_params_dict, diff --git a/tests/unit/llms/azure/response/test_azure_transformation.py b/tests/unit/llms/azure/response/test_azure_transformation.py index d46242fc5f7..29006901877 100644 --- a/tests/unit/llms/azure/response/test_azure_transformation.py +++ b/tests/unit/llms/azure/response/test_azure_transformation.py @@ -165,6 +165,44 @@ def test_explicit_v1_api_base_keeps_responses_api_version_selection( @pytest.mark.serial +@pytest.mark.parametrize( + ("api_base", "api_version", "expected_url"), + [ + ( + "https://x.openai.azure.com", + "2025-04-01-preview", + "https://x.openai.azure.com/openai/v1/responses/compact", + ), + ( + "https://x.openai.azure.com", + "preview", + "https://x.openai.azure.com/openai/v1/responses/compact?api-version=preview", + ), + ( + "https://gw.example.com/foundry/openai/openai/v1?tenant=example", + "v1", + "https://gw.example.com/foundry/openai/openai/v1/responses/compact?tenant=example&api-version=v1", + ), + ], +) +def test_compact_reaches_the_v1_route_and_drops_only_a_dated_api_version( + api_base: str, api_version: str, expected_url: str +) -> None: + config: Final = AzureOpenAIResponsesAPIConfig() + responses_url: Final = config.get_complete_url(api_base=api_base, litellm_params={"api_version": api_version}) + + url, _ = config.transform_compact_response_api_request( + model="gpt-5.4-mini", + input="hi", + response_api_optional_request_params={}, + api_base=responses_url, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert url == expected_url + + def test_response_id_path_requests_encode_response_id(): config = AzureOpenAIResponsesAPIConfig() api_base = ( diff --git a/tests/unit/llms/azure/test_azure_common_utils.py b/tests/unit/llms/azure/test_azure_common_utils.py index f7e2e73d426..8f083298555 100644 --- a/tests/unit/llms/azure/test_azure_common_utils.py +++ b/tests/unit/llms/azure/test_azure_common_utils.py @@ -1658,9 +1658,9 @@ def test_get_azure_ad_token_fallback_to_default_azure_credential( ) def test_is_azure_v1_api_version(api_version, expected): """ - Test that _is_azure_v1_api_version correctly identifies v1 API versions. + Test that is_azure_v1_api_version correctly identifies v1 API versions. """ - result = BaseAzureLLM._is_azure_v1_api_version(api_version=api_version) + result = BaseAzureLLM.is_azure_v1_api_version(api_version=api_version) assert result == expected diff --git a/tests/unit/llms/litellm_proxy/responses/test_transformation.py b/tests/unit/llms/litellm_proxy/responses/test_transformation.py index e710f4ac2c3..3d774b79eed 100644 --- a/tests/unit/llms/litellm_proxy/responses/test_transformation.py +++ b/tests/unit/llms/litellm_proxy/responses/test_transformation.py @@ -7,6 +7,7 @@ import respx import litellm from litellm.llms.litellm_proxy.responses.transformation import LiteLLMProxyResponsesAPIConfig +from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager from tests.unit.proxy.conftest import httpx_transport @@ -73,3 +74,23 @@ def test_get_complete_url_requires_api_base(monkeypatch: pytest.MonkeyPatch) -> monkeypatch.delenv("LITELLM_PROXY_API_BASE", raising=False) with pytest.raises(ValueError, match="api_base not set"): LiteLLMProxyResponsesAPIConfig().get_complete_url(api_base=None, litellm_params={}) + + +@pytest.mark.parametrize("drop_params", [False, True]) +def test_compact_request_forwards_metadata_and_user_to_the_downstream_proxy(drop_params: bool) -> None: + url, data = LiteLLMProxyResponsesAPIConfig().transform_compact_response_api_request( + model="gpt-5.5", + input="hi", + response_api_optional_request_params={"metadata": {"ticket": "qa-1"}, "user": "end-user-1", "temperature": 0.2}, + api_base="https://my-proxy.example.com/responses", + litellm_params=GenericLiteLLMParams(drop_params=drop_params), + headers={}, + ) + + assert url == "https://my-proxy.example.com/responses/compact" + assert {key: data[key] for key in data if key != "input"} == { + "model": "gpt-5.5", + "metadata": {"ticket": "qa-1"}, + "user": "end-user-1", + "temperature": 0.2, + } diff --git a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py index 0d1b1aca9bc..673e880a1c0 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -7,12 +7,14 @@ with guardrail transformations. import copy import json +import uuid from collections.abc import Callable from typing import Any, Final, List, Literal, Optional, Tuple from unittest.mock import AsyncMock, MagicMock, patch import logging +import httpx import pytest @@ -33,6 +35,10 @@ from litellm.llms.openai.responses.guardrail_translation.handler import ( ) from litellm.llms.openai.responses.guardrail_translation.tool_merge import merge_guardrailed_tools from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api import GenericGuardrailAPI +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import BedrockGuardrail +from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ContentFilterGuardrail +from litellm.types.guardrails import ContentFilterAction, ContentFilterPattern from litellm.types.llms.openai import ChatCompletionToolCallChunk from litellm.responses.litellm_completion_transformation.compaction_items import compaction_summary from litellm.responses.litellm_completion_transformation.transformation import ( @@ -83,6 +89,7 @@ class RecordingMaskingGuardrail(MockGuardrail): super().__init__(guardrail_name=guardrail_name) self.seen_texts: list[list[str]] = [] self.seen_message_contents: list[list[object]] = [] + self.seen_tools: list[object] = [] async def apply_guardrail( self, @@ -93,6 +100,7 @@ class RecordingMaskingGuardrail(MockGuardrail): ) -> GenericGuardrailAPIInputs: self.seen_texts.append(list(inputs.get("texts", []))) self.seen_message_contents.append([m["content"] for m in inputs.get("structured_messages") or []]) + self.seen_tools.append(inputs.get("tools")) return await super().apply_guardrail(inputs, request_data, input_type, logging_obj) @@ -588,6 +596,638 @@ class TestSkipSystemMessageScopesInstructions: ] +_TOOL_TURNS: Final = ( + ( + {"type": "function_call", "call_id": "call_1", "name": "get_weather", "arguments": '{"city": "Paris"}'}, + "function_call_output", + ), + ( + {"type": "custom_tool_call", "call_id": "call_1", "name": "run_sql", "input": "select 1"}, + "custom_tool_call_output", + ), +) + + +def _tool_result_request(call_item: dict[str, str], output_type: str, output: object) -> dict[str, object]: + return { + "model": "gpt-5.4-mini", + "input": [ + {"role": "user", "content": "Weather?"}, + call_item, + {"type": output_type, "call_id": "call_1", "output": output}, + ], + } + + +def _bedrock_guardrail( + name: str, bedrock_api: Callable[[httpx.Request], httpx.Response], **settings: bool +) -> BedrockGuardrail: + return BedrockGuardrail( + guardrail_name=name, + guardrailIdentifier="test-guardrail", + guardrailVersion="DRAFT", + aws_access_key_id="test-access-key", + aws_secret_access_key="test-secret-key", + aws_region_name="us-east-1", + async_handler=AsyncHTTPHandler(transport=httpx.MockTransport(bedrock_api)), + **settings, + ) + + +def _bedrock_block_texts(request: httpx.Request) -> list[str]: + return [block["text"]["text"] for block in json.loads(request.content)["content"]] + + +class TestToolResultItemsReachTextsGuardrails: + """A Responses tool result is scanned and rewritten in its `output` field the way a chat `tool` + message's content is, and skip_tool_message_in_guardrail keeps it out of the scan the same way.""" + + @pytest.mark.asyncio + @pytest.mark.parametrize(("call_item", "output_type"), _TOOL_TURNS, ids=["function", "custom"]) + async def test_string_output_is_scanned_and_rewritten_in_place( + self, call_item: dict[str, str], output_type: str + ) -> None: + guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test") + + result: Final = await OpenAIResponsesHandler().process_input_messages( + _tool_result_request(call_item, output_type, "SECRET forecast"), guardrail + ) + + assert guardrail.seen_texts == [["Weather?", "SECRET forecast"]] + assert result["input"] == [ + {"role": "user", "content": "Weather? [GUARDRAILED]"}, + call_item, + {"type": output_type, "call_id": "call_1", "output": "SECRET forecast [GUARDRAILED]"}, + ] + + @pytest.mark.asyncio + async def test_content_part_output_is_scanned_and_rewritten_in_its_part(self) -> None: + guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test") + call_item, output_type = _TOOL_TURNS[0] + image_part: Final = {"type": "input_image", "image_url": "https://example.com/map.png"} + + result: Final = await OpenAIResponsesHandler().process_input_messages( + _tool_result_request( + call_item, output_type, [{"type": "input_text", "text": "SECRET forecast"}, image_part] + ), + guardrail, + ) + + assert guardrail.seen_texts == [["Weather?", "SECRET forecast"]] + assert result["input"][2]["output"] == [ + {"type": "input_text", "text": "SECRET forecast [GUARDRAILED]"}, + image_part, + ] + + @pytest.mark.asyncio + @pytest.mark.parametrize("part_type", ["input_text", "output_text"]) + async def test_text_only_parts_are_scanned_and_rewritten_part_by_part(self, part_type: str) -> None: + guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test") + call_item, output_type = _TOOL_TURNS[0] + parts: Final = [ + {"type": part_type, "text": "SECRET forecast: ", "annotations": []}, + {"type": part_type, "text": "sunny"}, + ] + + result: Final = await OpenAIResponsesHandler().process_input_messages( + _tool_result_request(call_item, output_type, parts), guardrail + ) + + assert guardrail.seen_texts == [["Weather?", "SECRET forecast: ", "sunny"]] + assert result["input"][2]["output"] == [ + {"type": part_type, "text": "SECRET forecast: [GUARDRAILED]", "annotations": []}, + {"type": part_type, "text": "sunny [GUARDRAILED]"}, + ] + + @pytest.mark.asyncio + async def test_a_pattern_ending_at_a_part_boundary_blocks_like_it_does_on_chat(self) -> None: + access_key: Final = "AKIA" + "SYNTHETIC0000000" + content_filter: Final = ContentFilterGuardrail( + guardrail_name="secrets", + patterns=[ + ContentFilterPattern( + pattern_type="prebuilt", pattern_name="aws_access_key", action=ContentFilterAction.BLOCK + ) + ], + ) + call_item, output_type = _TOOL_TURNS[0] + parts: Final = [ + {"type": "input_text", "text": f"key={access_key}"}, + {"type": "input_text", "text": "region=us-east-1"}, + ] + + with pytest.raises(HTTPException) as blocked: + await OpenAIResponsesHandler().process_input_messages( + _tool_result_request(call_item, output_type, parts), content_filter + ) + + assert blocked.value.status_code == 400 + + @pytest.mark.asyncio + @pytest.mark.parametrize("empty_output", ["", None], ids=["empty_string", "null"]) + async def test_empty_tool_output_sends_no_text_and_stays_as_sent(self, empty_output: str | None) -> None: + guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test") + call_item, output_type = _TOOL_TURNS[0] + + result: Final = await OpenAIResponsesHandler().process_input_messages( + _tool_result_request(call_item, output_type, empty_output), guardrail + ) + + assert guardrail.seen_texts == [["Weather?"]] + assert result["input"][2] == {"type": output_type, "call_id": "call_1", "output": empty_output} + + @pytest.mark.asyncio + @pytest.mark.parametrize("latest_user_only", [False, True], ids=["all_texts", "latest_user_only"]) + @pytest.mark.parametrize( + "tool_output", + [ + {"type": "function_call_output", "call_id": "call_1", "output": ""}, + {"type": "function_call_output", "call_id": "call_1", "output": None}, + { + "type": "computer_call_output", + "call_id": "call_1", + "output": {"type": "computer_screenshot", "file_id": "file-screen"}, + }, + ], + ids=["empty_string", "null", "file_id_screenshot"], + ) + async def test_bedrock_masking_applies_to_a_request_with_a_textless_tool_output( + self, tool_output: dict[str, object], latest_user_only: bool + ) -> None: + sent_texts: Final[list[list[str]]] = [] + + def bedrock_masking_emails(request: httpx.Request) -> httpx.Response: + texts: Final = _bedrock_block_texts(request) + sent_texts.append(texts) + outputs: Final = [{"text": text.replace("jane@example.com", "{EMAIL}")} for text in texts] + return httpx.Response(200, json={"action": "GUARDRAIL_INTERVENED", "outputs": outputs}) + + guardrail: Final = _bedrock_guardrail( + "bedrock-pre", bedrock_masking_emails, experimental_use_latest_role_message_only=latest_user_only + ) + + request: Final = { + "model": "gpt-5.4-mini", + "input": [ + {"role": "user", "content": "Look up the order."}, + {"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"}, + copy.deepcopy(tool_output), + {"role": "user", "content": "Email jane@example.com the result."}, + ], + } + + result: Final = await OpenAIResponsesHandler().process_input_messages(request, guardrail) + + expected_texts: Final = ( + [["Email jane@example.com the result."]] + if latest_user_only + else [["Look up the order.", "Email jane@example.com the result."]] + ) + assert sent_texts == expected_texts + assert result["input"][3]["content"] == "Email {EMAIL} the result." + assert result["input"][2] == tool_output + + @pytest.mark.asyncio + async def test_bedrock_latest_user_masking_applies_with_a_multi_part_tool_output(self) -> None: + sent_texts: Final[list[list[str]]] = [] + + def bedrock_masking_emails(request: httpx.Request) -> httpx.Response: + texts: Final = _bedrock_block_texts(request) + sent_texts.append(texts) + outputs: Final = [{"text": text.replace("jane@example.com", "{EMAIL}")} for text in texts] + return httpx.Response(200, json={"action": "GUARDRAIL_INTERVENED", "outputs": outputs}) + + guardrail: Final = _bedrock_guardrail( + "bedrock-pre", bedrock_masking_emails, experimental_use_latest_role_message_only=True + ) + tool_output: Final = { + "type": "function_call_output", + "call_id": "call_1", + "output": [{"type": "input_text", "text": "order 42"}, {"type": "input_text", "text": "shipped"}], + } + request: Final = { + "model": "gpt-5.4-mini", + "input": [ + {"role": "user", "content": "Look up the order."}, + {"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"}, + copy.deepcopy(tool_output), + {"role": "user", "content": "Email jane@example.com the result."}, + ], + } + + result: Final = await OpenAIResponsesHandler().process_input_messages(request, guardrail) + + assert sent_texts == [["Email jane@example.com the result."]] + assert result["input"][3]["content"] == "Email {EMAIL} the result." + assert result["input"][2] == tool_output + + @pytest.mark.asyncio + async def test_bedrock_only_scan_new_messages_scans_only_the_new_tool_result(self) -> None: + sent_texts: Final[list[list[str]]] = [] + + def bedrock_allowing(request: httpx.Request) -> httpx.Response: + sent_texts.append(_bedrock_block_texts(request)) + return httpx.Response(200, json={"action": "NONE", "outputs": []}) + + guardrail: Final = _bedrock_guardrail("bedrock-incremental", bedrock_allowing, only_scan_new_messages=True) + + first_turn: Final = [ + {"role": "user", "content": "Weather in Paris and Rome?"}, + {"type": "function_call", "call_id": "call_1", "name": "get_weather", "arguments": '{"city": "Paris"}'}, + {"type": "function_call_output", "call_id": "call_1", "output": "Paris: sunny"}, + ] + second_turn: Final = [ + *first_turn, + {"type": "function_call", "call_id": "call_2", "name": "get_weather", "arguments": '{"city": "Rome"}'}, + {"type": "function_call_output", "call_id": "call_2", "output": "Rome: rain"}, + ] + session: Final = {"litellm_session_id": f"session-{uuid.uuid4()}"} + + for turn in (first_turn, second_turn): + await OpenAIResponsesHandler().process_input_messages( + {"model": "gpt-5.4-mini", "input": copy.deepcopy(turn), **session}, guardrail + ) + + assert sent_texts == [["Weather in Paris and Rome?", "Paris: sunny"], ["Rome: rain"]] + + @pytest.mark.asyncio + async def test_file_id_screenshot_is_not_scanned_as_text_or_rewritten(self) -> None: + guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test") + screenshot_output: Final = { + "type": "computer_call_output", + "call_id": "cu_1", + "output": {"type": "computer_screenshot", "file_id": "file-SECRET-screen"}, + } + request: Final = { + "model": "gpt-5.4-mini", + "input": [ + {"role": "user", "content": "Open the page"}, + {"type": "computer_call", "call_id": "cu_1", "id": "cu_1", "action": {"type": "screenshot"}}, + copy.deepcopy(screenshot_output), + ], + } + + result: Final = await OpenAIResponsesHandler().process_input_messages(request, guardrail) + + assert guardrail.seen_texts == [["Open the page"]] + assert result["input"][2] == screenshot_output + + @pytest.mark.asyncio + async def test_text_parts_between_file_images_are_rewritten_in_place(self) -> None: + guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test") + call_item, output_type = _TOOL_TURNS[0] + logo_part: Final = {"type": "input_image", "file_id": "file-logo"} + image_part: Final = {"type": "input_image", "file_id": "file-map"} + parts: Final = [ + logo_part, + {"type": "input_text", "text": "SECRET forecast: "}, + image_part, + {"type": "input_text", "text": "sunny"}, + ] + + result: Final = await OpenAIResponsesHandler().process_input_messages( + _tool_result_request(call_item, output_type, parts), guardrail + ) + + assert guardrail.seen_texts == [["Weather?", "SECRET forecast: ", "sunny"]] + assert result["input"][2]["output"] == [ + logo_part, + {"type": "input_text", "text": "SECRET forecast: [GUARDRAILED]"}, + image_part, + {"type": "input_text", "text": "sunny [GUARDRAILED]"}, + ] + + @pytest.mark.asyncio + async def test_dict_output_is_rewritten_as_the_text_the_guardrail_returns(self) -> None: + guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test") + call_item, output_type = _TOOL_TURNS[0] + + result: Final = await OpenAIResponsesHandler().process_input_messages( + _tool_result_request(call_item, output_type, {"result": "SECRET forecast"}), guardrail + ) + + assert result["input"][2]["output"] == '{"result": "SECRET forecast"} [GUARDRAILED]' + + @pytest.mark.asyncio + async def test_dict_output_the_guardrail_leaves_unchanged_stays_a_dict(self) -> None: + guardrail: Final = TextsReplacingGuardrail(guardrail_name="test", texts=("[MASKED]", '{"result": "sunny"}')) + call_item, output_type = _TOOL_TURNS[0] + + result: Final = await OpenAIResponsesHandler().process_input_messages( + _tool_result_request(call_item, output_type, {"result": "sunny"}), guardrail + ) + + assert result["input"][0]["content"] == "[MASKED]" + assert result["input"][2]["output"] == {"result": "sunny"} + + @pytest.mark.asyncio + async def test_tool_result_without_call_id_is_left_out_like_its_structured_row(self) -> None: + guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test") + call_item, output_type = _TOOL_TURNS[0] + orphan: Final = {"type": output_type, "output": "SECRET orphan"} + request: Final = { + "model": "gpt-5.4-mini", + "input": [ + {"role": "user", "content": "Weather?"}, + call_item, + dict(orphan), + {"type": output_type, "call_id": "call_1", "output": "SECRET forecast"}, + ], + } + + result: Final = await OpenAIResponsesHandler().process_input_messages(request, guardrail) + + assert guardrail.seen_texts == [["Weather?", "SECRET forecast"]] + assert "SECRET orphan" not in guardrail.seen_message_contents[0] + assert result["input"][2] == orphan + assert result["input"][3]["output"] == "SECRET forecast [GUARDRAILED]" + + @pytest.mark.asyncio + async def test_a_textless_output_sharing_a_call_id_keeps_the_text_output_in_structured_messages(self) -> None: + guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test") + call_item, output_type = _TOOL_TURNS[0] + request: Final = _tool_result_request(call_item, output_type, "") + request["input"].append({"type": output_type, "call_id": "call_1", "output": "SECRET forecast"}) + + result: Final = await OpenAIResponsesHandler().process_input_messages(request, guardrail) + + assert guardrail.seen_texts == [["Weather?", "SECRET forecast"]] + assert "SECRET forecast" in guardrail.seen_message_contents[0] + assert "" not in guardrail.seen_message_contents[0] + assert result["input"][3]["output"] == "SECRET forecast [GUARDRAILED]" + + @pytest.mark.asyncio + async def test_multi_part_outputs_sharing_a_call_id_each_keep_their_own_text_in_structured_messages( + self, + ) -> None: + guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test") + call_item, output_type = _TOOL_TURNS[0] + request: Final = _tool_result_request( + call_item, + output_type, + [{"type": "input_text", "text": "first "}, {"type": "input_text", "text": "output"}], + ) + request["input"].append( + { + "type": output_type, + "call_id": "call_1", + "output": [{"type": "input_text", "text": "second "}, {"type": "input_text", "text": "output"}], + } + ) + + result: Final = await OpenAIResponsesHandler().process_input_messages(request, guardrail) + + assert guardrail.seen_texts == [["Weather?", "first ", "output", "second ", "output"]] + assert guardrail.seen_message_contents[0][2:] == [ + [{"type": "text", "text": "first "}, {"type": "text", "text": "output"}], + [{"type": "text", "text": "second "}, {"type": "text", "text": "output"}], + ] + assert [part["text"] for part in result["input"][3]["output"]] == [ + "second [GUARDRAILED]", + "output [GUARDRAILED]", + ] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "textless_output", + [ + {"type": "function_call_output", "call_id": "call_1", "output": ""}, + {"type": "function_call_output", "call_id": "call_1", "output": None}, + { + "type": "computer_call_output", + "call_id": "call_1", + "output": {"type": "computer_screenshot", "file_id": "file-screen"}, + }, + ], + ids=["empty_string", "null", "file_id_screenshot"], + ) + async def test_multi_part_output_sharing_a_call_id_with_a_textless_output_lists_its_parts_in_structured_messages( + self, textless_output: dict[str, object] + ) -> None: + guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test") + call_item, output_type = _TOOL_TURNS[0] + request: Final = _tool_result_request( + call_item, + output_type, + [{"type": "input_text", "text": "first "}, {"type": "input_text", "text": "output"}], + ) + request["input"].append(copy.deepcopy(textless_output)) + + await OpenAIResponsesHandler().process_input_messages(request, guardrail) + + assert guardrail.seen_texts == [["Weather?", "first ", "output"]] + assert guardrail.seen_message_contents[0][2:] == [ + [{"type": "text", "text": "first "}, {"type": "text", "text": "output"}], + ] + + @pytest.mark.asyncio + async def test_tool_result_item_is_scanned_like_function_call_output(self) -> None: + guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test") + call_item, _ = _TOOL_TURNS[0] + + result: Final = await OpenAIResponsesHandler().process_input_messages( + _tool_result_request(call_item, "tool_result", "SECRET forecast"), guardrail + ) + + assert guardrail.seen_texts == [["Weather?", "SECRET forecast"]] + assert guardrail.seen_message_contents[0][2] == "SECRET forecast" + assert result["input"][2]["output"] == "SECRET forecast [GUARDRAILED]" + + @pytest.mark.asyncio + async def test_text_only_parts_the_guardrail_leaves_unchanged_keep_their_parts(self) -> None: + call_item, output_type = _TOOL_TURNS[0] + parts: Final = [{"type": "input_text", "text": "SECRET forecast: "}, {"type": "input_text", "text": "sunny"}] + + result: Final = await OpenAIResponsesHandler().process_input_messages( + _tool_result_request(call_item, output_type, copy.deepcopy(parts)), CustomGuardrail(guardrail_name="test") + ) + + assert result["input"][2]["output"] == parts + + @pytest.mark.asyncio + async def test_text_parts_a_structured_guardrail_returns_unchanged_keep_their_parts(self) -> None: + class StructuredEchoGuardrail(CustomGuardrail): + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict[str, object], + input_type: Literal["request", "response"], + logging_obj: LiteLLMLoggingObj | None = None, + ) -> GenericGuardrailAPIInputs: + return {**inputs, "structured_messages": copy.deepcopy(inputs.get("structured_messages") or [])} + + call_item, output_type = _TOOL_TURNS[0] + parts: Final = [ + {"type": "output_text", "text": "SECRET forecast: ", "annotations": []}, + {"type": "output_text", "text": "sunny"}, + ] + + result: Final = await OpenAIResponsesHandler().process_input_messages( + _tool_result_request(call_item, output_type, copy.deepcopy(parts)), + StructuredEchoGuardrail(guardrail_name="t"), + ) + + assert result["input"][2]["output"] == parts + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("output", "expected_texts"), + [ + pytest.param("SECRET forecast", ["SECRET forecast"], id="string"), + pytest.param([{"type": "input_text", "text": "SECRET forecast"}], ["SECRET forecast"], id="one_part"), + pytest.param( + [{"type": "input_text", "text": "SECRET "}, {"type": "output_text", "text": "forecast"}], + ["SECRET ", "forecast"], + id="parts", + ), + pytest.param( + [ + {"type": "input_text", "text": "SECRET "}, + {"type": "input_image", "image_url": "https://example.com/map.png"}, + {"type": "input_text", "text": ""}, + {"type": "input_text", "text": "forecast"}, + ], + ["SECRET ", "forecast"], + id="parts_with_image", + ), + pytest.param( + [ + {"type": "input_text", "text": "SECRET forecast"}, + {"type": "summary_text", "text": "not tool output"}, + ], + ["SECRET forecast"], + id="non_text_part_with_text_field", + ), + pytest.param( + ["SECRET forecast", {"type": "input_text", "text": "visible"}], + ["visible"], + id="non_dict_part", + ), + pytest.param([], ["[]"], id="empty"), + pytest.param({"result": "SECRET forecast"}, ['{"result": "SECRET forecast"}'], id="dict"), + pytest.param( + [{"type": "input_image", "file_id": "file-map"}], + ['[{"type": "input_image", "file_id": "file-map"}]'], + id="no_text_parts", + ), + pytest.param({"type": "input_image", "image_url": "https://example.com/map.png"}, [], id="single_image"), + ], + ) + async def test_tool_result_texts_match_the_structured_tool_message( + self, output: object, expected_texts: list[str] + ) -> None: + guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test") + call_item, output_type = _TOOL_TURNS[0] + + await OpenAIResponsesHandler().process_input_messages( + _tool_result_request(call_item, output_type, output), guardrail + ) + + tool_content: Final = guardrail.seen_message_contents[0][2] + structured_texts: Final = ( + [tool_content] + if isinstance(tool_content, str) + else [part["text"] for part in tool_content if part.get("type") == "text"] + ) + tool_texts: Final = guardrail.seen_texts[0][1:] + assert tool_texts == expected_texts + assert tool_texts == structured_texts + + @pytest.mark.asyncio + @pytest.mark.parametrize(("call_item", "output_type"), _TOOL_TURNS, ids=["function", "custom"]) + async def test_skip_tool_message_keeps_the_tool_result_out_of_texts_and_rows( + self, call_item: dict[str, str], output_type: str + ) -> None: + guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test") + guardrail.skip_tool_message_in_guardrail = True + tool_result: Final = {"type": output_type, "call_id": "call_1", "output": "SECRET forecast"} + + result: Final = await OpenAIResponsesHandler().process_input_messages( + _tool_result_request(call_item, output_type, "SECRET forecast"), guardrail + ) + + assert guardrail.seen_texts == [["Weather?"]] + assert "SECRET forecast" not in guardrail.seen_message_contents[0] + assert result["input"][0] == {"role": "user", "content": "Weather? [GUARDRAILED]"} + assert result["input"][2] == tool_result + + @pytest.mark.asyncio + @pytest.mark.parametrize(("call_item", "output_type"), _TOOL_TURNS, ids=["function", "custom"]) + async def test_scan_only_tool_results_sends_only_the_tool_result( + self, call_item: dict[str, str], output_type: str + ) -> None: + guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test") + guardrail.scan_only_tool_results = True + tool: Final = {"type": "function", "name": "get_weather", "parameters": {"type": "object", "properties": {}}} + request: Final = { + **_tool_result_request(call_item, output_type, "SECRET forecast"), + "instructions": "Answer briefly", + "tools": [tool], + } + + result: Final = await OpenAIResponsesHandler().process_input_messages(request, guardrail) + + assert guardrail.seen_texts == [["SECRET forecast"]] + assert guardrail.seen_message_contents == [["SECRET forecast"]] + assert guardrail.seen_tools == [None] + assert result["instructions"] == "Answer briefly" + assert result["tools"] == [tool] + assert result["input"] == [ + {"role": "user", "content": "Weather?"}, + call_item, + {"type": output_type, "call_id": "call_1", "output": "SECRET forecast [GUARDRAILED]"}, + ] + + @pytest.mark.asyncio + async def test_scan_only_tool_results_appends_tools_the_guardrail_adds(self) -> None: + added_tool: Final = { + "type": "function", + "function": {"name": "retrieve", "description": "Fetch the full text", "parameters": {"type": "object"}}, + } + renamed_original: Final = { + "type": "function", + "function": {"name": "get_weather", "description": "rewritten", "parameters": {"type": "object"}}, + } + + class ToolAddingGuardrail(RecordingMaskingGuardrail): + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict[str, object], + input_type: Literal["request", "response"], + logging_obj: LiteLLMLoggingObj | None = None, + ) -> GenericGuardrailAPIInputs: + masked: Final = await super().apply_guardrail(inputs, request_data, input_type, logging_obj) + return {**masked, "tools": [renamed_original, added_tool]} + + guardrail: Final = ToolAddingGuardrail(guardrail_name="test") + guardrail.scan_only_tool_results = True + call_item, output_type = _TOOL_TURNS[0] + tool: Final = {"type": "function", "name": "get_weather", "parameters": {"type": "object", "properties": {}}} + request: Final = {**_tool_result_request(call_item, output_type, "SECRET forecast"), "tools": [tool]} + + result: Final = await OpenAIResponsesHandler().process_input_messages(request, guardrail) + + assert guardrail.seen_tools == [None] + assert result["tools"][0] == tool + assert [(t["type"], t["name"]) for t in result["tools"]] == [ + ("function", "get_weather"), + ("function", "retrieve"), + ] + assert result["tools"][1]["description"] == "Fetch the full text" + + @pytest.mark.asyncio + async def test_scan_only_tool_results_leaves_a_request_without_tool_results_unscanned(self) -> None: + guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test") + guardrail.scan_only_tool_results = True + request: Final = {"model": "gpt-5.4-mini", "instructions": "Answer briefly", "input": "Weather?"} + + result: Final = await OpenAIResponsesHandler().process_input_messages(dict(request), guardrail) + + assert guardrail.seen_texts == [] + assert result == request + + class TestOpenAIResponsesHandlerOutputProcessing: """Test output processing functionality""" diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index 020f491dcfd..66e23115f3e 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -12,8 +12,11 @@ from pydantic import JsonValue, TypeAdapter import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig +from litellm.llms.azure_ai.responses.transformation import AzureAIResponsesAPIConfig from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig +from litellm.llms.hosted_vllm.responses.transformation import HostedVLLMResponsesAPIConfig from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.llms.xai.responses.transformation import XAIResponsesAPIConfig from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig from litellm.types.llms.openai import ( ImageGenerationPartialImageEvent, @@ -4277,3 +4280,116 @@ def test_dict_responses_input_filters_unset_reasoning_fields() -> None: function_call_item: Final = validated_input[2] assert function_call_item["type"] == "function_call" assert function_call_item["status"] == "completed" + + +_COMPACT_CONFIGS: Final = ( + pytest.param(OpenAIResponsesAPIConfig(), "https://api.openai.com/v1/responses", id="openai"), + pytest.param( + AzureOpenAIResponsesAPIConfig(), + "https://x.openai.azure.com/openai/responses?api-version=2025-04-01-preview", + id="azure", + ), + pytest.param(AzureAIResponsesAPIConfig(), "https://x.services.ai.azure.com/openai/v1/responses", id="azure_ai"), +) + + +@pytest.mark.parametrize(("config", "api_base"), _COMPACT_CONFIGS) +def test_compact_request_omits_metadata_and_user_and_keeps_the_rest_without_drop_params( + config: OpenAIResponsesAPIConfig, api_base: str +) -> None: + _, data = config.transform_compact_response_api_request( + model="gpt-5.4-mini", + input="hi", + response_api_optional_request_params={ + "metadata": {"ticket": "qa-1"}, + "user": "end-user-1", + "instructions": "be brief", + "previous_response_id": "resp_1", + "reasoning": {"effort": "low"}, + "temperature": 0.2, + }, + api_base=api_base, + litellm_params=GenericLiteLLMParams(drop_params=False), + headers={}, + ) + + assert {key: data[key] for key in data if key != "input"} == { + "model": "gpt-5.4-mini", + "instructions": "be brief", + "previous_response_id": "resp_1", + "reasoning": {"effort": "low"}, + "temperature": 0.2, + } + + +@pytest.mark.parametrize(("config", "api_base"), _COMPACT_CONFIGS) +def test_compact_request_drops_params_compact_rejects_under_drop_params( + config: OpenAIResponsesAPIConfig, api_base: str +) -> None: + _, data = config.transform_compact_response_api_request( + model="gpt-5.4-mini", + input="hi", + response_api_optional_request_params={ + "metadata": {"ticket": "qa-1"}, + "user": "end-user-1", + "temperature": 0.2, + "store": False, + "truncation": "auto", + "max_output_tokens": 100, + "tool_choice": "auto", + "reasoning": {"effort": "low"}, + "include": ["reasoning.encrypted_content"], + "text": {"format": {"type": "text"}}, + }, + api_base=api_base, + litellm_params=GenericLiteLLMParams(drop_params=True), + headers={}, + ) + + assert {key: data[key] for key in data if key != "input"} == { + "model": "gpt-5.4-mini", + "reasoning": {"effort": "low"}, + "include": ["reasoning.encrypted_content"], + "text": {"format": {"type": "text"}}, + } + + +@pytest.mark.parametrize( + ("config", "api_base"), + ( + pytest.param(XAIResponsesAPIConfig(), "https://api.x.ai/v1/responses", id="xai"), + pytest.param(HostedVLLMResponsesAPIConfig(), "http://vllm.example.com/v1/responses", id="hosted_vllm"), + ), +) +def test_compact_request_forwards_every_param_for_providers_not_known_to_reject_them( + config: OpenAIResponsesAPIConfig, api_base: str +) -> None: + _, data = config.transform_compact_response_api_request( + model="some-model", + input="hi", + response_api_optional_request_params={"metadata": {"ticket": "qa-1"}, "user": "end-user-1", "temperature": 0.2}, + api_base=api_base, + litellm_params=GenericLiteLLMParams(drop_params=True), + headers={}, + ) + + assert {key: data[key] for key in data if key != "input"} == { + "model": "some-model", + "metadata": {"ticket": "qa-1"}, + "user": "end-user-1", + "temperature": 0.2, + } + + +@pytest.mark.parametrize(("config", "api_base"), _COMPACT_CONFIGS) +def test_compact_request_keeps_stream_under_drop_params(config: OpenAIResponsesAPIConfig, api_base: str) -> None: + _, data = config.transform_compact_response_api_request( + model="gpt-5.4-mini", + input="hi", + response_api_optional_request_params={"stream": True, "stream_options": {"include_obfuscation": False}}, + api_base=api_base, + litellm_params=GenericLiteLLMParams(drop_params=True), + headers={}, + ) + + assert {key: data[key] for key in data if key != "input"} == {"model": "gpt-5.4-mini", "stream": True} diff --git a/tests/unit/llms/openai_like/test_streamlake_provider.py b/tests/unit/llms/openai_like/test_streamlake_provider.py new file mode 100644 index 00000000000..f1a7b9edba5 --- /dev/null +++ b/tests/unit/llms/openai_like/test_streamlake_provider.py @@ -0,0 +1,402 @@ +import json +from pathlib import Path +from typing import Final + +import httpx +import pytest +import respx + +import litellm +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.llms.openai_like.dynamic_config import create_config_class +from litellm.llms.openai_like.json_loader import JSONProviderRegistry +from litellm.proxy.auth.model_checks import get_known_models_from_wildcard +from litellm.types.utils import ModelResponse, Usage + +STREAMLAKE_BASE_URL: Final = "https://vanchin.streamlake.ai/api/gateway/v1/endpoints" +STREAMLAKE_CHAT_URL: Final = f"{STREAMLAKE_BASE_URL}/chat/completions" + +REPO_ROOT: Final = Path(__file__).parents[4] + + +class TestStreamlakeProviderConfig: + def test_streamlake_provider_resolution(self) -> None: + model, provider, api_key, api_base = get_llm_provider( + model="streamlake/ep-test-endpoint", + custom_llm_provider=None, + api_base=None, + api_key="sk-test", + ) + + assert model == "ep-test-endpoint" + assert provider == "streamlake" + assert api_base == STREAMLAKE_BASE_URL + assert api_key == "sk-test" + + def test_streamlake_api_base_override(self) -> None: + model, provider, api_key, api_base = get_llm_provider( + model="streamlake/ep-test-endpoint", + custom_llm_provider=None, + api_base="https://custom.streamlake.example/v1", + api_key="sk-test", + ) + + assert model == "ep-test-endpoint" + assert provider == "streamlake" + assert api_base == "https://custom.streamlake.example/v1" + assert api_key == "sk-test" + + def test_streamlake_url_autodetection(self) -> None: + _, provider, _, api_base = get_llm_provider( + model="some-public-model", + custom_llm_provider=None, + api_base=STREAMLAKE_BASE_URL, + api_key=None, + ) + assert provider == "streamlake" + assert api_base == STREAMLAKE_BASE_URL + + def test_streamlake_api_key_resolved_from_env(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("STREAMLAKE_API_KEY", "sk-env-key") + + _, provider, api_key, _ = get_llm_provider( + model="streamlake/ep-test-endpoint", + custom_llm_provider=None, + api_base=None, + api_key=None, + ) + assert provider == "streamlake" + assert api_key == "sk-env-key" + + def test_streamlake_validate_environment_sets_bearer(self) -> None: + provider: Final = JSONProviderRegistry.get("streamlake") + assert provider is not None + config: Final = create_config_class(provider)() + + headers: Final = config.validate_environment( + headers={}, + model="ep-test-endpoint", + messages=[{"role": "user", "content": "hi"}], + optional_params={}, + litellm_params={}, + api_key="sk-resolved-key", + api_base=None, + ) + assert headers["Authorization"] == "Bearer sk-resolved-key" + + def test_streamlake_complete_url_appends_chat_completions(self) -> None: + provider: Final = JSONProviderRegistry.get("streamlake") + assert provider is not None + config: Final = create_config_class(provider)() + + url: Final = config.get_complete_url( + api_base=None, + api_key="sk-test", + model="ep-test-endpoint", + optional_params={}, + litellm_params={}, + ) + assert url == STREAMLAKE_CHAT_URL + + @pytest.mark.parametrize("cap_param", ("max_tokens", "max_completion_tokens")) + def test_streamlake_token_cap_sent_as_max_completion_tokens(self, cap_param: str) -> None: + provider: Final = JSONProviderRegistry.get("streamlake") + assert provider is not None + config: Final = create_config_class(provider)() + + optional_params: Final = config.map_openai_params( + non_default_params={cap_param: 1024}, + optional_params={}, + model="ep-test-endpoint", + drop_params=False, + ) + assert optional_params == {"max_completion_tokens": 1024} + + def test_streamlake_chat_params_pass_through(self) -> None: + provider: Final = JSONProviderRegistry.get("streamlake") + assert provider is not None + config: Final = create_config_class(provider)() + + optional_params: Final = config.map_openai_params( + non_default_params={"temperature": 0.2, "top_p": 0.9, "stream": True}, + optional_params={}, + model="ep-test-endpoint", + drop_params=False, + ) + assert optional_params == {"temperature": 0.2, "top_p": 0.9, "stream": True} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "is_async, api_base", + ( + (False, None), + (False, "https://custom.streamlake.example/v1"), + (False, "https://custom.streamlake.example/v1/"), + (True, None), + (True, "https://custom.streamlake.example/v1"), + (True, "https://custom.streamlake.example/v1/"), + ), + ids=( + "sync-default", + "sync-override", + "sync-override-trailing-slash", + "async-default", + "async-override", + "async-override-trailing-slash", + ), +) +async def test_streamlake_completion_uses_openai_compatible_http( + is_async: bool, api_base: str | None, monkeypatch: pytest.MonkeyPatch +) -> None: + response_body: Final[dict[str, object]] = { + "id": "chatcmpl-streamlake-test", + "object": "chat.completion", + "created": 1710000000, + "model": "ep-test-endpoint", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "StreamLake response"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}, + } + expected_url: Final[str] = f"{(api_base or STREAMLAKE_BASE_URL).rstrip('/')}/chat/completions" + expected_request_body: Final[dict[str, object]] = { + "messages": [{"role": "user", "content": "hello"}], + "model": "ep-test-endpoint", + "max_completion_tokens": 512, + } + + def handler(request: httpx.Request) -> httpx.Response: + assert request.method == "POST" + assert str(request.url) == expected_url + assert request.headers["authorization"] == "Bearer sk-streamlake-test" + assert request.headers["content-type"] == "application/json" + assert json.loads(request.content) == expected_request_body + return httpx.Response(200, json=response_body, request=request) + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post(expected_url).mock(side_effect=handler) + if is_async: + response: Final = await litellm.acompletion( + model="streamlake/ep-test-endpoint", + messages=[{"role": "user", "content": "hello"}], + api_key="sk-streamlake-test", + api_base=api_base, + max_tokens=512, + ) + else: + response: Final = litellm.completion( + model="streamlake/ep-test-endpoint", + messages=[{"role": "user", "content": "hello"}], + api_key="sk-streamlake-test", + api_base=api_base, + max_tokens=512, + ) + + assert route.call_count == 1 + assert len(response.choices) == 1 + assert response.choices[0].message.role == "assistant" + assert response.choices[0].message.content == "StreamLake response" + assert response.usage is not None + assert response.usage.prompt_tokens == 3 + assert response.usage.completion_tokens == 2 + assert response.usage.total_tokens == 5 + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_endpoint_id_call_forwards_tools_and_reasoning_effort_to_streamlake(monkeypatch: pytest.MonkeyPatch) -> None: + tools: Final = [ + { + "type": "function", + "function": { + "name": "get_weather", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, + }, + } + ] + response_body: Final[dict[str, object]] = { + "id": "chatcmpl-streamlake-tools", + "object": "chat.completion", + "created": 1710000000, + "model": "glm-5.3", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}, + } + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post(STREAMLAKE_CHAT_URL).mock(return_value=httpx.Response(200, json=response_body)) + litellm.completion( + model="streamlake/ep-test-endpoint", + messages=[{"role": "user", "content": "weather in Paris?"}], + api_key="sk-streamlake-test", + tools=tools, + tool_choice="auto", + reasoning_effort="low", + ) + + sent: Final = json.loads(route.calls.last.request.content) + assert sent["tools"] == tools + assert sent["tool_choice"] == "auto" + assert sent["reasoning_effort"] == "low" + + +def _streamlake_keys(cost_map: dict[str, object]) -> tuple[str, ...]: + return tuple(key for key in cost_map if key.startswith("streamlake/")) + + +def test_root_and_backup_cost_maps_carry_the_same_streamlake_rows() -> None: + root: Final = json.loads((REPO_ROOT / "model_prices_and_context_window.json").read_text()) + backup: Final = json.loads((REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json").read_text()) + + assert _streamlake_keys(root) + assert {key: root[key] for key in _streamlake_keys(root)} == {key: backup[key] for key in _streamlake_keys(backup)} + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_endpoint_id_call_is_priced_from_the_lowercase_model_streamlake_returns() -> None: + keys: Final = _streamlake_keys(litellm.model_cost) + assert keys + + for key in keys: + info: Final = litellm.get_model_info(model=key) + response: Final = ModelResponse( + model=key.split("/", 1)[1].lower(), + usage=Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500), + ) + cost: Final = litellm.completion_cost( + completion_response=response, + model="streamlake/ep-test-endpoint", + custom_llm_provider="streamlake", + ) + expected: Final = 1000 * info["input_cost_per_token"] + 500 * info["output_cost_per_token"] + assert cost == pytest.approx(expected), key + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_named_model_params_follow_the_row_and_fall_back_to_the_provider_default() -> None: + provider: Final = JSONProviderRegistry.get("streamlake") + assert provider is not None + config: Final = create_config_class(provider)() + keys: Final = _streamlake_keys(litellm.model_cost) + assert keys + + for key in keys: + row: Final = litellm.model_cost[key] + supported: Final = config.get_supported_openai_params(model=key.split("/", 1)[1].lower()) + assert ("tools" in supported) is (row.get("supports_function_calling") is not False), key + assert ("reasoning_effort" in supported) is (row.get("supports_reasoning") is not False), key + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_streamlake_wildcard_expands_to_every_streamlake_row() -> None: + expanded: Final = get_known_models_from_wildcard("streamlake/*") + assert set(expanded) == set(_streamlake_keys(litellm.model_cost)) + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_cache_control_marks_reach_streamlake(monkeypatch: pytest.MonkeyPatch) -> None: + system_block: Final = {"type": "text", "text": "You are terse.", "cache_control": {"type": "ephemeral"}} + tool: Final = { + "type": "function", + "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}}, + "cache_control": {"type": "ephemeral"}, + } + response_body: Final[dict[str, object]] = { + "id": "chatcmpl-streamlake-cache", + "object": "chat.completion", + "created": 1710000000, + "model": "qwen3.8-27b", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}, + } + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post(STREAMLAKE_CHAT_URL).mock(return_value=httpx.Response(200, json=response_body)) + litellm.completion( + model="streamlake/qwen3.8-27b", + messages=[{"role": "system", "content": [system_block]}, {"role": "user", "content": "hi"}], + api_key="sk-streamlake-test", + tools=[tool], + ) + + sent: Final = json.loads(route.calls.last.request.content) + assert sent["messages"][0]["content"] == [system_block] + assert sent["tools"][0]["cache_control"] == {"type": "ephemeral"} + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize("model", ("kimi-k2.6", "qwen3.6-27b", "qwen3.6-35b-a3b")) +def test_reasoning_effort_is_dropped_for_models_streamlake_lists_without_reasoning( + model: str, monkeypatch: pytest.MonkeyPatch +) -> None: + # StreamLake's Models Overview (https://www.streamlake.ai/document/DOC/mh1gbfvrdn6hpbzxixv, read 2026-10-28) + # gives these models no Reasoning tag, and kimi-k2.6 answers 400 to reasoning_effort under an 8193 token cap + response_body: Final[dict[str, object]] = { + "id": "chatcmpl-streamlake-reasoning", + "object": "chat.completion", + "created": 1710000000, + "model": model, + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}, + } + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post(STREAMLAKE_CHAT_URL).mock(return_value=httpx.Response(200, json=response_body)) + litellm.completion( + model=f"streamlake/{model}", + messages=[{"role": "user", "content": "hi"}], + api_key="sk-streamlake-test", + reasoning_effort="low", + drop_params=True, + ) + + assert "reasoning_effort" not in json.loads(route.calls.last.request.content) + with pytest.raises(litellm.UnsupportedParamsError): + litellm.completion( + model=f"streamlake/{model}", + messages=[{"role": "user", "content": "hi"}], + api_key="sk-streamlake-test", + reasoning_effort="low", + ) + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_streamlake_rows_with_a_cache_read_price_declare_prompt_caching() -> None: + cached_rows: Final = tuple( + key for key in _streamlake_keys(litellm.model_cost) if "cache_read_input_token_cost" in litellm.model_cost[key] + ) + assert cached_rows + assert [key for key in cached_rows if litellm.model_cost[key].get("supports_prompt_caching") is not True] == [] + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize("model", ("qwen3.8-27b", "qwen3.8-2.4t-a95b")) +@pytest.mark.parametrize("cache_type", ("ephemeral", None)) +def test_qwen3_8_explicit_cache_hits_and_prefix_hits_use_their_own_read_price( + model: str, cache_type: str | None +) -> None: + # StreamLake's price page (https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce, read 2026-10-28) lists + # separate PrefixCache and Cache Read prices for Qwen3.8, and its usage tags only explicit hits with cache_type + rate_key: Final = "explicit_cache_read_input_token_cost" if cache_type else "cache_read_input_token_cost" + row: Final = litellm.model_cost[f"streamlake/{model}"] + details: Final = {"cached_tokens": 4000} | ({"cache_type": cache_type} if cache_type else {}) + response: Final = ModelResponse( + model=model, + usage=Usage(prompt_tokens=5000, completion_tokens=100, total_tokens=5100, prompt_tokens_details=details), + ) + + cost: Final = litellm.completion_cost( + completion_response=response, model=f"streamlake/{model}", custom_llm_provider="streamlake" + ) + + expected: Final = 4000 * row[rate_key] + 1000 * row["input_cost_per_token"] + 100 * row["output_cost_per_token"] + assert cost == pytest.approx(expected) diff --git a/tests/unit/proxy/auth/test_auth_checks.py b/tests/unit/proxy/auth/test_auth_checks.py index e1afa81d372..e783f01769e 100644 --- a/tests/unit/proxy/auth/test_auth_checks.py +++ b/tests/unit/proxy/auth/test_auth_checks.py @@ -12,7 +12,7 @@ load_dotenv() from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence from copy import deepcopy from types import SimpleNamespace -from typing import Final +from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch import pytest, litellm @@ -721,6 +721,51 @@ async def test_can_key_call_model_wildcard_access(key_models, model, expect_to_w ) +@pytest.fixture +def shipped_routing_rules() -> Iterator[None]: + from litellm.litellm_core_utils.fallback_generalizations import ( + get_fallback_generalization_rules, + set_fallback_generalizations, + ) + from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap + + previous_rules: Final = list(get_fallback_generalization_rules()) + set_fallback_generalizations(GetModelCostMap.load_local_model_cost_map()["fallback_generalizations"]["rules"]) + yield + set_fallback_generalizations(previous_rules) + + +@pytest.mark.parametrize("object_type", ["team", "project"]) +@pytest.mark.parametrize( + "allowed_models, model, expect_to_work", + [ + (["typesafe/*"], "jev-latest", True), + (["typesafe/*"], "jev-1.14.0", True), + (["jev-latest"], "jev-latest", True), + (["openai/*"], "jev-latest", False), + (["typesafe/*"], "foo/jev-latest", False), + (["typesafe/*"], "jev-internal-router", False), + (["typesafe/*"], "jev-2.0-internal", False), + (["typesafe/*"], "jev-1.0", False), + ], +) +def test_typesafe_provider_wildcard_covers_bare_jev_model( + shipped_routing_rules: None, + object_type: Literal["team", "project"], + allowed_models: list[str], + model: str, + expect_to_work: bool, +) -> None: + from litellm.proxy._types import ProxyException + from litellm.proxy.auth.auth_checks import can_object_call_model + + if expect_to_work: + assert can_object_call_model(model=model, llm_router=None, models=allowed_models, object_type=object_type) + return + with pytest.raises(ProxyException): + can_object_call_model(model=model, llm_router=None, models=allowed_models, object_type=object_type) + + @pytest.mark.parametrize( "key_models, model, expect_to_work", [ diff --git a/tests/unit/proxy/db/test_pgbouncer.py b/tests/unit/proxy/db/test_pgbouncer.py index c69c8d015a3..08861073008 100644 --- a/tests/unit/proxy/db/test_pgbouncer.py +++ b/tests/unit/proxy/db/test_pgbouncer.py @@ -277,7 +277,7 @@ def _free_port() -> int: def _fake_pooler( - tmp_path: Path, + directory: Path, port: int, exit_immediately: bool = False, port_file: Path | None = None, @@ -288,13 +288,13 @@ def _fake_pooler( """An executable that listens like PgBouncer: on the TCP port first, then on ``.s.PGSQL.`` in the socket dir. Port and socket dir come from the ini it is given, else from ``port`` and - ``tmp_path``. With ``port_file`` each start reads the port from that file + ``directory``. With ``port_file`` each start reads the port from that file instead. ``bind_delay_seconds`` holds the bind back, like a slow start. ``--version`` prints ``version_banner``. With ``auth_log`` it appends the ``auth_file`` it reads at startup and on every SIGHUP, one line per read, like PgBouncer loading its credentials. """ - script: Final = tmp_path / "fake-pgbouncer" + script: Final = directory / "fake-pgbouncer" script.write_text( textwrap.dedent( f"""\ @@ -316,7 +316,7 @@ def _fake_pooler( port = ini.getint("pgbouncer", "listen_port", fallback={port}) if not {port_file is None!r}: port = int(pathlib.Path({str(port_file)!r}).read_text()) - socket_dir = ini.get("pgbouncer", "unix_socket_dir", fallback={str(tmp_path)!r}) + socket_dir = ini.get("pgbouncer", "unix_socket_dir", fallback={str(directory)!r}) socket_path = f"{{socket_dir}}/.s.PGSQL.{{port}}" time.sleep({bind_delay_seconds!r}) listener = socket.socket() @@ -357,10 +357,10 @@ def _wait_until(condition: Callable[[], bool], timeout_seconds: float = 5.0) -> class TestPgBouncerProcess: - def test_start_waits_for_the_listener_and_stop_ends_it(self, tmp_path: Path): + def test_start_waits_for_the_listener_and_stop_ends_it(self, socket_dir: Path): port: Final = _free_port() pooler: Final = PgBouncerProcess( - argv=(str(_fake_pooler(tmp_path, port)),), port=port, socket_path=unix_socket_path(tmp_path, port) + argv=(str(_fake_pooler(socket_dir, port)),), port=port, socket_path=unix_socket_path(socket_dir, port) ) assert pooler.start() is None assert _listening(port) @@ -371,12 +371,12 @@ class TestPgBouncerProcess: with pytest.raises(ProcessLookupError): os.kill(pid, 0) - def test_a_crashed_pooler_is_restarted_with_a_new_pid(self, tmp_path: Path): + def test_a_crashed_pooler_is_restarted_with_a_new_pid(self, socket_dir: Path): port: Final = _free_port() pooler: Final = PgBouncerProcess( - argv=(str(_fake_pooler(tmp_path, port)),), + argv=(str(_fake_pooler(socket_dir, port)),), port=port, - socket_path=unix_socket_path(tmp_path, port), + socket_path=unix_socket_path(socket_dir, port), restart_delay_seconds=0.1, ) assert pooler.start() is None @@ -388,17 +388,17 @@ class TestPgBouncerProcess: assert _wait_until(lambda: not _listening(port)) def test_a_failed_restart_is_retried_until_the_pooler_is_back( - self, tmp_path: Path, caplog: pytest.LogCaptureFixture + self, socket_dir: Path, caplog: pytest.LogCaptureFixture ): port: Final = _free_port() - script: Final = _fake_pooler(tmp_path, port) + script: Final = _fake_pooler(socket_dir, port) pooler: Final = PgBouncerProcess( - argv=(str(script),), port=port, socket_path=unix_socket_path(tmp_path, port), restart_delay_seconds=0.1 + argv=(str(script),), port=port, socket_path=unix_socket_path(socket_dir, port), restart_delay_seconds=0.1 ) assert pooler.start() is None first_pid: Final = pooler.pid assert first_pid is not None - hidden: Final = script.rename(tmp_path / "hidden") + hidden: Final = script.rename(socket_dir / "hidden") with caplog.at_level(logging.ERROR, logger=verbose_proxy_logger.name): os.kill(first_pid, signal.SIGKILL) assert _wait_until(lambda: any("could not be restarted" in record.message for record in caplog.records)) @@ -408,15 +408,17 @@ class TestPgBouncerProcess: pooler.stop() assert _wait_until(lambda: not _listening(port)) - def test_a_replacement_that_never_listens_is_replaced_again(self, tmp_path: Path, caplog: pytest.LogCaptureFixture): + def test_a_replacement_that_never_listens_is_replaced_again( + self, socket_dir: Path, caplog: pytest.LogCaptureFixture + ): port: Final = _free_port() - port_file: Final = tmp_path / "port" + port_file: Final = socket_dir / "port" port_file.write_text(str(port)) - script: Final = _fake_pooler(tmp_path, port, port_file=port_file) + script: Final = _fake_pooler(socket_dir, port, port_file=port_file) pooler: Final = PgBouncerProcess( argv=(str(script),), port=port, - socket_path=unix_socket_path(tmp_path, port), + socket_path=unix_socket_path(socket_dir, port), restart_delay_seconds=0.1, ready_timeout_seconds=2.0, ) @@ -435,12 +437,12 @@ class TestPgBouncerProcess: pooler.stop() assert _wait_until(lambda: not _listening(port)) - def test_stopping_during_the_restart_delay_leaves_no_pooler_behind(self, tmp_path: Path): + def test_stopping_during_the_restart_delay_leaves_no_pooler_behind(self, socket_dir: Path): port: Final = _free_port() pooler: Final = PgBouncerProcess( - argv=(str(_fake_pooler(tmp_path, port)),), + argv=(str(_fake_pooler(socket_dir, port)),), port=port, - socket_path=unix_socket_path(tmp_path, port), + socket_path=unix_socket_path(socket_dir, port), restart_delay_seconds=0.3, ) assert pooler.start() is None @@ -453,12 +455,12 @@ class TestPgBouncerProcess: assert not _listening(port) assert pooler.pid == first_pid - def test_a_stopped_pooler_is_not_restarted(self, tmp_path: Path, caplog: pytest.LogCaptureFixture): + def test_a_stopped_pooler_is_not_restarted(self, socket_dir: Path, caplog: pytest.LogCaptureFixture): port: Final = _free_port() pooler: Final = PgBouncerProcess( - argv=(str(_fake_pooler(tmp_path, port)),), + argv=(str(_fake_pooler(socket_dir, port)),), port=port, - socket_path=unix_socket_path(tmp_path, port), + socket_path=unix_socket_path(socket_dir, port), restart_delay_seconds=0.1, ) assert pooler.start() is None @@ -468,31 +470,31 @@ class TestPgBouncerProcess: assert not _listening(port) assert caplog.records == [] - def test_a_pooler_that_exits_during_startup_is_reported(self, tmp_path: Path): + def test_a_pooler_that_exits_during_startup_is_reported(self, socket_dir: Path): port: Final = _free_port() pooler: Final = PgBouncerProcess( - argv=(str(_fake_pooler(tmp_path, port, exit_immediately=True)),), + argv=(str(_fake_pooler(socket_dir, port, exit_immediately=True)),), port=port, - socket_path=unix_socket_path(tmp_path, port), + socket_path=unix_socket_path(socket_dir, port), ) outcome: Final = pooler.start() assert isinstance(outcome, PgBouncerError) assert "status 3" in outcome.reason - def test_a_missing_binary_is_reported(self, tmp_path: Path): + def test_a_missing_binary_is_reported(self, socket_dir: Path): outcome: Final = PgBouncerProcess( - argv=("/nonexistent/pgbouncer",), port=_free_port(), socket_path=tmp_path / "sock" + argv=("/nonexistent/pgbouncer",), port=_free_port(), socket_path=socket_dir / "sock" ).start() assert isinstance(outcome, PgBouncerError) assert "/nonexistent/pgbouncer" in outcome.reason - def test_a_port_owned_by_someone_else_is_refused_before_spawning(self, tmp_path: Path): + def test_a_port_owned_by_someone_else_is_refused_before_spawning(self, socket_dir: Path): with socket.socket() as squatter: squatter.bind(("127.0.0.1", 0)) squatter.listen() port: Final = _bound_port(squatter) pooler: Final = PgBouncerProcess( - argv=(str(_fake_pooler(tmp_path, port)),), port=port, socket_path=unix_socket_path(tmp_path, port) + argv=(str(_fake_pooler(socket_dir, port)),), port=port, socket_path=unix_socket_path(socket_dir, port) ) outcome: Final = pooler.start() assert isinstance(outcome, PgBouncerError) @@ -500,13 +502,13 @@ class TestPgBouncerProcess: assert pooler.pid is None def test_a_replacement_waits_until_a_squatter_leaves_the_port( - self, tmp_path: Path, caplog: pytest.LogCaptureFixture + self, socket_dir: Path, caplog: pytest.LogCaptureFixture ): port: Final = _free_port() pooler: Final = PgBouncerProcess( - argv=(str(_fake_pooler(tmp_path, port)),), + argv=(str(_fake_pooler(socket_dir, port)),), port=port, - socket_path=unix_socket_path(tmp_path, port), + socket_path=unix_socket_path(socket_dir, port), restart_delay_seconds=0.5, ) assert pooler.start() is None @@ -524,12 +526,12 @@ class TestPgBouncerProcess: pooler.stop() assert _wait_until(lambda: not _listening(port)) - def test_a_listener_that_grabs_the_port_after_the_spawn_is_not_taken_for_the_pooler(self, tmp_path: Path): + def test_a_listener_that_grabs_the_port_after_the_spawn_is_not_taken_for_the_pooler(self, socket_dir: Path): port: Final = _free_port() pooler: Final = PgBouncerProcess( - argv=(str(_fake_pooler(tmp_path, port, bind_delay_seconds=0.5)),), + argv=(str(_fake_pooler(socket_dir, port, bind_delay_seconds=0.5)),), port=port, - socket_path=unix_socket_path(tmp_path, port), + socket_path=unix_socket_path(socket_dir, port), ready_timeout_seconds=3.0, ) with socket.socket() as squatter, ThreadPoolExecutor(max_workers=1) as starter: @@ -542,12 +544,12 @@ class TestPgBouncerProcess: assert isinstance(outcome, PgBouncerError) assert "exited with status 1" in outcome.reason - def test_a_port_served_by_a_stranger_while_the_pooler_is_still_starting_is_reported(self, tmp_path: Path): + def test_a_port_served_by_a_stranger_while_the_pooler_is_still_starting_is_reported(self, socket_dir: Path): port: Final = _free_port() pooler: Final = PgBouncerProcess( - argv=(str(_fake_pooler(tmp_path, port, bind_delay_seconds=30.0)),), + argv=(str(_fake_pooler(socket_dir, port, bind_delay_seconds=30.0)),), port=port, - socket_path=unix_socket_path(tmp_path, port), + socket_path=unix_socket_path(socket_dir, port), ready_timeout_seconds=0.5, ) with socket.socket() as squatter, ThreadPoolExecutor(max_workers=1) as starter: @@ -564,12 +566,12 @@ class TestPgBouncerProcess: with pytest.raises(ProcessLookupError): os.kill(pid, 0) - def test_a_pooler_that_never_listens_times_out(self, tmp_path: Path): + def test_a_pooler_that_never_listens_times_out(self, socket_dir: Path): port: Final = _free_port() pooler: Final = PgBouncerProcess( - argv=(str(_fake_pooler(tmp_path, _free_port())),), + argv=(str(_fake_pooler(socket_dir, _free_port())),), port=port, - socket_path=unix_socket_path(tmp_path, port), + socket_path=unix_socket_path(socket_dir, port), ready_timeout_seconds=0.5, ) outcome: Final = pooler.start() @@ -704,10 +706,10 @@ def _runtime_dir_listening_on(port: int) -> Path: class TestStartInContainerPgBouncer: - def test_returns_the_loopback_url_once_the_pooler_listens(self, tmp_path: Path): + def test_returns_the_loopback_url_once_the_pooler_listens(self, socket_tempdir: Path): port: Final = _free_port() - auth_log: Final = tmp_path / "auth.log" - binary: Final = _fake_pooler(tmp_path, port, auth_log=auth_log) + auth_log: Final = socket_tempdir / "auth.log" + binary: Final = _fake_pooler(socket_tempdir, port, auth_log=auth_log) settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(binary)) pooled: Final = start_in_container_pgbouncer(settings, "postgresql://app:pw@db/litellm?connection_limit=5") assert isinstance(pooled, str), pooled @@ -723,9 +725,9 @@ class TestStartInContainerPgBouncer: assert auth_log.read_text() == repr(f'"app" "pw"\n"litellm_pgbouncer" "{parsed.password}"\n') + "\n" @pytest.mark.filterwarnings("ignore:This process .* is multi-threaded:DeprecationWarning") - def test_a_forked_worker_exiting_leaves_the_pooler_and_its_files_to_the_parent(self, tmp_path: Path): + def test_a_forked_worker_exiting_leaves_the_pooler_and_its_files_to_the_parent(self, socket_tempdir: Path): port: Final = _free_port() - settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pooler(tmp_path, port))) + settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pooler(socket_tempdir, port))) exit_hooks: Final[list[Callable[[], None]]] = [] pooled: Final = start_in_container_pgbouncer( settings, "postgresql://app:pw@db/litellm", register_exit_hook=exit_hooks.append @@ -766,10 +768,10 @@ class TestStartInContainerPgBouncer: assert "IAM_TOKEN_DB_AUTH" in outcome.reason assert not _listening(port) - def test_token_auth_mints_the_first_token_into_the_auth_file_before_the_pooler_starts(self, tmp_path: Path): + def test_token_auth_mints_the_first_token_into_the_auth_file_before_the_pooler_starts(self, socket_tempdir: Path): port: Final = _free_port() - auth_log: Final = tmp_path / "auth.log" - binary: Final = _fake_pooler(tmp_path, port, auth_log=auth_log) + auth_log: Final = socket_tempdir / "auth.log" + binary: Final = _fake_pooler(socket_tempdir, port, auth_log=auth_log) settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(binary)) token: Final = _entra_jwt(datetime.now(tz=timezone.utc) + timedelta(hours=1)) pooled: Final = start_in_container_pgbouncer( @@ -798,25 +800,25 @@ class TestStartInContainerPgBouncer: assert "no Azure credential" in outcome.reason assert not _listening(port) - def test_a_renewed_token_is_written_and_picked_up_by_the_running_and_by_a_restarted_pooler(self, tmp_path: Path): + def test_a_renewed_token_is_written_and_picked_up_by_the_running_and_by_a_restarted_pooler(self, socket_dir: Path): port: Final = _free_port() - auth_log: Final = tmp_path / "auth.log" + auth_log: Final = socket_dir / "auth.log" plan: Final = plan_pgbouncer( - "postgresql://app@db/litellm", PgBouncerSettings(enabled=True, port=port), tmp_path, None + "postgresql://app@db/litellm", PgBouncerSettings(enabled=True, port=port), socket_dir, None ) assert isinstance(plan, PgBouncerPlan), plan - ini_path: Final = write_pgbouncer_ini(plan, tmp_path, None) - write_userlist(plan.userlist("first"), tmp_path, None) + ini_path: Final = write_pgbouncer_ini(plan, socket_dir, None) + write_userlist(plan.userlist("first"), socket_dir, None) pooler: Final = PgBouncerProcess( - argv=(str(_fake_pooler(tmp_path, port, auth_log=auth_log)), str(ini_path)), + argv=(str(_fake_pooler(socket_dir, port, auth_log=auth_log)), str(ini_path)), port=port, - socket_path=unix_socket_path(tmp_path, port), + socket_path=unix_socket_path(socket_dir, port), restart_delay_seconds=0.1, ) assert pooler.start() is None first_pid: Final = pooler.pid assert first_pid is not None - install_pgbouncer_token(plan, tmp_path, None, pooler, "second") + install_pgbouncer_token(plan, socket_dir, None, pooler, "second") assert _wait_until(lambda: auth_log.read_text().count("\n") == 2) os.kill(first_pid, signal.SIGKILL) assert _wait_until(lambda: pooler.pid not in (None, first_pid) and _listening(port)) @@ -837,9 +839,9 @@ class TestStartInContainerPgBouncer: assert "1.19" in outcome.reason assert not _listening(port) - def test_the_first_version_that_dies_on_a_failed_tcp_bind_is_accepted(self, tmp_path: Path): + def test_the_first_version_that_dies_on_a_failed_tcp_bind_is_accepted(self, socket_tempdir: Path): port: Final = _free_port() - binary: Final = _fake_pooler(tmp_path, port, version_banner="PgBouncer 1.19.0") + binary: Final = _fake_pooler(socket_tempdir, port, version_banner="PgBouncer 1.19.0") settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(binary)) pooled: Final = start_in_container_pgbouncer(settings, "postgresql://app:pw@db/litellm") assert isinstance(pooled, str), pooled diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py index 83301db5d86..58055daea4c 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py @@ -4834,6 +4834,83 @@ class TestPanwAirsLatestRoleMessageOnlyEveryRequestShape: None, id="function_call_output", ), + pytest.param( + ( + {"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"}, + { + "type": "function_call_output", + "call_id": "call_1", + "output": [{"type": "input_text", "text": "tool "}, {"type": "input_text", "text": "result"}], + }, + ), + None, + id="function_call_output_text_parts", + ), + pytest.param( + ( + {"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"}, + { + "type": "function_call_output", + "call_id": "call_1", + "output": [ + {"type": "input_text", "text": "tool "}, + {"type": "input_image", "image_url": "https://example.com/chart.png"}, + {"type": "input_text", "text": "result"}, + ], + }, + ), + None, + id="function_call_output_text_and_image_parts", + ), + pytest.param( + ( + {"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"}, + {"type": "function_call_output", "output": "orphan tool result"}, + {"type": "function_call_output", "call_id": "call_1", "output": "tool result"}, + ), + None, + id="function_call_output_without_call_id", + ), + pytest.param( + ( + {"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"}, + {"type": "tool_result", "call_id": "call_1", "output": "tool result"}, + ), + None, + id="tool_result", + ), + pytest.param( + ( + {"type": "computer_call", "call_id": "cu_1", "id": "cu_1", "action": {"type": "screenshot"}}, + { + "type": "computer_call_output", + "call_id": "cu_1", + "output": {"type": "computer_screenshot", "image_url": "https://example.com/screen.png"}, + }, + ), + None, + id="computer_call_output", + ), + pytest.param( + ( + {"type": "computer_call", "call_id": "cu_1", "id": "cu_1", "action": {"type": "screenshot"}}, + { + "type": "computer_call_output", + "call_id": "cu_1", + "output": {"type": "computer_screenshot", "file_id": "file-screen"}, + }, + ), + None, + id="computer_call_output_file_id", + ), + pytest.param( + ( + {"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "call_1", "output": ""}, + ), + None, + id="function_call_output_empty", + ), pytest.param( ({"type": "reasoning", "id": "rs_1", "summary": [{"type": "summary_text", "text": "thinking"}]},), None, @@ -4898,6 +4975,25 @@ class TestPanwAirsLatestRoleMessageOnlyEveryRequestShape: assert [call.kwargs["content"] for call in mock_api.call_args_list] == [self.LATEST] + @pytest.mark.asyncio + async def test_flag_true_with_skip_tool_still_scans_only_the_latest_turn_on_responses(self) -> None: + from litellm.llms.openai.responses.guardrail_translation.handler import ( + OpenAIResponsesHandler, + ) + + handler = make_handler(experimental_use_latest_role_message_only=True) + handler.skip_tool_message_in_guardrail = True + request_data = self._responses_request( + {"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "call_1", "output": "tool result"}, + {"role": "user", "content": self.LATEST}, + ) + patcher, mock_api = self._scan(handler) + with patcher: + await OpenAIResponsesHandler().process_input_messages(data=request_data, guardrail_to_apply=handler) + + assert [call.kwargs["content"] for call in mock_api.call_args_list] == [self.LATEST] + @pytest.mark.asyncio async def test_flag_false_responses_scans_instructions_and_full_history(self) -> None: from litellm.llms.openai.responses.guardrail_translation.handler import ( diff --git a/tests/unit/proxy/guardrails/test_content_utils.py b/tests/unit/proxy/guardrails/test_content_utils.py index 21d96a3ea97..c50646d233d 100644 --- a/tests/unit/proxy/guardrails/test_content_utils.py +++ b/tests/unit/proxy/guardrails/test_content_utils.py @@ -12,6 +12,7 @@ from litellm.proxy.guardrails._content_utils import ( build_inspection_messages, has_non_string_content, is_non_conversational_call_type, + is_non_conversational_request, is_string_batch_input, iter_message_text, iter_tool_json_text, @@ -819,6 +820,30 @@ def test_is_non_conversational_call_type_defaults_to_inspecting_unknown_call_typ assert is_non_conversational_call_type("some_future_call_type") is False +@pytest.mark.parametrize( + ("call_type", "endpoint", "expected"), + [ + ("aembedding", None, True), + ("allm_passthrough_route", "embeddings", True), + ("allm_passthrough_route", "/v1/embeddings/", True), + ("allm_passthrough_route", "chat/completions", False), + ("allm_passthrough_route", "model/amazon.titan-embed-text-v2:0/invoke", False), + ("allm_passthrough_route", "fine_tuned_embeddings_jobs", False), + ("allm_passthrough_route", "chat/completions#embeddings", False), + ("allm_passthrough_route", "chat/completions?x=embeddings", False), + ("allm_passthrough_route", "chat/completions;embeddings", False), + ("allm_passthrough_route", None, False), + ("acompletion", "embeddings", False), + ], +) +def test_is_non_conversational_request_skips_only_embeddings_requests( + call_type: str, endpoint: str | None, expected: bool +): + data: Final[dict[str, object]] = {"endpoint": endpoint} if endpoint is not None else {} + + assert is_non_conversational_request(call_type, data) is expected + + @pytest.mark.parametrize( "tool_input", ( diff --git a/tests/unit/proxy/guardrails/test_guardrail_coverage.py b/tests/unit/proxy/guardrails/test_guardrail_coverage.py index 1feae213eba..493eac430d0 100644 --- a/tests/unit/proxy/guardrails/test_guardrail_coverage.py +++ b/tests/unit/proxy/guardrails/test_guardrail_coverage.py @@ -12,17 +12,19 @@ inspection by the relevant guardrail hook: not just ``choices[0]``. """ +from types import SimpleNamespace from typing import Any, Dict, Final from unittest.mock import AsyncMock, MagicMock, patch import pytest +from fastapi import HTTPException from httpx import Request, Response import litellm from litellm import DualCache from litellm.constants import DEFAULT_OPENAI_MODERATIONS_MODEL from litellm.proxy._types import UserAPIKeyAuth -from litellm.types.utils import Choices, Message, ModelResponse +from litellm.types.utils import CallTypesLiteral, Choices, Message, ModelResponse @pytest.fixture @@ -905,3 +907,78 @@ async def test_google_text_moderation_inspects_multimodal_content(user_api_key): ) assert seen_documents == ["hello world"] + + +class _FinanceFlaggingLanguageClient: + def __init__(self) -> None: + self.moderated_texts: Final[list[str]] = [] # mutable-ok: records what reached Google + + def moderate_text(self, request: SimpleNamespace) -> SimpleNamespace: + self.moderated_texts.append(request.document.content) + return SimpleNamespace(moderation_categories=[SimpleNamespace(name="Finance", confidence=0.885)]) + + +def _google_text_moderation_flagging_finance(): + from enterprise.enterprise_hooks.google_text_moderation import ( + ENTERPRISE_GoogleTextModeration, + ) + + guard: Final = ENTERPRISE_GoogleTextModeration.__new__(ENTERPRISE_GoogleTextModeration) + guard.client = _FinanceFlaggingLanguageClient() + guard.language_document = lambda content, type_: SimpleNamespace(content=content) + guard.moderate_text_request = lambda document: SimpleNamespace(document=document) + guard.document_type = "PLAIN_TEXT" + guard.finance_confidence_threshold = 0.8 + return guard + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("call_type", "data"), + [ + ("aembedding", {"model": "text-embedding-ada-002", "input": "stocks"}), + ("embedding", {"model": "text-embedding-ada-002", "input": ["stocks", "mortgage"]}), + ("allm_passthrough_route", {"model": "gigachat-embeddings", "endpoint": "embeddings", "input": ["stocks"]}), + ], +) +async def test_google_text_moderation_leaves_embedding_input_unmoderated( + user_api_key: UserAPIKeyAuth, call_type: CallTypesLiteral, data: dict[str, object] +): + guard: Final = _google_text_moderation_flagging_finance() + + await guard.async_moderation_hook(data=data, user_api_key_dict=user_api_key, call_type=call_type) + + assert guard.client.moderated_texts == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("call_type", "data"), + [ + ("acompletion", {"model": "gpt-4o", "messages": [{"role": "user", "content": "stocks"}]}), + ("anthropic_messages", {"model": "claude", "messages": [{"role": "user", "content": "stocks"}]}), + ("aresponses", {"model": "gpt-4o", "input": "stocks"}), + ( + "allm_passthrough_route", + {"model": "gigachat", "endpoint": "chat/completions", "messages": [{"role": "user", "content": "stocks"}]}, + ), + ( + "allm_passthrough_route", + { + "model": "gigachat", + "endpoint": "chat/completions#embeddings", + "messages": [{"role": "user", "content": "stocks"}], + }, + ), + ], +) +async def test_google_text_moderation_still_blocks_conversations( + user_api_key: UserAPIKeyAuth, call_type: CallTypesLiteral, data: dict[str, object] +): + guard: Final = _google_text_moderation_flagging_finance() + + with pytest.raises(HTTPException) as exc_info: + await guard.async_moderation_hook(data=data, user_api_key_dict=user_api_key, call_type=call_type) + + assert exc_info.value.status_code == 400 + assert guard.client.moderated_texts == ["stocks"] diff --git a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py index 79193d37b04..4b5957a1237 100644 --- a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py @@ -3,6 +3,7 @@ import json import logging from collections.abc import Mapping from datetime import datetime, timedelta, timezone +from pathlib import Path from types import SimpleNamespace from typing import Final, cast from unittest.mock import AsyncMock, MagicMock, create_autospec, patch @@ -3062,10 +3063,10 @@ async def _no_fallback(line: bytes) -> None: @pytest.mark.asyncio -async def test_async_log_success_event_hands_the_sidecar_a_compact_event_and_skips_the_pipeline(tmp_path): +async def test_async_log_success_event_hands_the_sidecar_a_compact_event_and_skips_the_pipeline(socket_path: Path): handler = _RecordingHandler() consumer = SpendEventConsumer(handler) - address = UnixAddress(path=str(tmp_path / "spend.sock")) + address = UnixAddress(path=str(socket_path)) server = await consumer.serve(address) producer = SpendEventProducer( address=address, on_unavailable="fallback", buffer_size=10, connect_timeout=1.0, fallback=_no_fallback diff --git a/tests/unit/proxy/proxy_server/conftest.py b/tests/unit/proxy/proxy_server/conftest.py index 9baf3206fc4..3f48a3b8558 100644 --- a/tests/unit/proxy/proxy_server/conftest.py +++ b/tests/unit/proxy/proxy_server/conftest.py @@ -11,11 +11,14 @@ import contextlib import os import sys from pathlib import Path -from typing import Any, AsyncIterator, Callable, Dict, Iterator, List, Optional +from typing import TYPE_CHECKING, Any, AsyncIterator, Callable, Dict, Final, Iterator, List, Optional from unittest.mock import AsyncMock, MagicMock import pytest +if TYPE_CHECKING: + from litellm import Router + # Repo root, anchored to this file (not CWD) so the path is correct no # matter where pytest is invoked from. With the project installed via # uv this is defensive — `litellm` already resolves through site-packages @@ -536,3 +539,25 @@ def reset_login_throttle(monkeypatch): _drop_throttle_keys() yield _drop_throttle_keys _drop_throttle_keys() + + +@pytest.fixture +def openai_only_router(monkeypatch: pytest.MonkeyPatch) -> Router: + from litellm import Router + from litellm.proxy import proxy_server + + router: Final = Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-fake"}, + "model_info": {"id": "gpt-4-id"}, + } + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "llm_model_list", router.model_list) + monkeypatch.setattr(proxy_server, "prisma_client", None) + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(proxy_server, "user_model", None) + return router diff --git a/tests/unit/proxy/proxy_server/test_routes_models.py b/tests/unit/proxy/proxy_server/test_routes_models.py index 7be30c61f0c..36f2a8d5acd 100644 --- a/tests/unit/proxy/proxy_server/test_routes_models.py +++ b/tests/unit/proxy/proxy_server/test_routes_models.py @@ -9,15 +9,20 @@ Pins (PR2): from __future__ import annotations +from typing import Final from unittest.mock import MagicMock import pytest import litellm +from litellm import Router +from litellm.integrations.custom_secret_manager import CustomSecretManager from litellm.proxy import proxy_server +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy import utils as proxy_utils from litellm.proxy.utils import create_model_info_response from litellm.types.router import DeploymentModelListingInfo +from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem from .conftest import normalize # type: ignore[import-not-found] @@ -237,6 +242,7 @@ def test_anthropic_display_name_resolved_via_internal_team_key( return_value=[ { "model_name": internal_name, + "litellm_params": {"model": "openai/gpt-4"}, "model_info": { "team_id": "team-1", "team_public_model_name": "gpt-4-team", @@ -320,6 +326,7 @@ def test_anthropic_format_returns_public_team_model_name( return_value=[ { "model_name": internal_name, + "litellm_params": {"model": "openai/gpt-4"}, "model_info": { "team_id": "team-1", "team_public_model_name": "gpt-4-team", @@ -494,7 +501,11 @@ def test_codex_service_tiers_resolved_via_internal_team_key(client, auth_as, pat patched_models.get_model_list = MagicMock( return_value=[ - {"model_name": internal_name, "model_info": {"team_id": "team-1", "team_public_model_name": "gpt-4-team"}} + { + "model_name": internal_name, + "litellm_params": {"model": "openai/gpt-4"}, + "model_info": {"team_id": "team-1", "team_public_model_name": "gpt-4-team"}, + } ] ) patched_models.get_model_names = MagicMock(return_value=[internal_name]) @@ -514,3 +525,206 @@ def test_codex_service_tiers_resolved_via_internal_team_key(client, auth_as, pat assert response.status_code == 200 (entry,) = response.json()["models"] assert (entry["slug"], [tier["id"] for tier in entry["service_tiers"]]) == ("gpt-4-team", ["ultrafast"]) + + +def _priced_jev_models() -> frozenset[str]: + return frozenset(name.removeprefix("typesafe/") for name in litellm.model_cost if name.startswith("typesafe/")) + + +async def _admin_picker_models() -> set[str]: + admin: Final = UserAPIKeyAuth(api_key="sk-test", user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN) + response: Final = await proxy_server.model_list( + user_api_key_dict=admin, return_wildcard_routes=True, include_model_access_groups=True, scope="expand" + ) + return {model["id"] for model in response["data"]} + + +@pytest.mark.asyncio +async def test_scope_expand_offers_typesafe_models_when_typesafe_is_configured( + openai_only_router: Router, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake") + jev_models: Final = _priced_jev_models() + + picker_models: Final = await _admin_picker_models() + + assert jev_models + assert {"gpt-4", "typesafe/*", *jev_models} <= picker_models + + +@pytest.mark.asyncio +async def test_scope_expand_still_lists_models_when_the_typesafe_key_reference_cannot_resolve( + openai_only_router: Router, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("LITELLM_QA_UNSET_TYPESAFE_TOKEN", raising=False) + monkeypatch.setenv("TYPESAFE_API_KEY", "oidc/env/LITELLM_QA_UNSET_TYPESAFE_TOKEN") + + picker_models: Final = await _admin_picker_models() + + assert {"gpt-4", "typesafe/*"} <= picker_models + + +@pytest.mark.asyncio +async def test_scope_expand_offers_no_typesafe_models_without_typesafe_credentials( + openai_only_router: Router, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("TYPESAFE_API_KEY", raising=False) + + picker_models: Final = await _admin_picker_models() + + assert "gpt-4" in picker_models + assert picker_models.isdisjoint({"typesafe/*", *_priced_jev_models()}) + + +@pytest.mark.asyncio +async def test_scope_expand_does_not_read_the_secret_manager_for_typesafe_credentials( + openai_only_router: Router, monkeypatch: pytest.MonkeyPatch +) -> None: + secret_manager: Final = MagicMock(spec=CustomSecretManager) + secret_manager.sync_read_secret.return_value = None + monkeypatch.delenv("TYPESAFE_API_KEY", raising=False) + monkeypatch.setattr(litellm, "secret_manager_client", secret_manager) + monkeypatch.setattr(litellm, "_key_management_system", KeyManagementSystem.CUSTOM) + monkeypatch.setattr(litellm, "_key_management_settings", KeyManagementSettings(access_mode="read_only")) + + picker_models: Final = await _admin_picker_models() + + secrets_read: Final = { + call.kwargs["secret_name"] + for call in (*secret_manager.sync_read_secret.call_args_list, *secret_manager.async_read_secret.call_args_list) + } + assert "gpt-4" in picker_models + assert "TYPESAFE_API_KEY" not in secrets_read + + +@pytest.mark.asyncio +async def test_default_model_list_does_not_offer_typesafe_models( + openai_only_router: Router, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake") + admin: Final = UserAPIKeyAuth(api_key="sk-test", user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN) + + response: Final = await proxy_server.model_list(user_api_key_dict=admin) + + assert {model["id"] for model in response["data"]}.isdisjoint({"typesafe/*", *_priced_jev_models()}) + + +async def _key_dialog_models(user_role: LitellmUserRoles, team_id: str | None) -> set[str]: + caller: Final = UserAPIKeyAuth(api_key="sk-test", user_id="u", user_role=user_role) + response: Final = await proxy_server.model_list( + user_api_key_dict=caller, return_wildcard_routes=True, include_model_access_groups=True, team_id=team_id + ) + return {model["id"] for model in response["data"]} + + +@pytest.mark.asyncio +async def test_key_dialog_without_a_team_offers_typesafe_models_to_an_admin( + openai_only_router: Router, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake") + + dialog_models: Final = await _key_dialog_models(LitellmUserRoles.PROXY_ADMIN, team_id=None) + + assert {"gpt-4", "typesafe/*", *_priced_jev_models()} <= dialog_models + + +@pytest.mark.asyncio +@pytest.mark.parametrize("scope", ["expand", None]) +async def test_pickers_offer_typesafe_names_once_and_no_bare_name_for_registered_typesafe_deployments( + monkeypatch: pytest.MonkeyPatch, scope: str | None +) -> None: + monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake") + jev_models: Final = _priced_jev_models() + monkeypatch.setattr(litellm, "model_cost", {**litellm.model_cost}) + router: Final = Router( + model_list=[ + {"model_name": "typesafe/*", "litellm_params": {"model": "typesafe/*", "api_key": "sk-typesafe-fake"}}, + { + "model_name": "ticket-router", + "litellm_params": {"model": "typesafe/my-classifier", "api_key": "sk-typesafe-fake"}, + }, + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "llm_model_list", router.model_list) + monkeypatch.setattr(proxy_server, "prisma_client", None) + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(proxy_server, "user_model", None) + admin: Final = UserAPIKeyAuth(api_key="sk-test", user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN) + + response: Final = await proxy_server.model_list( + user_api_key_dict=admin, return_wildcard_routes=True, include_model_access_groups=True, scope=scope + ) + + listed: Final = [model["id"] for model in response["data"]] + assert {"ticket-router", *jev_models} <= set(listed) + assert listed.count("typesafe/*") == 1 + assert set(listed).isdisjoint({"*", "my-classifier"}) + + +@pytest.mark.asyncio +async def test_key_dialog_offers_no_typesafe_models_to_an_internal_user( + openai_only_router: Router, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake") + + dialog_models: Final = await _key_dialog_models(LitellmUserRoles.INTERNAL_USER, team_id=None) + + assert "gpt-4" in dialog_models + assert dialog_models.isdisjoint({"typesafe/*", *_priced_jev_models()}) + + +@pytest.mark.asyncio +async def test_key_dialog_for_a_team_offers_no_typesafe_models_the_team_lacks( + openai_only_router: Router, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake") + + dialog_models: Final = await _key_dialog_models(LitellmUserRoles.PROXY_ADMIN, team_id="team-without-typesafe") + + assert dialog_models.isdisjoint({"typesafe/*", *_priced_jev_models()}) + + +@pytest.mark.asyncio +async def test_scope_expand_lists_no_typesafe_models_to_a_client_not_asking_for_allowlist_names( + openai_only_router: Router, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake") + admin: Final = UserAPIKeyAuth(api_key="sk-test", user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN) + + response: Final = await proxy_server.model_list(user_api_key_dict=admin, scope="expand") + + listed: Final = {model["id"] for model in response["data"]} + assert "gpt-4" in listed + assert listed.isdisjoint({"typesafe/*", *_priced_jev_models()}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("scope", "return_wildcard_routes", "only_model_access_groups"), + [ + pytest.param(None, False, True, id="access-group-dropdown"), + pytest.param(None, None, None, id="dashboard-list-without-wildcard-routes"), + pytest.param("expand", None, None, id="expanded-list-without-wildcard-routes"), + pytest.param(None, True, True, id="access-groups-only-with-wildcard-routes"), + ], +) +async def test_dashboard_lists_not_asking_for_allowlist_names_offer_no_typesafe_models( + openai_only_router: Router, + monkeypatch: pytest.MonkeyPatch, + scope: str | None, + return_wildcard_routes: bool | None, + only_model_access_groups: bool | None, +) -> None: + monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake") + admin: Final = UserAPIKeyAuth(api_key="sk-test", user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN) + + response: Final = await proxy_server.model_list( + user_api_key_dict=admin, + include_model_access_groups=True, + return_wildcard_routes=return_wildcard_routes, + only_model_access_groups=only_model_access_groups, + scope=scope, + ) + + assert {model["id"] for model in response["data"]}.isdisjoint({"typesafe/*", *_priced_jev_models()}) diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index d55ce25d878..013d3e59125 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -1141,6 +1141,25 @@ def test_get_supported_endpoints_chat_completions_present(reset_endpoints_cache) assert len(chat["providers"]) > 0 +def test_get_supported_endpoints_lists_streamlake_on_chat_messages_and_responses( + reset_endpoints_cache: None, +) -> None: + app: Final[FastAPI] = FastAPI() + app.include_router(router) + client: Final[TestClient] = TestClient(app) + with client: + response: Final = client.get("/public/endpoints") + assert response.status_code == 200, response.text + + streamlake_endpoints: Final = { + endpoint["key"] + for endpoint in response.json()["endpoints"] + if any(provider["slug"] == "streamlake" for provider in endpoint["providers"]) + } + assert {"chat_completions", "messages", "responses"} <= streamlake_endpoints + assert "embeddings" not in streamlake_endpoints + + def test_get_supported_endpoints_display_names_have_no_slug_suffix( reset_endpoints_cache, ): diff --git a/tests/unit/proxy/spend_tracking/test_spend_event_producer.py b/tests/unit/proxy/spend_tracking/test_spend_event_producer.py index adfb1251d1c..585616677e1 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_event_producer.py +++ b/tests/unit/proxy/spend_tracking/test_spend_event_producer.py @@ -177,9 +177,9 @@ def test_settings_read_the_documented_env(monkeypatch): @pytest.mark.asyncio -async def test_events_reach_the_sidecar_once_and_in_order(tmp_path: Path): +async def test_events_reach_the_sidecar_once_and_in_order(socket_path: Path): fallback: Final = _Fallback() - async with _Sidecar(tmp_path / "spend.sock") as sidecar: + async with _Sidecar(socket_path) as sidecar: producer: Final = _producer(sidecar.path, fallback) outcomes: Final = [await producer.publish(f"event-{i}\n".encode()) for i in range(20)] await producer.close(drain_timeout=5.0) @@ -193,9 +193,9 @@ async def test_events_reach_the_sidecar_once_and_in_order(tmp_path: Path): @pytest.mark.asyncio -async def test_unreachable_sidecar_falls_back_in_process_and_backs_off(tmp_path: Path): +async def test_unreachable_sidecar_falls_back_in_process_and_backs_off(socket_path: Path): fallback: Final = _Fallback() - producer: Final = _producer(tmp_path / "missing.sock", fallback) + producer: Final = _producer(socket_path.with_name("missing.sock"), fallback) first: Final = await producer.publish(b"event-1\n") await asyncio.sleep(0.05) second: Final = await producer.publish(b"event-2\n") @@ -210,11 +210,11 @@ async def test_unreachable_sidecar_falls_back_in_process_and_backs_off(tmp_path: @pytest.mark.parametrize("loop_factory", [asyncio.new_event_loop, uvloop.new_event_loop], ids=["asyncio", "uvloop"]) def test_sidecar_hang_up_falls_back_instead_of_losing_events( - tmp_path: Path, loop_factory: Callable[[], asyncio.AbstractEventLoop] + socket_path: Path, loop_factory: Callable[[], asyncio.AbstractEventLoop] ): async def scenario() -> tuple[list[bytes], list[bytes], tuple[int, int, int]]: fallback: Final = _Fallback() - sidecar: Final = _Sidecar(tmp_path / "spend.sock") + sidecar: Final = _Sidecar(socket_path) async with sidecar: producer: Final = _producer(sidecar.path, fallback) await producer.publish(b"event-1\n") @@ -236,7 +236,7 @@ def test_sidecar_hang_up_falls_back_instead_of_losing_events( @pytest.mark.parametrize("loop_factory", [asyncio.new_event_loop, uvloop.new_event_loop], ids=["asyncio", "uvloop"]) def test_mid_stream_crash_never_bills_an_event_on_both_sides( - tmp_path: Path, loop_factory: Callable[[], asyncio.AbstractEventLoop] + socket_path: Path, loop_factory: Callable[[], asyncio.AbstractEventLoop] ): """Events large enough to straddle the kernel buffer, a sidecar that reads some and then drops the socket: a failed write may only fall back when the sidecar cannot have read the whole line.""" @@ -244,7 +244,7 @@ def test_mid_stream_crash_never_bills_an_event_on_both_sides( async def scenario() -> tuple[list[bytes], list[bytes], tuple[int, int, int]]: fallback: Final = _Fallback() - sidecar: Final = _CrashingSidecar(tmp_path / "spend.sock", lines_before_crash=3) + sidecar: Final = _CrashingSidecar(socket_path, lines_before_crash=3) async with sidecar: producer: Final = _producer(sidecar.path, fallback) for event in events: @@ -266,11 +266,11 @@ def test_mid_stream_crash_never_bills_an_event_on_both_sides( @pytest.mark.asyncio -async def test_drain_timeout_hands_the_in_flight_event_to_fallback(tmp_path: Path): +async def test_drain_timeout_hands_the_in_flight_event_to_fallback(socket_path: Path): """A sidecar that stops reading leaves one event half-written; cancelling the writer must not lose it.""" fallback: Final = _Fallback() stuck: Final = b"x" * (4 * 1024 * 1024) + b"\n" - async with _Sidecar(tmp_path / "spend.sock", reads=False) as sidecar: + async with _Sidecar(socket_path, reads=False) as sidecar: producer: Final = _producer(sidecar.path, fallback) assert await producer.publish(stuck) == "queued" await asyncio.sleep(0.1) @@ -282,10 +282,10 @@ async def test_drain_timeout_hands_the_in_flight_event_to_fallback(tmp_path: Pat @pytest.mark.asyncio -async def test_shutdown_lets_the_writer_finish_a_fallback_already_in_progress(tmp_path: Path): +async def test_shutdown_lets_the_writer_finish_a_fallback_already_in_progress(socket_path: Path): """Cancelling the writer while it runs the pipeline in-process must neither lose nor repeat that event.""" fallback: Final = _GatedFallback() - producer: Final = _producer(tmp_path / "missing.sock", fallback) + producer: Final = _producer(socket_path.with_name("missing.sock"), fallback) assert await producer.publish(b"event-1\n") == "queued" await asyncio.wait_for(fallback.started.wait(), 5.0) closing: Final = asyncio.ensure_future(producer.close(drain_timeout=0.05)) @@ -299,10 +299,10 @@ async def test_shutdown_lets_the_writer_finish_a_fallback_already_in_progress(tm @pytest.mark.asyncio -async def test_shutdown_does_not_replay_an_event_the_kernel_already_took(tmp_path: Path): +async def test_shutdown_does_not_replay_an_event_the_kernel_already_took(socket_path: Path): """Cancelling a drain whose bytes already left the process must not run the event a second time in-process.""" fallback: Final = _Fallback() - async with _Sidecar(tmp_path / "spend.sock") as sidecar: + async with _Sidecar(socket_path) as sidecar: producer: Final = _producer(sidecar.path, fallback, open_connection=_open_with_stalled_drain) assert await producer.publish(b"event-1\n") == "queued" await asyncio.sleep(0.1) @@ -316,9 +316,9 @@ async def test_shutdown_does_not_replay_an_event_the_kernel_already_took(tmp_pat @pytest.mark.asyncio -async def test_drop_policy_counts_instead_of_running_in_process(tmp_path: Path): +async def test_drop_policy_counts_instead_of_running_in_process(socket_path: Path): fallback: Final = _Fallback() - producer: Final = _producer(tmp_path / "missing.sock", fallback, on_unavailable="drop") + producer: Final = _producer(socket_path.with_name("missing.sock"), fallback, on_unavailable="drop") await producer.publish(b"event-1\n") await producer.close(drain_timeout=5.0) assert await producer.publish(b"event-2\n") == "dropped" @@ -328,9 +328,9 @@ async def test_drop_policy_counts_instead_of_running_in_process(tmp_path: Path): @pytest.mark.asyncio -async def test_full_buffer_applies_the_unavailable_policy_immediately(tmp_path: Path): +async def test_full_buffer_applies_the_unavailable_policy_immediately(socket_path: Path): fallback: Final = _Fallback() - async with _Sidecar(tmp_path / "spend.sock") as sidecar: + async with _Sidecar(socket_path) as sidecar: producer: Final = _producer(sidecar.path, fallback, buffer_size=2) outcomes: Final = [await producer.publish(f"event-{i}\n".encode()) for i in range(3)] await producer.close(drain_timeout=5.0) @@ -342,9 +342,9 @@ async def test_full_buffer_applies_the_unavailable_policy_immediately(tmp_path: @pytest.mark.asyncio -async def test_close_flushes_buffered_events_then_refuses_new_ones(tmp_path: Path): +async def test_close_flushes_buffered_events_then_refuses_new_ones(socket_path: Path): fallback: Final = _Fallback() - async with _Sidecar(tmp_path / "spend.sock") as sidecar: + async with _Sidecar(socket_path) as sidecar: producer: Final = _producer(sidecar.path, fallback) for i in range(50): await producer.publish(f"event-{i}\n".encode()) diff --git a/tests/unit/proxy/test_collector.py b/tests/unit/proxy/test_collector.py index ac3e1f5e566..4dbaccc3cbf 100644 --- a/tests/unit/proxy/test_collector.py +++ b/tests/unit/proxy/test_collector.py @@ -49,14 +49,14 @@ class _Fallback: @pytest.mark.asyncio @pytest.mark.parametrize("transport", ["unix", "tcp"]) -async def test_consumer_handles_each_producer_line_once_in_order(tmp_path: Path, transport: str): +async def test_consumer_handles_each_producer_line_once_in_order(socket_path: Path, transport: str): handler: Final = _Handler(fail_on=b"event-3\n") consumer: Final = SpendEventConsumer(handler) server: Final = await consumer.serve( - UnixAddress(path=str(tmp_path / "spend.sock")) if transport == "unix" else TcpAddress("127.0.0.1", 0) + UnixAddress(path=str(socket_path)) if transport == "unix" else TcpAddress("127.0.0.1", 0) ) address: Final = ( - UnixAddress(path=str(tmp_path / "spend.sock")) + UnixAddress(path=str(socket_path)) if transport == "unix" else TcpAddress("127.0.0.1", server.sockets[0].getsockname()[1]) ) @@ -74,10 +74,10 @@ async def test_consumer_handles_each_producer_line_once_in_order(tmp_path: Path, @pytest.mark.asyncio -async def test_consumer_discards_a_truncated_trailing_event(tmp_path: Path): +async def test_consumer_discards_a_truncated_trailing_event(socket_path: Path): handler: Final = _Handler() consumer: Final = SpendEventConsumer(handler) - address: Final = UnixAddress(path=str(tmp_path / "spend.sock")) + address: Final = UnixAddress(path=str(socket_path)) server: Final = await consumer.serve(address) _, writer = await open_collector_connection(address, timeout=1.0) writer.write(b"whole\npartial-without-newline") @@ -93,9 +93,9 @@ async def test_consumer_discards_a_truncated_trailing_event(tmp_path: Path): @pytest.mark.asyncio -async def test_drain_reports_producers_still_connected_after_the_timeout(tmp_path: Path): +async def test_drain_reports_producers_still_connected_after_the_timeout(socket_path: Path): consumer: Final = SpendEventConsumer(_Handler()) - address: Final = UnixAddress(path=str(tmp_path / "spend.sock")) + address: Final = UnixAddress(path=str(socket_path)) server: Final = await consumer.serve(address) _, writer = await open_collector_connection(address, timeout=1.0) await asyncio.sleep(0.05) @@ -108,11 +108,11 @@ async def test_drain_reports_producers_still_connected_after_the_timeout(tmp_pat @pytest.mark.asyncio -async def test_graceful_stop_hands_the_producer_over_to_its_fallback_without_losing_events(tmp_path: Path): +async def test_graceful_stop_hands_the_producer_over_to_its_fallback_without_losing_events(socket_path: Path): handler: Final = _Handler() fallback: Final = _Fallback() consumer: Final = SpendEventConsumer(handler) - address: Final = UnixAddress(path=str(tmp_path / "spend.sock")) + address: Final = UnixAddress(path=str(socket_path)) server: Final = await consumer.serve(address) producer: Final = SpendEventProducer( address=address, on_unavailable="fallback", buffer_size=100, connect_timeout=1.0, fallback=fallback @@ -134,7 +134,7 @@ async def test_graceful_stop_hands_the_producer_over_to_its_fallback_without_los @pytest.mark.parametrize("loop_factory", [asyncio.new_event_loop, uvloop.new_event_loop], ids=["asyncio", "uvloop"]) def test_drain_still_hands_over_live_producers_when_another_connection_already_died( - tmp_path: Path, loop_factory: Callable[[], asyncio.AbstractEventLoop] + socket_path: Path, loop_factory: Callable[[], asyncio.AbstractEventLoop] ): """A transport the loop force-closed under a busy handler must not abort the half-close of the others.""" @@ -145,7 +145,7 @@ def test_drain_still_hands_over_live_producers_when_another_connection_already_d await release.wait() consumer: Final = SpendEventConsumer(slow_handler) - address: Final = UnixAddress(path=str(tmp_path / "spend.sock")) + address: Final = UnixAddress(path=str(socket_path)) server: Final = await consumer.serve(address) _, dead = await open_collector_connection(address, timeout=1.0) dead.write(b"stuck\n") diff --git a/tests/unit/proxy/test_proxy_server.py b/tests/unit/proxy/test_proxy_server.py index a26a64c6fb0..976ed4f28a5 100644 --- a/tests/unit/proxy/test_proxy_server.py +++ b/tests/unit/proxy/test_proxy_server.py @@ -22,7 +22,9 @@ import json import asyncio import logging +import httpx import pytest +import respx import litellm from litellm import RateLimitError, Timeout, completion, completion_cost, embedding @@ -3857,3 +3859,104 @@ async def test_model_list_return_wildcard_routes_defaults_to_general_setting( assert "gpt-6.1-sol" in listed, listed assert ("openai/*" in listed) is wildcard_listed, listed + + +_DEBUG_LOG_SENTINEL: Final = "sentinel-request-body-7f3a" +_LONG_PROMPT: Final = f"{_DEBUG_LOG_SENTINEL} " + "Keep edits minimal and explain them briefly. " * 20 +_TOOL_PARAMETERS: Final = {"type": "object", "properties": {"cmd": {"type": "string"}}} + + +class _MessageCollector(logging.Handler): + def __init__(self) -> None: + super().__init__(level=logging.DEBUG) + self.messages: list[str] = [] # mutable-ok: a logging.Handler collects records as they are emitted + + def emit(self, record: logging.LogRecord) -> None: + self.messages.append(record.getMessage()) + + +@pytest.mark.parametrize( + "route, path_suffix, body, upstream_json", + [ + ( + "/v1/chat/completions", + "/chat/completions", + { + "model": "test_openai_models", + "messages": [{"role": "user", "content": _LONG_PROMPT}], + "tools": [ + { + "type": "function", + "function": {"name": "shell", "description": _LONG_PROMPT, "parameters": _TOOL_PARAMETERS}, + } + ], + }, + { + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 1, + "model": "gpt-3.5-turbo", + "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "hi"}}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ), + ( + "/v1/responses", + "/responses", + { + "model": "test_openai_models", + "instructions": _LONG_PROMPT, + "input": "hi", + "tools": [ + {"type": "function", "name": "shell", "description": _LONG_PROMPT, "parameters": _TOOL_PARAMETERS} + ], + }, + { + "id": "resp_1", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-3.5-turbo", + "output": [ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "hi", "annotations": []}], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + }, + ), + ], +) +def test_detailed_debug_logs_request_body_only_as_received_and_as_sent( + client_no_auth: TestClient, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + route: str, + path_suffix: str, + body: Mapping[str, object], + upstream_json: Mapping[str, object], +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post(url__regex=rf".*{path_suffix}$").mock(return_value=httpx.Response(200, json=upstream_json)) + collector: Final = _MessageCollector() + loggers: Final = tuple(logging.getLogger(name) for name in ("LiteLLM", "LiteLLM Router", "LiteLLM Proxy")) + original_levels: Final = tuple(lg.level for lg in loggers) + for lg in loggers: + lg.addHandler(collector) + lg.setLevel(logging.DEBUG) + try: + response: Final = client_no_auth.post(route, json=body) + finally: + for lg, level in zip(loggers, original_levels): + lg.removeHandler(collector) + lg.setLevel(level) + + assert response.status_code == 200, response.text + body_dumps: Final = [message for message in collector.messages if _DEBUG_LOG_SENTINEL in message] + assert len(body_dumps) == 2, [message[:120] for message in body_dumps] + assert body_dumps[0].startswith("Request received by LiteLLM:") + assert "POST Request Sent from LiteLLM" in body_dumps[1] diff --git a/tests/unit/responses/litellm_completion_transformation/test_function_call_output_normalization.py b/tests/unit/responses/litellm_completion_transformation/test_function_call_output_normalization.py index 326583d94f5..65800e42124 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_function_call_output_normalization.py +++ b/tests/unit/responses/litellm_completion_transformation/test_function_call_output_normalization.py @@ -111,6 +111,8 @@ def test_media_normalization_preserves_formats_references_and_metadata(kind: str "keep", ), ([{"type": "input_image", "file_id": "file-only"}], '[{"type": "input_image", "file_id": "file-only"}]'), + ({"ids": {7}}, "{'ids': {7}}"), + ([{"type": "unknown", "ids": {7}}], "[{'type': 'unknown', 'ids': {7}}]"), ], ) def test_tool_output_without_usable_media_keeps_its_existing_fallback(output: object, expected: str) -> None: diff --git a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index 0b971bc1696..d9a0033b179 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -10,6 +10,7 @@ before response.completed, and that every event of a bridged stream carries the spend tracking stores, so a follow-up previous_response_id still finds the conversation. """ +import itertools import json from collections.abc import Callable from typing import Final @@ -23,6 +24,7 @@ from litellm.responses.litellm_completion_transformation.streaming_iterator impo from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ( BaseLiteLLMOpenAIResponseObject, + OutputTextDeltaEvent, ResponsesAPIStreamEvents, ) from litellm.types.responses.main import build_web_search_call @@ -1658,3 +1660,34 @@ async def test_a_compaction_block_without_its_summary_streams_no_compaction_item assert [event["item"]["type"] for event in events if "item" in event] == ["message", "message"] assert [item["type"] for item in events[-1]["response"]["output"]] == ["message"] + + +def test_the_partial_response_billed_on_disconnect_carries_the_logging_objects_bill_as_hidden_response_cost() -> None: + from litellm.litellm_core_utils.hidden_params import get_hidden_params + + wrapper: Final = _FakeStreamWrapper([_chunk("Hello"), _chunk(" world", finish_reason="stop")]) + wrapper.logging_obj.response_cost_calculator.return_value = 0.00032 + iterator: Final = LiteLLMCompletionStreamingIterator( + model="claude-haiku-4-5", + litellm_custom_stream_wrapper=wrapper, # pyright: ignore[reportArgumentType] # the fake stands in for CustomStreamWrapper, as in _build_iterator + request_input="What is the weather in San Francisco?", + responses_api_request={}, + custom_llm_provider="anthropic", + litellm_metadata={}, + ) + relayed: Final = tuple( + itertools.takewhile(lambda event: not isinstance(event, OutputTextDeltaEvent), iterator) + ) + assert relayed, "the stream must relay at least one chunk before the client goes away" + + partial: Final = iterator.partial_response_on_disconnect() + + assert partial is not None + hidden: Final = get_hidden_params(partial.response) + assert hidden is not None and hidden["response_cost"] == 0.00032 + priced: Final = wrapper.logging_obj.response_cost_calculator.call_args.kwargs["result"] + assert priced.id == partial.response.id + assert (priced.usage.prompt_tokens, priced.usage.completion_tokens) == ( + partial.response.usage.input_tokens, + partial.response.usage.output_tokens, + ), "the bill is priced on the partial's own usage" diff --git a/tests/unit/responses/test_streaming_iterator.py b/tests/unit/responses/test_streaming_iterator.py index 664f4dba163..b9454ae6c1a 100644 --- a/tests/unit/responses/test_streaming_iterator.py +++ b/tests/unit/responses/test_streaming_iterator.py @@ -24,7 +24,7 @@ from litellm.exceptions import MidStreamFallbackError from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig -from litellm.responses.streaming_iterator import( +from litellm.responses.streaming_iterator import ( CachedResponsesAPIStreamingIterator, MockResponsesAPIStreamingIterator, ResponsesAPIStreamingIterator, @@ -845,25 +845,33 @@ def test_stamp_responses_usage_cost_stamps_computed_cost(): response = _responses_api_response_with_usage() logging_obj = Mock(spec=LiteLLMLoggingObj) + logging_obj.model_call_details = {"custom_llm_provider": "openai"} logging_obj.response_cost_calculator.return_value = 0.000704 stamp_responses_usage_cost(response, logging_obj) assert getattr(response.usage, "cost", None) == pytest.approx(0.000704) - logging_obj.response_cost_calculator.assert_called_once_with(result=response) + priced = logging_obj.response_cost_calculator.call_args.kwargs["result"] + assert priced.usage.prompt_tokens == response.usage.input_tokens, ( + "priced in the chat usage shape the spend row uses" + ) -def test_stamp_responses_usage_cost_keeps_provider_reported_cost(): +def test_stamp_responses_usage_cost_bills_the_spend_row_price_over_the_provider_reported_cost(): + from litellm.litellm_core_utils.core_helpers import response_cost_in_hidden_params from litellm.responses.streaming_iterator import stamp_responses_usage_cost response = _responses_api_response_with_usage() setattr(response.usage, "cost", 0.5) logging_obj = Mock(spec=LiteLLMLoggingObj) + logging_obj.model_call_details = {"custom_llm_provider": "openai"} + logging_obj.response_cost_calculator.return_value = 0.00032 stamp_responses_usage_cost(response, logging_obj) - assert getattr(response.usage, "cost", None) == pytest.approx(0.5) - logging_obj.response_cost_calculator.assert_not_called() + assert getattr(response.usage, "cost", None) == pytest.approx(0.00032), "the client sees what the row bills" + priced = logging_obj.response_cost_calculator.call_args.kwargs["result"] + assert response_cost_in_hidden_params(priced) == pytest.approx(0.5), "the reported cost reaches the calculator" def _unvalidated_response_with_dict_usage(usage: dict) -> ResponsesAPIResponse: @@ -881,6 +889,7 @@ def _unvalidated_response_with_dict_usage(usage: dict) -> ResponsesAPIResponse: def test_stamp_responses_usage_cost_keeps_provider_cost_from_dict_usage(): from litellm.responses.streaming_iterator import stamp_responses_usage_cost + response = _unvalidated_response_with_dict_usage( { "input_tokens": 29, @@ -891,26 +900,29 @@ def test_stamp_responses_usage_cost_keeps_provider_cost_from_dict_usage(): } ) logging_obj = Mock(spec=LiteLLMLoggingObj) + logging_obj.model_call_details = {"custom_llm_provider": "openai"} + logging_obj.response_cost_calculator.return_value = 3e-05 stamp_responses_usage_cost(response, logging_obj) assert isinstance(response.usage, ResponseAPIUsage) assert response.usage.cost == pytest.approx(3e-05) assert response.usage.output_tokens_details.reasoning_tokens == 117 - logging_obj.response_cost_calculator.assert_not_called() def test_stamp_responses_usage_cost_computes_cost_for_dict_usage_without_cost(): from litellm.responses.streaming_iterator import stamp_responses_usage_cost + response = _unvalidated_response_with_dict_usage({"input_tokens": 29, "output_tokens": 120, "total_tokens": 149}) logging_obj = Mock(spec=LiteLLMLoggingObj) + logging_obj.model_call_details = {"custom_llm_provider": "openai"} logging_obj.response_cost_calculator.return_value = 0.000704 stamp_responses_usage_cost(response, logging_obj) assert isinstance(response.usage, ResponseAPIUsage) assert response.usage.cost == pytest.approx(0.000704) - logging_obj.response_cost_calculator.assert_called_once_with(result=response) + logging_obj.response_cost_calculator.assert_called_once() def test_stamp_responses_usage_cost_survives_calculator_failure(): @@ -918,6 +930,7 @@ def test_stamp_responses_usage_cost_survives_calculator_failure(): response = _responses_api_response_with_usage() logging_obj = Mock(spec=LiteLLMLoggingObj) + logging_obj.model_call_details = {"custom_llm_provider": "openai"} logging_obj.response_cost_calculator.side_effect = RuntimeError("cost map unavailable") stamp_responses_usage_cost(response, logging_obj) @@ -1207,9 +1220,7 @@ async def test_completed_event_survives_a_failing_usage_estimate(): falls back to usage None.""" malformed_input: Final = [{"type": "message", "role": "user", "content": 42}] with pytest.raises(ValueError, match="Invalid content type"): - _estimate_usage_from_text( - "gpt-4o-mini", malformed_input, {"input": malformed_input}, "hello world" - ) + _estimate_usage_from_text("gpt-4o-mini", malformed_input, {"input": malformed_input}, "hello world") response = _responses_api_response_without_usage() iterator = _make_iterator( @@ -1305,7 +1316,14 @@ async def test_completed_event_with_a_dict_response_is_typed_and_billed(): assert usage.input_tokens > 0 assert usage.output_tokens > 0 assert usage.cost == pytest.approx(0.000704) - logging_obj.response_cost_calculator.assert_any_call(result=completed_response) + priced: Final = [ + call.kwargs["result"] + for call in logging_obj.response_cost_calculator.call_args_list + if isinstance(call.kwargs.get("result"), ResponsesAPIResponse) + ] + assert [(r.id, r.usage.prompt_tokens) for r in priced] == [(completed_response.id, usage.input_tokens)], ( + "priced once, in the chat usage shape the spend row uses" + ) def test_billed_terminal_response_keeps_a_response_that_already_has_usage(): @@ -1375,6 +1393,7 @@ def _vcr_outcome_gate(request, vcr): yield record_vcr_outcome(request, vcr) + @pytest.fixture(scope="function") def setup_and_teardown(): """ @@ -1392,6 +1411,7 @@ def setup_and_teardown(): loop.close() asyncio.set_event_loop(None) + class _FakeLoggingObj: def __init__(self): self.success_calls = 0 @@ -1432,10 +1452,12 @@ class _FakeLoggingObj: self.completion_start_time = completion_start_time self.model_call_details["completion_start_time"] = completion_start_time + async def _await_tasks_spawned_since(existing: set["asyncio.Task[object]"]) -> None: spawned: Final = asyncio.all_tasks() - existing await asyncio.wait_for(asyncio.gather(*spawned, return_exceptions=True), timeout=5) + def _make_completed_response(response_id: str = "resp_test") -> ResponseCompletedEvent: return ResponseCompletedEvent( type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, @@ -1463,6 +1485,7 @@ def _make_completed_response(response_id: str = "resp_test") -> ResponseComplete ), ) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_log_background_task_failure_logs_task_exceptions(monkeypatch): @@ -1485,6 +1508,7 @@ async def test_log_background_task_failure_logs_task_exceptions(monkeypatch): task.exception(), ) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_log_background_task_failure_ignores_cancelled_tasks(monkeypatch): @@ -1500,6 +1524,7 @@ async def test_log_background_task_failure_ignores_cancelled_tasks(monkeypatch): error_logger.assert_not_called() + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_content_part_done_event_supports_refusal_and_reasoning_text(): refusal_event = streaming_module._build_content_part_done_event( @@ -1527,6 +1552,7 @@ def test_content_part_done_event_supports_refusal_and_reasoning_text(): assert reasoning_event.part.reasoning == "because" assert unsupported_event is None + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_dump_response_object_handles_model_and_unknown_values(): response = ResponsesAPIResponse( @@ -1542,6 +1568,7 @@ def test_dump_response_object_handles_model_and_unknown_values(): assert streaming_module._dump_response_object({"type": "message"}) == {"type": "message"} assert streaming_module._dump_response_object(object()) == {} + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_responses_streaming_triggers_hooks(monkeypatch): @@ -1598,6 +1625,7 @@ async def test_responses_streaming_triggers_hooks(monkeypatch): assert seen["request_data"].get("litellm_params") is not None assert seen["call_type"] == CallTypes.responses + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_responses_streaming_calls_post_streaming_deployment_hook(monkeypatch): @@ -1638,6 +1666,7 @@ async def test_responses_streaming_calls_post_streaming_deployment_hook(monkeypa # reset callbacks litellm.callbacks = original_callbacks + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_responses_streaming_failure_triggers_failure_handlers(): @@ -1667,6 +1696,7 @@ async def test_responses_streaming_failure_triggers_failure_handlers(): assert logging_obj.failure_calls >= 1 assert logging_obj.async_failure_calls >= 1 + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_process_chunk_requires_provider_config(): iterator = ResponsesAPIStreamingIterator( @@ -1681,6 +1711,7 @@ def test_process_chunk_requires_provider_config(): with pytest.raises(ValueError, match="responses_api_provider_config is required"): iterator._process_chunk(json.dumps({"type": "response.completed"})) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_process_chunk_wraps_encrypted_content_with_model_id(): openai_types = streaming_module._get_openai_response_types() @@ -1715,6 +1746,7 @@ def test_process_chunk_wraps_encrypted_content_with_model_id(): assert event.item.encrypted_content.startswith("litellm_enc:") assert event.item.encrypted_content.endswith(";ciphertext") + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_process_chunk_completed_response_updates_id_and_usage_cost(monkeypatch): original_include_cost = litellm.include_cost_in_streaming_usage @@ -1768,6 +1800,7 @@ def test_process_chunk_completed_response_updates_id_and_usage_cost(monkeypatch) assert event.response.usage.cost == 1.23 completion_handler.assert_called_once() + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_process_chunk_failed_response_triggers_failure_logging(monkeypatch): openai_types = streaming_module._get_openai_response_types() @@ -1803,6 +1836,7 @@ def test_process_chunk_failed_response_triggers_failure_logging(monkeypatch): assert iterator.completed_response is event failure_handler.assert_called_once() + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_handle_logging_failed_response_uses_response_error_message(): @@ -1835,6 +1869,7 @@ async def test_handle_logging_failed_response_uses_response_error_message(): assert logging_obj.failure_calls == 1 assert logging_obj.async_failure_calls == 1 + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_process_chunk_returns_none_for_invalid_json_and_non_dict_payload(): class _NoopConfig: @@ -1853,6 +1888,7 @@ def test_process_chunk_returns_none_for_invalid_json_and_non_dict_payload(): assert iterator._process_chunk("not-json") is None assert iterator._process_chunk(json.dumps(["not", "a", "dict"])) is None + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_process_chunk_cost_annotation_failure_is_nonfatal(monkeypatch): original_include_cost = litellm.include_cost_in_streaming_usage @@ -1900,6 +1936,7 @@ def test_process_chunk_cost_annotation_failure_is_nonfatal(monkeypatch): assert event.response.usage.cost is None completion_handler.assert_called_once() + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_get_completed_response_object_accepts_direct_response(): logging_obj = _FakeLoggingObj() @@ -1916,6 +1953,7 @@ def test_get_completed_response_object_accepts_direct_response(): assert iterator._get_completed_response_object() is direct_response + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_responses_streaming_completed_event_persists_async_cache(): @@ -1964,6 +2002,7 @@ async def test_responses_streaming_completed_event_persists_async_cache(): assert json.loads(litellm.cache.async_add_cache.call_args.args[0])["id"] == iterator.completed_response.response.id litellm.cache = original_cache + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_responses_streaming_completed_event_persists_sync_cache(): logging_obj = _FakeLoggingObj() @@ -2009,6 +2048,7 @@ def test_responses_streaming_completed_event_persists_sync_cache(): assert json.loads(litellm.cache.add_cache.call_args.args[0])["id"] == iterator.completed_response.response.id litellm.cache = original_cache + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_log_completed_response_sync_direct_path(monkeypatch): hook_calls = {"post_call": 0, "metadata": 0} @@ -2050,6 +2090,7 @@ def test_log_completed_response_sync_direct_path(monkeypatch): assert hook_calls["post_call"] == 1 assert hook_calls["metadata"] == 1 + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_log_completed_response_falls_back_when_model_validate_fails(monkeypatch): class _BadSerializableResponse: @@ -2079,6 +2120,7 @@ def test_log_completed_response_falls_back_when_model_validate_fails(monkeypatch assert logging_obj.success_calls == 1 assert logging_obj.async_success_calls == 1 + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.parametrize( "scenario", @@ -2143,6 +2185,7 @@ def test_persist_completed_response_to_cache_guard_branches(monkeypatch, scenari expected_cached_flag = scenario == "already_cached" assert iterator._completed_response_cached is expected_cached_flag + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_build_synthetic_response_events_covers_annotations_function_calls_and_refusals(): original_include_cost = litellm.include_cost_in_streaming_usage @@ -2201,6 +2244,7 @@ def test_build_synthetic_response_events_covers_annotations_function_calls_and_r assert "response.function_call_arguments.done" in event_types assert event_types[-1] == "response.completed" + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_mock_responses_streaming_iterator_async_iteration_logs_completion( @@ -2251,6 +2295,7 @@ async def test_mock_responses_streaming_iterator_async_iteration_logs_completion assert hook_calls["post_call"] == 1 assert hook_calls["metadata"] == 1 + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_mock_responses_streaming_iterator_sync_iteration_logs_completion(monkeypatch): hook_calls = {"post_call": 0, "metadata": 0} @@ -2296,6 +2341,7 @@ def test_mock_responses_streaming_iterator_sync_iteration_logs_completion(monkey assert hook_calls["post_call"] == 1 assert hook_calls["metadata"] == 1 + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_cached_responses_stream_async_hit_triggers_success_callbacks( @@ -2355,6 +2401,7 @@ async def test_cached_responses_stream_async_hit_triggers_success_callbacks( litellm.cache.add_cache.assert_not_called() litellm.cache = original_cache + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_cached_responses_stream_sync_hit_triggers_success_callbacks(monkeypatch): hook_calls = {"post_call": 0, "metadata": 0} @@ -2453,7 +2500,9 @@ def test_faked_stream_bridged_to_chat_sends_tool_arguments_once(): tool_call_deltas: Final = tuple( tool_call for chunk in chunks - for tool_call in (chunk.choices[0].delta.tool_calls or ()) # comprehension-ok: flattening each chunk's tool calls + for tool_call in ( + chunk.choices[0].delta.tool_calls or () + ) # comprehension-ok: flattening each chunk's tool calls ) joined_arguments: Final = "".join(tool_call.function.arguments or "" for tool_call in tool_call_deltas) joined_text: Final = "".join(chunk.choices[0].delta.content or "" for chunk in chunks) @@ -2509,7 +2558,6 @@ def test_faked_stream_opens_items_in_progress_and_empty_like_a_real_stream(): assert [part.text for part in added_parts] == [""] - def _answer_with_reasoning_refusal_and_custom_tool_call() -> ResponsesAPIResponse: return ResponsesAPIResponse( id="resp_synthetic", @@ -2643,7 +2691,9 @@ async def test_shutdown_drain_waits_for_a_pending_logging_task(): @pytest.mark.asyncio async def test_shutdown_drain_gives_up_on_a_stuck_logging_task_at_its_timeout(): - stuck: Final = streaming_module.spawn_logging_task(asyncio.get_running_loop(), asyncio.Event().wait(), task_name="stuck") + stuck: Final = streaming_module.spawn_logging_task( + asyncio.get_running_loop(), asyncio.Event().wait(), task_name="stuck" + ) await asyncio.wait_for(streaming_module.drain_logging_tasks(timeout=0.05), timeout=1) diff --git a/tests/unit/rust_bridge/native_route_wheel_test.py b/tests/unit/rust_bridge/native_route_wheel_test.py index bc3269f5dd9..ed3fe4f6de5 100644 --- a/tests/unit/rust_bridge/native_route_wheel_test.py +++ b/tests/unit/rust_bridge/native_route_wheel_test.py @@ -87,7 +87,7 @@ def assert_native_request( def native_response(status: int) -> bytes: if status == 429: return b'{"error":"native-rate-limit"}' - return b'{"output":{"message":{"content":[{"text":"native-transcription"}]}}}' + return b'{"output":{"message":{"content":[{"text":"native-transcription"}]}},"usage":{"inputTokens":390,"outputTokens":60}}' def load_native(native_path: Path) -> object: @@ -130,6 +130,9 @@ def assert_success(route: str, response: object) -> None: raise TypeError(f"{route} returned {type(response).__name__}, expected dict") if response["text"] != "native-transcription": raise AssertionError(f"{route} returned {response['text']!r}, expected 'native-transcription'") + expected_usage: Final = {"type": "tokens", "input_tokens": 390, "output_tokens": 60, "total_tokens": 450} + if response.get("usage") != expected_usage: + raise AssertionError(f"{route} returned usage {response.get('usage')!r}, expected {expected_usage!r}") def assert_rate_limit(route: str, error: BaseException) -> None: diff --git a/tests/unit/test_audio_transcription_rust_bridge.py b/tests/unit/test_audio_transcription_rust_bridge.py index 715423c59d9..cd94fb32af2 100644 --- a/tests/unit/test_audio_transcription_rust_bridge.py +++ b/tests/unit/test_audio_transcription_rust_bridge.py @@ -11,6 +11,7 @@ from litellm.rust_bridge import configuration from litellm.rust_bridge._native import RustBridgeDeclined, RustUpstreamError from litellm.rust_bridge.public_call import NativeCall from litellm.rust_bridge.transcription.native import NATIVE_ATRANSCRIPTION, NATIVE_TRANSCRIPTION +from litellm.types.utils import TranscriptionUsageTokensObject MODEL: Final = "bedrock/mistral.voxtral-mini-3b-2507" AUDIO_FILE: Final = ("audio.wav", b"audio", "audio/wav") @@ -59,6 +60,14 @@ class AsyncBridge: return {"text": "async rust"} +class UsageBridge: + def __call__(self, call: NativeCall) -> dict[str, object]: + return { + "text": "rust", + "usage": {"type": "tokens", "input_tokens": 390, "output_tokens": 60, "total_tokens": 450}, + } + + def dispatch_sync() -> litellm.TranscriptionResponse: return BedrockAudioTranscriptionRustDispatch().audio_transcriptions( model=MODEL, @@ -196,3 +205,99 @@ async def test_async_missing_native_binding_raises_without_python_fallback() -> optional_params={}, timeout=None, ) + + +@pytest.mark.parametrize( + ("region_kwargs", "region_env", "priced_model"), + [ + ({}, {}, MODEL), + ({"aws_region_name": "ap-southeast-2"}, {}, "bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507"), + ({}, {"AWS_REGION_NAME": "eu-west-2"}, "bedrock/eu-west-2/mistral.voxtral-mini-3b-2507"), + ({}, {"AWS_REGION": "ap-northeast-1"}, "bedrock/ap-northeast-1/mistral.voxtral-mini-3b-2507"), + ( + {"aws_region_name": "ap-southeast-2"}, + {"AWS_REGION_NAME": "eu-west-2"}, + "bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507", + ), + ], +) +def test_bedrock_transcription_bills_the_converse_token_usage( + region_kwargs: dict[str, str], + region_env: dict[str, str], + priced_model: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + for name in ("AWS_REGION_NAME", "AWS_REGION"): + monkeypatch.delenv(name, raising=False) + for name, value in region_env.items(): + monkeypatch.setenv(name, value) + NATIVE_TRANSCRIPTION.override(UsageBridge()) + + response: Final = litellm.transcription(model=MODEL, file=AUDIO_FILE, **region_kwargs) + + model_info: Final = litellm.get_model_info(priced_model) + expected_cost: Final = 390 * (model_info.get("input_cost_per_token") or 0.0) + 60 * ( + model_info.get("output_cost_per_token") or 0.0 + ) + assert response.usage == TranscriptionUsageTokensObject( + type="tokens", input_tokens=390, output_tokens=60, total_tokens=450 + ) + assert expected_cost > 0 + assert litellm.completion_cost( + completion_response=response, model=MODEL, call_type="transcription" + ) == pytest.approx(expected_cost) + + +def test_bedrock_transcription_forwards_language_and_text_formats_to_rust() -> None: + bridge: Final = SyncBridge() + NATIVE_TRANSCRIPTION.override(bridge) + + litellm.transcription(model=MODEL, file=AUDIO_FILE, language="fr", response_format="text", prompt="Names") + + forwarded: Final = bridge.calls[0].resolved["optional_params"] + assert {key: forwarded.get(key) for key in ("language", "response_format", "prompt")} == { + "language": "fr", + "response_format": "text", + "prompt": "Names", + } + + +@pytest.mark.parametrize(("sent", "expected"), [("0.2", 0.2), ("0", 0.0), (0.5, 0.5), (1, 1.0)]) +def test_bedrock_transcription_forwards_form_temperature_as_a_number(sent: object, expected: float) -> None: + bridge: Final = SyncBridge() + NATIVE_TRANSCRIPTION.override(bridge) + + litellm.transcription(model=MODEL, file=AUDIO_FILE, temperature=sent) + + forwarded: Final = bridge.calls[0].resolved["optional_params"]["temperature"] + assert (type(forwarded), forwarded) == (float, expected) + + +@pytest.mark.parametrize( + "unsupported", + [ + {"response_format": "srt"}, + {"response_format": "verbose_json"}, + {"timestamp_granularities": ["word"]}, + {"temperature": "warm"}, + {"temperature": "inf"}, + {"temperature": True}, + ], +) +def test_bedrock_transcription_rejects_params_it_cannot_honor(unsupported: dict[str, object]) -> None: + bridge: Final = SyncBridge() + NATIVE_TRANSCRIPTION.override(bridge) + + with pytest.raises(litellm.BadRequestError) as raised: + litellm.transcription(model=MODEL, file=AUDIO_FILE, language="fr", **unsupported) + + assert raised.value.status_code == 400 + assert next(iter(unsupported)) in str(raised.value) + assert bridge.calls == () + + dropped: Final = litellm.transcription(model=MODEL, file=AUDIO_FILE, language="fr", drop_params=True, **unsupported) + + assert dropped.text == "rust" + forwarded: Final = bridge.calls[0].resolved["optional_params"] + assert forwarded["language"] == "fr" + assert forwarded.keys().isdisjoint(unsupported) diff --git a/tests/unit/test_router/test_enforce_model_rate_limits.py b/tests/unit/test_router/test_enforce_model_rate_limits.py index 2a260fdf0b8..ff3ffc0db0c 100644 --- a/tests/unit/test_router/test_enforce_model_rate_limits.py +++ b/tests/unit/test_router/test_enforce_model_rate_limits.py @@ -7,9 +7,11 @@ regardless of the routing strategy being used. import asyncio from datetime import datetime, timedelta, timezone +from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest +from pydantic import TypeAdapter import litellm from litellm import Router @@ -724,3 +726,159 @@ class TestDailyLimits: router.discard() assert served == ["a", "b", "b", "b"] + + +_LOGGED_TOKENS: Final = TypeAdapter[float | None](float | None) + + +async def _tpd_tokens_once_logged(cache: DualCache, key: str) -> int: + for _ in range(200): + logged: Final = _LOGGED_TOKENS.validate_python( + cache.in_memory_cache.get_cache(key) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # InMemoryCache.get_cache is untyped + ) + if logged is not None: + return int(logged) + await asyncio.sleep(0.01) + pytest.fail(f"the success callback never logged tpd tokens under {key}") + + +RESET_AT_EIGHT = datetime(2026, 10, 28, 8, 0, 0, tzinfo=timezone.utc) +DAILY_RESET_DEPLOYMENT = { + **RPD_DEPLOYMENT, + "litellm_params": {"model": "gpt-4.1-mini", "rpd": 2, "daily_reset_time": "08:00"}, +} + + +class TestDailyResetTime: + def test_rpd_window_starts_over_at_daily_reset_time_not_midnight(self): + clock = Clock(RESET_AT_EIGHT - timedelta(hours=9)) + check = ModelRateLimitingCheck(dual_cache=DualCache(), clock=clock) + + assert _pre_call(check, DAILY_RESET_DEPLOYMENT) == DAILY_RESET_DEPLOYMENT + clock.now = RESET_AT_EIGHT - timedelta(hours=7) + assert _pre_call(check, DAILY_RESET_DEPLOYMENT) == DAILY_RESET_DEPLOYMENT + clock.now = RESET_AT_EIGHT - timedelta(minutes=1) + with pytest.raises(litellm.RateLimitError, match="RPD limit=2, current usage=3") as exc_info: + _pre_call(check, DAILY_RESET_DEPLOYMENT) + assert exc_info.value.response.headers["retry-after"] == "60" + + clock.now = RESET_AT_EIGHT + assert _pre_call(check, DAILY_RESET_DEPLOYMENT) == DAILY_RESET_DEPLOYMENT + + @pytest.mark.asyncio + async def test_async_rpd_window_starts_over_at_daily_reset_time_not_midnight(self): + clock = Clock(RESET_AT_EIGHT - timedelta(hours=9)) + check = ModelRateLimitingCheck(dual_cache=DualCache(), clock=clock) + + assert await _async_pre_call(check, DAILY_RESET_DEPLOYMENT) == DAILY_RESET_DEPLOYMENT + clock.now = RESET_AT_EIGHT - timedelta(hours=7) + assert await _async_pre_call(check, DAILY_RESET_DEPLOYMENT) == DAILY_RESET_DEPLOYMENT + clock.now = RESET_AT_EIGHT - timedelta(minutes=1) + with pytest.raises(litellm.RateLimitError, match="RPD limit=2, current usage=3") as exc_info: + await _async_pre_call(check, DAILY_RESET_DEPLOYMENT) + assert exc_info.value.response.headers["retry-after"] == "60" + + clock.now = RESET_AT_EIGHT + assert await _async_pre_call(check, DAILY_RESET_DEPLOYMENT) == DAILY_RESET_DEPLOYMENT + + @pytest.mark.parametrize( + ("now", "seconds_to_reset"), + [ + (RESET_AT_EIGHT + timedelta(seconds=1), 86399), + (datetime(2026, 10, 28, 23, 59, 30, tzinfo=timezone.utc), 28830), + (datetime(2026, 10, 29, 7, 30, 0, tzinfo=timezone.utc), 1800), + ], + ) + def test_daily_429_retry_after_is_seconds_until_daily_reset_time(self, now: datetime, seconds_to_reset: int): + deployment = { + **RPD_DEPLOYMENT, + "litellm_params": {"model": "gpt-4.1-mini", "rpd": 0, "daily_reset_time": "08:00"}, + } + check = ModelRateLimitingCheck(dual_cache=DualCache(), clock=Clock(now)) + + with pytest.raises(litellm.RateLimitError) as exc_info: + _pre_call(check, deployment) + + assert exc_info.value.response.headers["retry-after"] == str(seconds_to_reset) + + @pytest.mark.asyncio + async def test_tokens_logged_before_daily_reset_time_do_not_count_after_it(self): + deployment = { + **RPD_DEPLOYMENT, + "litellm_params": {"model": "gpt-4.1-mini", "tpd": 100, "daily_reset_time": "08:00"}, + } + clock = Clock(RESET_AT_EIGHT - timedelta(minutes=1)) + check = ModelRateLimitingCheck(dual_cache=DualCache(), clock=clock) + await _async_log_success( + check, + { + "litellm_params": {"daily_reset_time": "08:00", "metadata": {}}, + "standard_logging_object": { + "model_id": "daily-test-id", + "total_tokens": 120, + "hidden_params": {"litellm_model_name": "gpt-4.1-mini"}, + }, + }, + ) + with pytest.raises(litellm.RateLimitError, match="TPD limit=100, current usage=120"): + await _async_pre_call(check, deployment) + + clock.now = RESET_AT_EIGHT + assert await _async_pre_call(check, deployment) == deployment + + @pytest.mark.parametrize( + "deployment", + [ + {"litellm_params": {"model": "gpt-4.1-mini"}, "model_info": {"id": "x", "rpd": 0}, "model_name": "m"}, + {"litellm_params": {"model": "gpt-4.1-mini"}, "model_info": {"id": "x"}, "model_name": "m", "rpd": 0}, + ], + ) + def test_rpd_outside_litellm_params_is_not_a_limit(self, deployment: dict[str, object]): + check = ModelRateLimitingCheck(dual_cache=DualCache(), clock=Clock(DAY_ONE)) + + assert _pre_call(check, deployment) == deployment + + @pytest.mark.parametrize("daily_reset_time", ["8am", "08:00:30", "08:00+05:00", "25:00"]) + def test_router_rejects_a_daily_reset_time_that_is_not_hh_mm(self, daily_reset_time: str): + with pytest.raises(Exception, match="daily_reset_time"): + Router( + model_list=[ + { + "model_name": "daily-group", + "litellm_params": {"model": "gpt-4.1-mini", "rpd": 2, "daily_reset_time": daily_reset_time}, + } + ] + ) + + @pytest.mark.asyncio + async def test_a_daily_reset_time_sent_in_the_request_does_not_move_the_tokens_out_of_the_deployment_window(self): + router = Router( + model_list=[ + { + "model_name": "daily-group", + "litellm_params": { + "model": "openai/gpt-4.1-mini", + "api_key": "k", + "mock_response": "a", + "tpd": 100, + }, + "model_info": {"id": "daily-test-id"}, + } + ] + ) + check = ModelRateLimitingCheck(dual_cache=router.cache, clock=Clock(RESET_AT_EIGHT - timedelta(minutes=1))) + router.optional_callbacks = [check] + litellm.logging_callback_manager.add_litellm_callback(check) + try: + await router.acompletion( # pyright: ignore[reportUnknownMemberType] # Router.acompletion kwargs are untyped + model="daily-group", messages=[{"role": "user", "content": "hi"}], daily_reset_time="08:00" + ) + logged = await _tpd_tokens_once_logged(router.cache, "daily-test-id:openai/gpt-4.1-mini:tpd:2026-10-28") + finally: + router.discard() + + assert logged > 0 + day_before: Final = _LOGGED_TOKENS.validate_python( + router.cache.in_memory_cache.get_cache("daily-test-id:openai/gpt-4.1-mini:tpd:2026-10-27") # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # InMemoryCache.get_cache is untyped + ) + assert day_before is None diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 4cae52289a6..b99077b43b5 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -10883,3 +10883,23 @@ def test_get_optional_params_forwards_a_tool_config_cache_point_to_a_downstream_ ) assert optional_params["extra_body"] == {"kept": True, "cache_control_injection_points": [point]} + + +@pytest.mark.parametrize( + ("provider", "expected_warnings"), + (("streamlake", 1), ("veniceai", 1), ("my-own-llm", 0)), +) +def test_custom_provider_named_like_a_built_in_warns_once( + provider: str, expected_warnings: int, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + monkeypatch.setattr(litellm, "custom_provider_map", [{"provider": provider, "custom_handler": litellm.CustomLLM()}]) + monkeypatch.setattr(litellm, "provider_list", list(litellm.provider_list)) + monkeypatch.setattr(litellm, "_custom_providers", []) + + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + litellm.utils.custom_llm_setup() + litellm.utils.custom_llm_setup() + + warnings: Final = [r for r in caplog.records if f"provider '{provider}'" in r.getMessage()] + assert len(warnings) == expected_warnings + assert provider in litellm._custom_providers diff --git a/tests/unit/types/test_litellm_params.py b/tests/unit/types/test_litellm_params.py index cc1ea645012..e369b7e93ab 100644 --- a/tests/unit/types/test_litellm_params.py +++ b/tests/unit/types/test_litellm_params.py @@ -156,6 +156,7 @@ OPTION_NAMES: Final = ( "tpm", "rpd", "tpd", + "daily_reset_time", "itpm", "otpm", "default_api_key_rpm_limit", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx index 43b9a5541db..c9b3defcd28 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx @@ -1,5 +1,5 @@ import { useSyncExternalStore } from "react"; -import { act, renderWithProviders, screen, waitFor, within } from "../../../../../tests/test-utils"; +import { act, fireEvent, renderWithProviders, screen, waitFor, within } from "../../../../../tests/test-utils"; import type { OnUrlUpdateFunction } from "nuqs/adapters/testing"; import userEvent, { PointerEventsCheckLevel } from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi } from "vitest"; @@ -80,6 +80,13 @@ vi.mock("@/app/(dashboard)/hooks/providers/useProviderFields", () => ({ default_model_placeholder: "jev-latest", credential_fields: [{ key: "api_key", label: "API Key", field_type: "password", required: false }], }, + { + provider: "StreamLake", + provider_display_name: "StreamLake", + litellm_provider: "streamlake", + default_model_placeholder: "streamlake/", + credential_fields: [{ key: "api_key", label: "API Key", field_type: "password", required: false }], + }, ], isLoading: false, error: null, @@ -278,6 +285,28 @@ describe("AddModelPanel submit payload contract", () => { }); }); + it("creates a StreamLake model from a public model selection and API key", async () => { + mockUseModelCostMap.mockReturnValue({ data: { "streamlake/GLM-5.3": { litellm_provider: "streamlake" } } }); + const { user, submit } = await setup(); + + await user.click(screen.getByRole("combobox", { name: /provider/i })); + await user.click(await screen.findByText("StreamLake")); + await user.click(screen.getByRole("combobox", { name: "Select models" })); + await user.click(await screen.findByText("streamlake/GLM-5.3")); + fireEvent.change(screen.getByLabelText("API Key"), { target: { value: "sk-streamlake-test" } }); + await submit(); + + expect(lastCreatedModel()).toStrictEqual({ + model_name: "streamlake/GLM-5.3", + litellm_params: { + model: "streamlake/GLM-5.3", + custom_llm_provider: "streamlake", + api_key: "sk-streamlake-test", + }, + model_info: { ...baseModelInfo }, + }); + }); + it("registers four more keys as undefined once Advanced Settings opens", async () => { const { openAdvanced, fillRequired, submit } = await setup(); await fillRequired(); diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx index 83e80911cff..53e54e2b032 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx @@ -222,6 +222,7 @@ describe("provider_info_helpers", () => { Providers.PG_VECTOR, Providers.PREDIBASE, Providers.Sail, + Providers.StreamLake, Providers.WANDB, Providers.ZAI, Providers.ZeroGPU, @@ -462,6 +463,19 @@ describe("provider_info_helpers", () => { expect(result).toEqual(["gpt-3.5-turbo", "gpt-4"]); }); + it("should return StreamLake models and exclude models from other providers", () => { + const modelMap = { + "streamlake/GLM-5.3": { litellm_provider: "streamlake" }, + "streamlake/DeepSeek-V3": { litellm_provider: "streamlake" }, + "gpt-4o": { litellm_provider: "openai" }, + }; + const result = getProviderModels(Providers.StreamLake, modelMap); + + expect(provider_map.StreamLake).toBe("streamlake"); + expect(result).toEqual(["streamlake/GLM-5.3", "streamlake/DeepSeek-V3"]); + expect(result).not.toContain("gpt-4o"); + }); + it("should return models whose litellm_provider is a prefix-anchored variant of the provider", () => { const modelMap = { "anthropic-text-model": { litellm_provider: "anthropic_text" }, diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index bfb578b218a..1e1aafde1b5 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -177,6 +177,7 @@ export enum Providers { Snowflake = "Snowflake", Soniox = "Soniox", StrandsDecider = "Strands Decider", + StreamLake = "StreamLake", TEXT_COMPLETION_CODESTRAL = "Text-Completion-Codestral", Tencent = "Tencent", TogetherAI = "TogetherAI", @@ -306,6 +307,7 @@ export const provider_map: Record = { Snowflake: "snowflake", Soniox: "soniox", StrandsDecider: "strands_decider", + StreamLake: "streamlake", TEXT_COMPLETION_CODESTRAL: "text-completion-codestral", Tencent: "tencent", TogetherAI: "together_ai", @@ -494,6 +496,7 @@ const providerPlaceholderMap: Partial> = { [Providers.Tencent]: "tencent/deepseek-v4-pro", [Providers.TypeSafe]: "typesafe/jev-latest", [Providers.Nace]: "nace/drex-latest", + [Providers.StreamLake]: "streamlake/", [Providers.Vertex_AI]: "gemini-pro", [Providers.VolcEngine]: "volcengine/", [Providers.Voyage]: "voyage/", diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index a49f1099121..51348fa9c75 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -35397,6 +35397,8 @@ export interface components { cost_per_second?: number | null; /** Custom Llm Provider */ custom_llm_provider?: string | null; + /** Daily Reset Time */ + daily_reset_time?: string | null; /** Default Api Key Rpm Limit */ default_api_key_rpm_limit?: number | null; /** Default Api Key Tpm Limit */ @@ -50793,6 +50795,8 @@ export interface components { cost_per_second?: number | null; /** Custom Llm Provider */ custom_llm_provider?: string | null; + /** Daily Reset Time */ + daily_reset_time?: string | null; /** Default Api Key Rpm Limit */ default_api_key_rpm_limit?: number | null; /** Default Api Key Tpm Limit */