Merge remote-tracking branch 'origin/main' into add-vispark-litellm

# Conflicts:
#	litellm/llms/openai_like/json_loader.py
#	litellm/llms/openai_like/providers.json
This commit is contained in:
mateo-berri 2026-10-28 22:39:06 -07:00
commit d4ce13aef8
83 changed files with 4295 additions and 380 deletions

View file

@ -15,7 +15,10 @@ import litellm
from litellm._logging import verbose_proxy_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails._content_utils import iter_message_text
from litellm.proxy.guardrails._content_utils import (
is_non_conversational_request,
iter_message_text,
)
from litellm.types.utils import CallTypesLiteral
_THRESHOLD_VALUE: Final = TypeAdapter(object)
@ -94,7 +97,7 @@ class ENTERPRISE_GoogleTextModeration(CustomLogger):
async def async_moderation_hook(
self,
data: dict,
data: dict[str, object],
user_api_key_dict: UserAPIKeyAuth,
call_type: CallTypesLiteral,
):
@ -102,6 +105,9 @@ class ENTERPRISE_GoogleTextModeration(CustomLogger):
- Calls Google's Text Moderation API
- Rejects request if it fails safety check
"""
if is_non_conversational_request(call_type, data):
verbose_proxy_logger.debug("Google text moderation: skipping non-conversational call type %s", call_type)
return
# Covers multimodal list content + Responses-API input.
text = "".join(iter_message_text(data))
if text:

View file

@ -33,9 +33,8 @@ class EnterpriseCallbackControls:
try:
disabled_callbacks = EnterpriseCallbackControls.get_disabled_callbacks(litellm_params, standard_callback_dynamic_params)
verbose_logger.debug(f"Dynamically disabled callbacks from {X_LITELLM_DISABLE_CALLBACKS}: {disabled_callbacks}")
verbose_logger.debug(f"Checking if {callback} is disabled via headers. Disable callbacks from headers: {disabled_callbacks}")
if disabled_callbacks is not None:
verbose_logger.debug(f"Checking if {callback} is in dynamically disabled callbacks: {disabled_callbacks}")
#########################################################
# premium user check
#########################################################

View file

@ -1,14 +1,31 @@
use serde_json::Value;
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct AudioTranscriptionUsage {
pub input_tokens: u64,
pub output_tokens: u64,
pub total_tokens: u64,
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct AudioTranscriptionResponseData {
pub text: String,
pub usage: Option<AudioTranscriptionUsage>,
}
impl AudioTranscriptionResponseData {
pub fn into_json(self) -> Value {
serde_json::json!({
"text": self.text,
})
match self.usage {
None => serde_json::json!({"text": self.text}),
Some(usage) => serde_json::json!({
"text": self.text,
"usage": {
"type": "tokens",
"input_tokens": usage.input_tokens,
"output_tokens": usage.output_tokens,
"total_tokens": usage.total_tokens,
},
}),
}
}
}

View file

@ -6,8 +6,8 @@ use litellm_auth_aws::{
};
use litellm_core_utils::core_helpers::json_type_name;
use litellm_llms_types::{
formats::audio_transcription::AudioTranscriptionResponseData,
providers::bedrock::ConverseResponse,
formats::audio_transcription::{AudioTranscriptionResponseData, AudioTranscriptionUsage},
providers::bedrock::{ConverseResponse, ConverseUsage},
};
use serde::Deserialize;
use serde_json::{Map, Value, json};
@ -101,10 +101,10 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
if let Some(prompt) = optional_string(&optional_params, "prompt") {
instruction.push_str(&format!(" Additional context: {prompt}"));
}
let mut inference_config = Map::from_iter([("maxTokens".to_string(), json!(4096))]);
if let Some(temperature) = optional_params.get("temperature") {
inference_config.insert("temperature".to_string(), temperature.clone());
}
let temperature = optional_params
.get("temperature")
.cloned()
.unwrap_or(json!(0));
Ok(AudioTranscriptionRequestData {
body: json!({
"messages": [{
@ -114,8 +114,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
{"text": instruction}
]
}],
"system": [{"text": "You are a transcription assistant."}],
"inferenceConfig": inference_config,
"inferenceConfig": {"maxTokens": 4096, "temperature": temperature},
}),
})
}
@ -125,6 +124,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
_model: &str,
response_json: Value,
) -> Result<AudioTranscriptionResponseData, Error> {
let reports_usage = response_json.get("usage").is_some();
let response: ConverseResponse =
serde_json::from_value(response_json).map_err(|error| {
Error::InvalidResponse(ErrorDetail::invalid("Converse response", error))
@ -137,6 +137,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
}
Ok(AudioTranscriptionResponseData {
text: response.content_text(),
usage: reports_usage.then(|| transcription_usage(&response.usage)),
})
}
@ -191,22 +192,41 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
}
}
fn transcription_usage(usage: &ConverseUsage) -> AudioTranscriptionUsage {
AudioTranscriptionUsage {
input_tokens: usage.input_tokens,
output_tokens: usage.output_tokens,
total_tokens: usage
.total_tokens
.unwrap_or(usage.input_tokens + usage.output_tokens),
}
}
#[cfg(test)]
mod tests {
use super::*;
use rstest::rstest;
fn no_env(_: &str) -> Option<String> {
None
}
#[test]
fn request_matches_python_shape() {
let params = Map::from_iter([
("language".to_string(), json!("en")),
("prompt".to_string(), json!("Speaker names")),
("temperature".to_string(), json!(0)),
("timestamp_granularities".to_string(), json!(["word"])),
]);
#[rstest]
#[case::caller_temperature(Some(json!(0.4)), json!(0.4))]
#[case::default_temperature(None, json!(0))]
fn request_sends_audio_and_instruction_without_system_block(
#[case] temperature: Option<Value>,
#[case] expected_temperature: Value,
) {
let params = Map::from_iter(
[
("language".to_string(), json!("en")),
("prompt".to_string(), json!("Speaker names")),
("timestamp_granularities".to_string(), json!(["word"])),
]
.into_iter()
.chain(temperature.map(|value| ("temperature".to_string(), value))),
);
let params = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG.map_transcription_params(&params);
let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG
.transform_audio_transcription_request(
@ -225,22 +245,39 @@ mod tests {
{"text": "Transcribe the audio. Respond with only the transcript. The audio language is en. Additional context: Speaker names"}
]
}],
"system": [{"text": "You are a transcription assistant."}],
"inferenceConfig": {"maxTokens": 4096, "temperature": 0}
"inferenceConfig": {"maxTokens": 4096, "temperature": expected_temperature}
})
);
}
#[test]
fn response_concatenates_content_blocks() {
#[rstest]
#[case::reported_total(
json!({"inputTokens": 390, "outputTokens": 60, "totalTokens": 450}),
json!({"text": "hello world", "usage": {"type": "tokens", "input_tokens": 390, "output_tokens": 60, "total_tokens": 450}})
)]
#[case::derived_total(
json!({"inputTokens": 1, "outputTokens": 2}),
json!({"text": "hello world", "usage": {"type": "tokens", "input_tokens": 1, "output_tokens": 2, "total_tokens": 3}})
)]
fn response_carries_text_and_converse_usage(#[case] usage: Value, #[case] expected: Value) {
let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG
.transform_audio_transcription_response(
"model",
json!({"output": {"message": {"content": [{"text": "hello "}, {"text": "world"}]}}, "usage": {"inputTokens": 1, "outputTokens": 2}}),
json!({"output": {"message": {"content": [{"text": "hello "}, {"text": "world"}]}}, "usage": usage}),
)
.expect("response");
assert_eq!(result.text, "hello world");
assert_eq!(result.into_json(), json!({"text": "hello world"}));
assert_eq!(result.into_json(), expected);
}
#[rstest]
fn response_without_usage_reports_none() {
let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG
.transform_audio_transcription_response(
"model",
json!({"output": {"message": {"content": [{"text": "hello"}]}}}),
)
.expect("response");
assert_eq!(result.into_json(), json!({"text": "hello"}));
}
#[test]

View file

@ -681,6 +681,7 @@ moonshot_models: Set = set()
publicai_models: Set = set()
darkbloom_models: Set = set()
openinfer_models: Final[Set[str]] = set()
streamlake_models: Final[Set[str]] = set()
zerogpu_models: Final[Set[str]] = set()
v0_models: Set = set()
vispark_models: Set[str] = set()
@ -971,6 +972,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
darkbloom_models.add(key)
elif value.get("litellm_provider") == "openinfer":
openinfer_models.add(key)
elif value.get("litellm_provider") == "streamlake":
streamlake_models.add(key)
elif value.get("litellm_provider") == "zerogpu":
zerogpu_models.add(key)
elif value.get("litellm_provider") == "v0":
@ -1148,6 +1151,7 @@ model_list = list(
| publicai_models
| darkbloom_models
| openinfer_models
| streamlake_models
| zerogpu_models
| v0_models
| vispark_models
@ -1269,6 +1273,7 @@ def _build_models_by_provider() -> dict:
"publicai": publicai_models,
"darkbloom": darkbloom_models,
"openinfer": openinfer_models,
"streamlake": streamlake_models,
"zerogpu": zerogpu_models,
"v0": v0_models,
"vispark": vispark_models,

View file

@ -1065,6 +1065,7 @@ openai_compatible_endpoints: Final[list[str]] = [
"https://api.reka.ai/v1",
"https://gigachat.devices.sberbank.ru/api/v1",
"https://api.openinfer.ai/v1",
"https://vanchin.streamlake.ai/api/gateway/v1/endpoints",
]
@ -1147,6 +1148,7 @@ openai_compatible_providers: Final[list[str]] = [
"prism",
"sail",
"reka",
"streamlake",
]
OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS: Final = frozenset({"openai"} | frozenset(openai_compatible_providers))

View file

@ -693,7 +693,7 @@ def cost_per_token(
elif call_type == "atranscription" or call_type == "transcription":
if _transcription_usage_has_token_details(usage_block):
return generic_cost_per_token(
model=model_without_prefix,
model=model,
usage=usage_block,
custom_llm_provider=custom_llm_provider,
service_tier=service_tier,

View file

@ -77,6 +77,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
"otpm",
"tpd",
"rpd",
"daily_reset_time",
"use_xai_oauth",
"chatgpt_auth_file",
"github_copilot_auth_type",

View file

@ -1130,9 +1130,6 @@ class Logging(LiteLLMLoggingBaseClass):
}
self.litellm_request_debug = litellm_params.get("litellm_request_debug", False)
self.logger_fn = litellm_params.get("logger_fn", None)
if is_debugging_on() or self.litellm_request_debug:
verbose_logger.debug("self.optional_params: %s", self.optional_params)
self.model_call_details.update(
{
"model": self.model,

View file

@ -27,13 +27,18 @@ class TranscriptionUsageObjectTransformation:
if isinstance(usage_object, TranscriptionUsageDurationObject):
return None
elif isinstance(usage_object, TranscriptionUsageTokensObject):
input_token_details: Final = usage_object.input_token_details
return Usage(
prompt_tokens=usage_object.input_tokens,
completion_tokens=usage_object.output_tokens,
total_tokens=usage_object.total_tokens,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=usage_object.input_token_details.text_tokens,
audio_tokens=usage_object.input_token_details.audio_tokens,
prompt_tokens_details=(
None
if input_token_details is None
else PromptTokensDetailsWrapper(
text_tokens=input_token_details.text_tokens,
audio_tokens=input_token_details.audio_tokens,
)
),
)
return None

View file

@ -92,6 +92,18 @@ def _next_sync_or_exhausted(it: Iterator[object]) -> object:
return _SYNC_ITER_EXHAUSTED
def _stamp_usage_cost(
usage_chunk: ModelResponse | TextCompletionResponse | ModelResponseStream,
complete_response: ModelResponse | TextCompletionResponse | ModelResponseStream,
) -> None:
bill: Final = (get_hidden_params(complete_response) or {}).get("response_cost")
usage: Final[object] = getattr(usage_chunk, "usage", None)
if usage is None or isinstance(bill, bool) or not isinstance(bill, (int, float)):
return
if bill > 0 or isinstance(getattr(usage, "cost", None), (int, float)):
setattr(usage, "cost", float(bill)) # noqa: B010 # cost is an optional extra on the usage model
def _stamp_served_service_tier(response: ModelResponseStream, complete_streaming_response: ModelResponse) -> None:
served_tier: Final = complete_streaming_response.model_dump().get("service_tier")
if isinstance(served_tier, str) and served_tier:
@ -2141,6 +2153,7 @@ class CustomStreamWrapper:
getattr(complete_streaming_response, "usage"),
)
_stamp_served_service_tier(response, complete_streaming_response)
_stamp_usage_cost(response, complete_streaming_response)
try:
_cache_copy = complete_streaming_response.model_copy(deep=True)
_log_copy = complete_streaming_response.model_copy(deep=True)
@ -2361,6 +2374,7 @@ class CustomStreamWrapper:
getattr(complete_streaming_response, "usage"),
)
_stamp_served_service_tier(response, complete_streaming_response)
_stamp_usage_cost(response, complete_streaming_response)
try:
_copy = complete_streaming_response.model_copy(deep=True)
except RuntimeError:

View file

@ -621,7 +621,7 @@ class BaseAzureLLM(BaseOpenAILLM):
# For Azure v1 API, use standard OpenAI client instead of AzureOpenAI
# See: https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#api-specs
if self._is_azure_v1_api_version(api_version):
if self.is_azure_v1_api_version(api_version):
# Extract only params that OpenAI client accepts
# Always use /openai/v1/ regardless of whether user passed "v1", "latest", or "preview"
# The OpenAI client accepts a callable for `api_key` and re-invokes it
@ -922,7 +922,7 @@ class BaseAzureLLM(BaseOpenAILLM):
else:
new_url = api_base
if BaseAzureLLM._is_azure_v1_api_version(api_version):
if BaseAzureLLM.is_azure_v1_api_version(api_version):
# ensure the request go to /openai/v1 and not just /openai
if "/openai/v1" not in new_url:
parsed_url: Final = httpx.URL(new_url)
@ -952,7 +952,7 @@ class BaseAzureLLM(BaseOpenAILLM):
Returns None when ``api_version`` is a dated one, which still uses the deployment route.
"""
if not BaseAzureLLM._is_azure_v1_api_version(api_version):
if not BaseAzureLLM.is_azure_v1_api_version(api_version):
return None
base_url: Final = httpx.URL(api_base)
@ -970,7 +970,7 @@ class BaseAzureLLM(BaseOpenAILLM):
)
@staticmethod
def _is_azure_v1_api_version(api_version: str | None) -> bool:
def is_azure_v1_api_version(api_version: str | None) -> bool:
if api_version is None:
return False
return api_version in {"preview", "latest", "v1"}

View file

@ -54,6 +54,16 @@ def _resolve_responses_api_version(api_base: str | None, litellm_params: Mapping
return api_version
def _v1_responses_url(api_base: str) -> httpx.URL:
parsed_url: Final = httpx.URL(api_base)
path: Final = parsed_url.path.rstrip("/")
root: Final = next(
(path.removesuffix(route) for route in ("/openai/v1/responses", "/openai/responses") if path.endswith(route)),
path,
)
return parsed_url.copy_with(path=f"{root}/openai/v1/responses", query=None)
def _with_responses_api_version(api_base: str | None, litellm_params: Mapping[str, object]) -> Mapping[str, object]:
return MappingProxyType({**litellm_params, "api_version": _resolve_responses_api_version(api_base, litellm_params)})
@ -247,20 +257,20 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
if api_base is None:
raise ValueError("api_base is required for Azure WebSocket")
parsed_url: Final = httpx.URL(api_base)
path = parsed_url.path.rstrip("/")
# Strip existing /openai/responses path if the api_base already contains it
for suffix in ("/openai/v1/responses", "/openai/responses"):
if path.endswith(suffix):
path = path[: -len(suffix)]
break
scheme: Final = "wss" if parsed_url.scheme == "https" else "ws"
return str(parsed_url.copy_with(scheme=scheme, path=f"{path}/openai/v1/responses", query=None))
v1_url: Final = _v1_responses_url(api_base)
return str(v1_url.copy_with(scheme="wss" if v1_url.scheme == "https" else "ws"))
def model_in_websocket_url(self) -> bool:
# Azure sends the model in the response.create body, not the URL
return False
def compact_url(self, api_base: str) -> str:
params: Final = httpx.URL(api_base).params
v1_params: Final = (
params if BaseAzureLLM.is_azure_v1_api_version(params.get("api-version")) else params.remove("api-version")
)
return super().compact_url(str(_v1_responses_url(api_base).copy_with(params=v1_params)))
#########################################################
########## DELETE RESPONSE API TRANSFORMATION ##############
#########################################################

View file

@ -1,7 +1,11 @@
import base64
import math
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Final
import httpx
from pydantic import BaseModel, ConfigDict
from litellm.litellm_core_utils.audio_utils.utils import process_audio_file
from litellm.rust_bridge import runtime
@ -14,7 +18,72 @@ from litellm.rust_bridge.transcription.native import (
RustAtranscription,
RustTranscription,
)
from litellm.types.utils import FileTypes, TranscriptionResponse
from litellm.secret_managers.main import get_secret_str
from litellm.types.utils import FileTypes, TranscriptionResponse, TranscriptionUsageTokensObject
SUPPORTED_OPENAI_TRANSCRIPTION_PARAMS: Final = ("language", "prompt", "temperature", "response_format")
TEXT_RESPONSE_FORMATS: Final = frozenset({"json", "text"})
def _as_float(value: object) -> float | None:
if isinstance(value, bool) or not isinstance(value, (int, float, str)):
return None
try:
number: Final = float(value)
except ValueError:
return None
return number if math.isfinite(number) else None
def _rejection(key: str, value: object) -> str | None:
if key not in SUPPORTED_OPENAI_TRANSCRIPTION_PARAMS:
return f"bedrock transcription does not support {key}."
if key == "response_format" and value not in TEXT_RESPONSE_FORMATS:
return f"bedrock transcription returns only the transcript text, so response_format={value!r} is not supported."
if key == "temperature" and _as_float(value) is None:
return f"bedrock transcription needs a number for temperature, got {value!r}."
return None
@dataclass(frozen=True, slots=True)
class UnsupportedTranscriptionParams:
message: str
def map_openai_transcription_params(
non_default_params: Mapping[str, object], drop_params: bool
) -> Mapping[str, object] | UnsupportedTranscriptionParams:
rejections: Final = {
key: reason for key, value in non_default_params.items() if (reason := _rejection(key, value)) is not None
}
if rejections and not drop_params:
return UnsupportedTranscriptionParams(
message=f"{' '.join(rejections.values())} To drop it from the call, set `litellm.drop_params = True`."
)
return {
key: _as_float(value) if key == "temperature" else value
for key, value in non_default_params.items()
if key not in rejections
}
class _RustTranscriptionResult(BaseModel):
model_config = ConfigDict(frozen=True)
text: str
usage: TranscriptionUsageTokensObject | None = None
def _transcription_response(
result: Mapping[str, object], optional_params: Mapping[str, object]
) -> TranscriptionResponse:
parsed: Final = _RustTranscriptionResult.model_validate(result)
response: Final = TranscriptionResponse(text=parsed.text).model_copy(update={"usage": parsed.usage})
region_name: Final = (
optional_params.get("aws_region_name") or get_secret_str("AWS_REGION_NAME") or get_secret_str("AWS_REGION")
)
response.hidden_params = {**response.hidden_params, "region_name": region_name}
return response
class BedrockAudioTranscriptionRustDispatch:
@ -64,7 +133,7 @@ class BedrockAudioTranscriptionRustDispatch:
"timeout_seconds": timeout_to_seconds(timeout),
}
call: Final = NativeCall(args=(), kwargs=fields, base={})
return TranscriptionResponse(**rust(call))
return _transcription_response(rust(call), optional_params)
return runtime.run(
RouteContext(Route.TRANSCRIPTION, provider=custom_llm_provider, model=model),
@ -97,7 +166,7 @@ class BedrockAudioTranscriptionRustDispatch:
"timeout_seconds": timeout_to_seconds(timeout),
}
call: Final = NativeCall(args=(), kwargs=fields, base={})
return TranscriptionResponse(**await rust(call))
return _transcription_response(await rust(call), optional_params)
return await runtime.arun(
RouteContext(Route.TRANSCRIPTION, provider=custom_llm_provider, model=model),

View file

@ -32,11 +32,11 @@ import copy
import json
import time
import uuid
from collections.abc import Mapping, MutableSequence, Sequence
from collections.abc import Iterable, Mapping, MutableSequence, Sequence
from dataclasses import dataclass
from itertools import accumulate, chain, repeat
from itertools import accumulate, chain, groupby, repeat
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, Union, cast
from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, TypeVar, Union, cast
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
@ -55,7 +55,9 @@ from litellm.llms.base_llm.guardrail_translation.base_translation import (
)
from litellm.llms.base_llm.guardrail_translation.utils import (
blocked_responses_stream_usage,
effective_scan_only_tool_results_for_guardrail,
effective_skip_system_message_for_guardrail,
effective_skip_tool_message_for_guardrail,
list_field,
merge_guardrailed_scoped_messages,
role_out_of_guardrail_scope,
@ -65,7 +67,10 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
stream_item_items,
unappliable_request_rewrite,
)
from litellm.llms.openai.responses.guardrail_translation.tool_merge import merge_guardrailed_tools
from litellm.llms.openai.responses.guardrail_translation.tool_merge import (
append_returned_tools,
merge_guardrailed_tools,
)
from litellm.responses.litellm_completion_transformation.compaction_items import (
compaction_summary,
decode_compaction_item,
@ -74,7 +79,9 @@ from litellm.responses.litellm_completion_transformation.compaction_items import
from litellm.responses.litellm_completion_transformation.custom_tools import without_malformed_tool_flags
from litellm.responses.litellm_completion_transformation.tool_search import is_server_tool_search_tool
from litellm.responses.litellm_completion_transformation.transformation import (
TOOL_CALL_OUTPUT_ITEM_TYPES,
LiteLLMCompletionResponsesConfig,
tool_output_to_chat_content,
)
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.openai import (
@ -270,6 +277,7 @@ _TOOL_CALL_PAYLOAD_EVENT_TYPES: Final = _TOOL_CALL_PAYLOAD_DELTA_EVENT_TYPES | f
)
_OUTPUT_ITEM_EVENT_TYPES: Final = frozenset({"response.output_item.added", "response.output_item.done"})
_OUTPUT_TEXT_EVENT_TYPES: Final = frozenset({"response.output_text.delta", "response.output_text.done"})
_TOOL_OUTPUT_TEXT_PART_TYPES: Final = frozenset({"input_text", "output_text", "text"})
_PATCHABLE_ITEM_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
{"function_call_output": "output", "custom_tool_call_output": "output", "message": "content"}
)
@ -519,10 +527,109 @@ def scannable_instructions(data: Mapping[str, object], *, skip_system: bool = Fa
def _input_item_role(item: object) -> str:
if stream_item_field(item, "type") in TOOL_CALL_OUTPUT_ITEM_TYPES:
return "tool"
role: Final = item.get("role") if isinstance(item, Mapping) else None
return role.lower() if isinstance(role, str) else ""
def _scanned_text_field(item: Mapping[str, object]) -> str:
return "output" if item.get("type") in TOOL_CALL_OUTPUT_ITEM_TYPES else "content"
def _tool_output_part_text(part: object) -> str | None:
if not isinstance(part, dict):
return None
fields: Final = _item_fields_adapter.validate_python(part)
text: Final = fields.get("text")
return text if fields.get("type") in _TOOL_OUTPUT_TEXT_PART_TYPES and isinstance(text, str) else None
def tool_output_texts(item: object) -> tuple[tuple[int | None, str], ...]:
if not stream_item_field(item, "call_id") or stream_item_field(item, "type") == "computer_call_output":
return ()
parts: Final = stream_item_items(item, "output")
part_texts: Final = tuple(
(index, text) for index, part in enumerate(parts) if (text := _tool_output_part_text(part))
)
if part_texts:
return part_texts
tool_content: Final = tool_output_to_chat_content(stream_item_field(item, "output"))
return ((None, tool_content),) if isinstance(tool_content, str) and tool_content else ()
_Grouped: Final = TypeVar("_Grouped")
def _pair_call_id(pair: tuple[str, object]) -> str:
return pair[0]
def _grouped_by_call_id(pairs: Iterable[tuple[str, _Grouped]]) -> Mapping[str, tuple[_Grouped, ...]]:
ordered: Final = sorted(pairs, key=_pair_call_id)
return MappingProxyType(
{call_id: tuple(value for _, value in group) for call_id, group in groupby(ordered, key=_pair_call_id)}
)
def _texts_by_tool_row(
messages: Sequence[Mapping[str, object]], input_data: "str | ResponseInputParam"
) -> Mapping[int, tuple[str, ...]]:
if isinstance(input_data, str):
return MappingProxyType({})
output_texts_by_call_id: Final = _grouped_by_call_id(
(str(stream_item_field(item, "call_id")), tuple(text for _, text in tool_output_texts(item)))
for item in input_data
if stream_item_field(item, "type") in TOOL_CALL_OUTPUT_ITEM_TYPES
)
row_indices_by_call_id: Final = _grouped_by_call_id(
(str(message.get("tool_call_id")), index)
for index, message in enumerate(messages)
if message.get("role") == "tool"
)
return MappingProxyType(
dict(
chain.from_iterable(
zip(indices, output_texts_by_call_id[call_id], strict=True)
for call_id, indices in row_indices_by_call_id.items()
if len(indices) == len(output_texts_by_call_id.get(call_id, ()))
)
)
)
def _is_string_tool_row(message: Mapping[str, object]) -> bool:
return message.get("role") == "tool" and isinstance(message.get("content"), str)
def _split_into_text_parts(message: AllMessageValues, texts: tuple[str, ...]) -> AllMessageValues:
if not _is_string_tool_row(message) or len(texts) < 2:
return message
return cast( # cast-ok: a tool row whose content is its text parts as chat text blocks
"AllMessageValues", {**message, "content": [{"type": "text", "text": text} for text in texts]}
)
def _with_split_rows_restored(
rows: Sequence[AllMessageValues], split_rows: Sequence[tuple[AllMessageValues, AllMessageValues]]
) -> Sequence[AllMessageValues]:
if not isinstance(rows, list):
return rows
return [next((original for split, original in split_rows if row == split), row) for row in rows]
def _is_textless_tool_row(message: Mapping[str, object], texts: tuple[str, ...] | None) -> bool:
return _is_string_tool_row(message) and (not message.get("content") or texts == ())
def _rewritten_whole_text(item: object, field: str, guardrail_response: str) -> object | None:
if isinstance(stream_item_field(item, field), str):
return guardrail_response
if tool_output_texts(item) == ((None, guardrail_response),):
return None
return guardrail_response
def _patched_request_fields(
raw_input: object,
instructions: object,
@ -636,17 +743,28 @@ class OpenAIResponsesHandler(BaseTranslation):
Handles both string input and list of message objects.
"""
input_data: Final[str | ResponseInputParam | None] = data.get("input")
if not isinstance(input_data, (str, list)):
raw_input: Final[str | ResponseInputParam | None] = data.get("input")
if not isinstance(raw_input, (str, list)):
return data
input_data: Final = cast("str | ResponseInputParam", raw_input) # cast-ok: fields read via stream_item_field
skip_system: Final = effective_skip_system_message_for_guardrail(guardrail_to_apply)
skip_tool: Final = effective_skip_tool_message_for_guardrail(guardrail_to_apply)
scan_only: Final = effective_scan_only_tool_results_for_guardrail(guardrail_to_apply)
structured_messages: Final = self.get_structured_messages(data)
scoped_indices: Final = scoped_structured_message_indices(
structured_messages or [], scan_only_tool_results=False, skip_system=skip_system, skip_tool=False
all_messages: Final = structured_messages or []
texts_by_tool_row: Final = _texts_by_tool_row(all_messages, input_data)
scoped_indices: Final = tuple(
index
for index in scoped_structured_message_indices(
all_messages, scan_only_tool_results=scan_only, skip_system=skip_system, skip_tool=skip_tool
)
if not _is_textless_tool_row(all_messages[index], texts_by_tool_row.get(index))
)
scoped_structured_messages: Final = (
[structured_messages[index] for index in scoped_indices] if structured_messages else None
split_rows: Final = tuple(
(_split_into_text_parts(all_messages[index], texts_by_tool_row.get(index, ())), all_messages[index])
for index in scoped_indices
)
scoped_structured_messages: Final = [split for split, _ in split_rows] if structured_messages else None
raw_tools: Final = data.get("tools")
original_tools: Final[tuple[Mapping[str, object], ...]] = (
tuple(raw_tools) if isinstance(raw_tools, list) else ()
@ -661,7 +779,12 @@ class OpenAIResponsesHandler(BaseTranslation):
)
)
extracted: Final = self._extract_guardrail_inputs(
data, input_data, flattened_tool_groups, skip_system=skip_system
data,
input_data,
() if scan_only else flattened_tool_groups,
skip_system=skip_system,
skip_tool=skip_tool,
scan_only=scan_only,
)
if not extracted.inputs.get("texts"):
return data
@ -674,13 +797,14 @@ class OpenAIResponsesHandler(BaseTranslation):
logging_obj=litellm_logging_obj,
)
self._apply_guardrailed_tools_to_data(
data, original_tools, flattened_tool_groups, guardrailed_inputs.get("tools")
data, original_tools, flattened_tool_groups, guardrailed_inputs.get("tools"), scan_only=scan_only
)
written_back: Final = self._written_back_request_fields(
data,
structured_messages or (),
scoped_indices,
scoped_structured_messages,
tuple((split, original) for split, original in split_rows if split is not original),
guardrail_to_apply,
guardrailed_inputs,
)
@ -730,8 +854,10 @@ class OpenAIResponsesHandler(BaseTranslation):
flattened_tool_groups: Sequence[Sequence[Mapping[str, object]]],
*,
skip_system: bool = False,
skip_tool: bool = False,
scan_only: bool = False,
) -> _ExtractedInputs:
instructions: Final = scannable_instructions(data, skip_system=skip_system)
instructions: Final = scannable_instructions(data, skip_system=skip_system or scan_only)
texts_to_check: Final[list[str]] = [] if instructions is None else [instructions]
images_to_check: Final[list[str]] = []
task_mappings: Final[list[tuple[int, int | None]]] = []
@ -745,11 +871,15 @@ class OpenAIResponsesHandler(BaseTranslation):
)
)
if isinstance(input_data, str):
texts_to_check.append(input_data)
if not scan_only:
texts_to_check.append(input_data)
else:
for msg_idx, message in enumerate(input_data):
if role_out_of_guardrail_scope(
_input_item_role(message), skip_system_message=skip_system, skip_tool_message=False
_input_item_role(message),
skip_system_message=skip_system,
skip_tool_message=skip_tool,
scan_only_tool_results=scan_only,
):
continue
self._extract_input_text_and_images(
@ -775,12 +905,14 @@ class OpenAIResponsesHandler(BaseTranslation):
structured_messages: Sequence[AllMessageValues],
scoped_indices: Sequence[int],
scoped_structured_messages: Sequence[AllMessageValues] | None,
split_rows: Sequence[tuple[AllMessageValues, AllMessageValues]],
guardrail_to_apply: "CustomGuardrail",
guardrailed_inputs: GenericGuardrailAPIInputs,
) -> _RequestFields | None:
guardrailed: Final = guardrailed_inputs.get("structured_messages")
if guardrailed is None or guardrailed is scoped_structured_messages:
returned: Final = guardrailed_inputs.get("structured_messages")
if returned is None or returned is scoped_structured_messages:
return None
guardrailed: Final = _with_split_rows_restored(returned, split_rows)
covers_full_request: Final = len(scoped_indices) == len(structured_messages) or (
guardrail_to_apply.structured_messages_cover_full_request() and len(guardrailed) == len(structured_messages)
)
@ -814,11 +946,14 @@ class OpenAIResponsesHandler(BaseTranslation):
original_tools: Sequence[Mapping[str, object]],
flattened_tool_groups: Sequence[Sequence[Mapping[str, object]]],
guardrailed_tools: Sequence[ChatCompletionToolParam] | None,
*,
scan_only: bool,
) -> None:
if guardrailed_tools is None:
return
merge: Final = append_returned_tools if scan_only else merge_guardrailed_tools
data["tools"] = list( # rebind-ok: in-place request rewrite
merge_guardrailed_tools(original_tools, flattened_tool_groups, guardrailed_tools)
merge(original_tools, flattened_tool_groups, guardrailed_tools)
)
def _extract_input_text_and_images(
@ -839,6 +974,11 @@ class OpenAIResponsesHandler(BaseTranslation):
texts_to_check.append(summary)
task_mappings.append((msg_idx, None))
return
if message.get("type") in TOOL_CALL_OUTPUT_ITEM_TYPES:
for content_idx, text in tool_output_texts(message):
texts_to_check.append(text)
task_mappings.append((msg_idx, content_idx))
return
content: Final = message.get("content", None)
if content is None:
return
@ -890,18 +1030,20 @@ class OpenAIResponsesHandler(BaseTranslation):
)
continue
content = messages[msg_idx].get("content", None)
field = _scanned_text_field(messages[msg_idx])
content = messages[msg_idx].get(field, None)
if content is None:
continue
if isinstance(content, str) and content_idx_optional is None:
# Replace string content with guardrail response
messages[msg_idx]["content"] = guardrail_response
if content_idx_optional is None:
rewritten = _rewritten_whole_text(messages[msg_idx], field, guardrail_response)
if rewritten is not None:
messages[msg_idx][field] = rewritten
elif isinstance(content, list) and content_idx_optional is not None:
elif isinstance(content, list):
# Replace specific text item in list content
if isinstance(messages[msg_idx]["content"][content_idx_optional], dict):
messages[msg_idx]["content"][content_idx_optional]["text"] = guardrail_response
if isinstance(messages[msg_idx][field][content_idx_optional], dict):
messages[msg_idx][field][content_idx_optional]["text"] = guardrail_response
async def process_output_response(
self,

View file

@ -6,6 +6,7 @@ from typing import Final, TypeAlias
from pydantic import BaseModel, TypeAdapter, ValidationError
from litellm._logging import verbose_logger
from litellm.llms.base_llm.guardrail_translation.utils import merge_returned_tools_into_request_tools, openai_tool_name
from litellm.responses.litellm_completion_transformation.custom_tools import custom_tool_grammar_suffix
from litellm.responses.litellm_completion_transformation.transformation import (
NAMESPACE_DESCRIPTION_SEPARATOR,
@ -183,3 +184,18 @@ def merge_guardrailed_tools(
tuple(tool for key, tool in zip(guardrailed_keys, guardrailed) if key not in owned_keys)
)
return tuple(chain(merged_originals, appended))
def append_returned_tools(
original_tools: Sequence[Tool],
flattened_groups: Sequence[Sequence[Tool]],
returned_tools: Iterable[object],
) -> tuple[Tool, ...]:
request_chat_tools: Final = tuple(chain.from_iterable(flattened_groups))
additions: Final = merge_returned_tools_into_request_tools(
request_tools=request_chat_tools, returned_tools=_validated_tools(returned_tools), tool_name=openai_tool_name
)[len(request_chat_tools) :]
return (
*original_tools,
*LiteLLMCompletionResponsesConfig.transform_chat_completion_tool_params_to_responses_api_tools(additions),
)

View file

@ -66,6 +66,29 @@ _PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS: Final = frozenset({LlmProviders.AZURE,
_PROVIDERS_REPLAYING_ONLY_THEIR_OWN_REASONING: Final = _PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS
_ROLES_READING_PROMPT_CACHE_BREAKPOINTS: Final = frozenset({"system", "developer", "user"})
_PART_TYPES_READING_PROMPT_CACHE_BREAKPOINTS: Final = frozenset({"input_text", "input_image", "input_file"})
_COMPACT_TRACKING_PARAMS: Final = frozenset({"metadata", "user"})
_COMPACT_REJECTED_PARAMS: Final = frozenset(
{
"background",
"context_management",
"max_output_tokens",
"max_tool_calls",
"moderation",
"partial_images",
"prompt",
"safety_identifier",
"store",
"stream_options",
"temperature",
"tool_choice",
"top_logprobs",
"top_p",
"truncation",
}
)
_PROVIDERS_WITH_STRICT_COMPACT_PARAMS: Final = frozenset(
{LlmProviders.OPENAI, LlmProviders.AZURE, LlmProviders.AZURE_AI}
)
_INPUT_ITEM: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
_CONTENT_PARTS: Final[TypeAdapter[Sequence[Mapping[str, object]]]] = TypeAdapter(Sequence[Mapping[str, object]])
@ -1182,6 +1205,17 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
#########################################################
########## COMPACT RESPONSE API TRANSFORMATION ##########
#########################################################
def compact_url(self, api_base: str) -> str:
parsed_url: Final = httpx.URL(api_base)
return str(parsed_url.copy_with(path=parsed_url.path.rstrip("/") + "/compact"))
def compact_dropped_params(self, litellm_params: GenericLiteLLMParams) -> frozenset[str]:
if self.custom_llm_provider not in _PROVIDERS_WITH_STRICT_COMPACT_PARAMS:
return frozenset()
if _drops_params(litellm_params):
return _COMPACT_TRACKING_PARAMS | _COMPACT_REJECTED_PARAMS
return _COMPACT_TRACKING_PARAMS
def transform_compact_response_api_request(
self,
model: str,
@ -1197,10 +1231,8 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
OpenAI API expects the following request
- POST /v1/responses/compact
"""
# Preserve query params (e.g., api-version) while appending /compact.
parsed_url: Final = httpx.URL(api_base)
compact_path: Final = parsed_url.path.rstrip("/") + "/compact"
url: Final = str(parsed_url.copy_with(path=compact_path))
url: Final = self.compact_url(api_base)
dropped_params: Final = self.compact_dropped_params(litellm_params)
request_params: Final = cast( # cast-ok: the base signature leaves the params bag untyped
"Mapping[str, object]", response_api_optional_request_params
@ -1214,14 +1246,11 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
)
if sanitized_tools is not None:
response_api_optional_request_params["tools"] = sanitized_tools
data: Final = cast( # cast-ok: typed dict spread is inferred as Unknown by basedpyright
dict[str, object],
{
"model": model,
"input": replay_safe_input,
**response_api_optional_request_params,
},
)
data: Final = {
"model": model,
"input": replay_safe_input,
**{key: value for key, value in request_params.items() if key not in dropped_params},
}
return url, data

View file

@ -110,6 +110,15 @@ def create_config_class(provider: SimpleProviderConfig):
return super()._transform_messages(messages=transformed, model=model, is_async=True)
return super()._transform_messages(messages=transformed, model=model, is_async=False)
def _should_preserve_cache_control_for_endpoint(
self,
custom_llm_provider: str | None,
api_base: str | None,
) -> bool:
return provider.special_handling.get(
"preserve_cache_control", False
) or super()._should_preserve_cache_control_for_endpoint(custom_llm_provider, api_base)
def _get_openai_compatible_provider_info(
self, api_base: str | None, api_key: str | None
) -> tuple[str | None, str | None]:
@ -243,7 +252,7 @@ def create_config_class(provider: SimpleProviderConfig):
return JSONProviderConfig
_responses_config_cache: Final[dict] = {}
_responses_config_cache: Final[dict[str, "type[OpenAILikeResponsesConfig]"]] = {}
def create_responses_config_class(provider: SimpleProviderConfig) -> "type[OpenAILikeResponsesConfig]":

View file

@ -3,6 +3,7 @@ JSON-based provider configuration loader for OpenAI-compatible providers.
"""
import json
from collections.abc import Mapping
from pathlib import Path
from typing import Final
@ -12,7 +13,7 @@ from litellm._logging import verbose_logger
from litellm.types.utils import ProviderSpecificModelInfo
_PROVIDER_INFO_ADAPTER: Final = TypeAdapter(ProviderSpecificModelInfo)
_SPECIAL_HANDLING_ADAPTER: Final = TypeAdapter(dict[str, bool])
_SPECIAL_HANDLING_ADAPTER: Final = TypeAdapter(Mapping[str, bool])
class SimpleProviderConfig:
@ -21,12 +22,14 @@ class SimpleProviderConfig:
def __init__(self, slug: str, data: dict):
self.slug = slug
self.base_url = data["base_url"]
self.api_key_env = data["api_key_env"]
self.api_base_env = data.get("api_base_env")
self.api_key_env: str = data["api_key_env"]
self.api_base_env: str | None = data.get("api_base_env")
self.base_class = data.get("base_class", "openai_gpt")
self.param_mappings = data.get("param_mappings", {})
self.constraints = data.get("constraints", {})
self.special_handling: Final = _SPECIAL_HANDLING_ADAPTER.validate_python(data.get("special_handling", {}))
self.special_handling: Mapping[str, bool] = _SPECIAL_HANDLING_ADAPTER.validate_python(
data.get("special_handling", {})
)
self.supported_endpoints = data.get("supported_endpoints", [])
self.provider_info: ProviderSpecificModelInfo = (
_PROVIDER_INFO_ADAPTER.validate_python(data["provider_info"]) if "provider_info" in data else {}

View file

@ -270,6 +270,22 @@
"/v1/responses"
]
},
"streamlake": {
"base_url": "https://vanchin.streamlake.ai/api/gateway/v1/endpoints",
"api_key_env": "STREAMLAKE_API_KEY",
"api_base_env": "STREAMLAKE_API_BASE",
"param_mappings": {
"max_tokens": "max_completion_tokens"
},
"provider_info": {
"supports_function_calling": true,
"supports_reasoning": true
},
"special_handling": {
"preserve_cache_control": true
},
"supported_endpoints": ["/v1/chat/completions"]
},
"vispark": {
"base_url": "https://api.lab.vispark.in/v1",
"api_key_env": "VISPARK_LAB_API_KEY",

View file

@ -41939,6 +41939,120 @@
"supports_system_messages": true,
"supports_native_structured_output": true
},
"bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 4.12e-08,
"input_cost_per_token_batches": 2.06e-08,
"input_cost_per_token_flex": 2.06e-08,
"input_cost_per_token_priority": 7.21e-08,
"litellm_provider": "bedrock",
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 4.12e-08,
"output_cost_per_token_batches": 2.06e-08,
"output_cost_per_token_flex": 2.06e-08,
"output_cost_per_token_priority": 7.21e-08,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json",
"supports_audio_input": true,
"supports_system_messages": true,
"supports_native_structured_output": true
},
"bedrock/ap-southeast-2/mistral.voxtral-small-24b-2507": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 1.03e-07,
"input_cost_per_token_batches": 5.15e-08,
"input_cost_per_token_flex": 5.15e-08,
"input_cost_per_token_priority": 1.8025e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 3.09e-07,
"output_cost_per_token_batches": 1.545e-07,
"output_cost_per_token_flex": 1.545e-07,
"output_cost_per_token_priority": 5.4075e-07,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json",
"supports_audio_input": true,
"supports_system_messages": true,
"supports_native_structured_output": true
},
"bedrock/ap-northeast-1/mistral.voxtral-mini-3b-2507": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 5e-08,
"input_cost_per_token_batches": 3e-08,
"input_cost_per_token_flex": 3e-08,
"input_cost_per_token_priority": 9e-08,
"litellm_provider": "bedrock",
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 5e-08,
"output_cost_per_token_batches": 3e-08,
"output_cost_per_token_flex": 3e-08,
"output_cost_per_token_priority": 9e-08,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-northeast-1/index.json",
"supports_audio_input": true,
"supports_system_messages": true,
"supports_native_structured_output": true
},
"bedrock/ap-northeast-1/mistral.voxtral-small-24b-2507": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 1.2e-07,
"input_cost_per_token_batches": 6e-08,
"input_cost_per_token_flex": 6e-08,
"input_cost_per_token_priority": 2.1e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 3.6e-07,
"output_cost_per_token_batches": 1.8e-07,
"output_cost_per_token_flex": 1.8e-07,
"output_cost_per_token_priority": 6.3e-07,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-northeast-1/index.json",
"supports_audio_input": true,
"supports_system_messages": true,
"supports_native_structured_output": true
},
"bedrock/eu-west-2/mistral.voxtral-mini-3b-2507": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 6e-08,
"input_cost_per_token_batches": 3e-08,
"input_cost_per_token_flex": 3e-08,
"input_cost_per_token_priority": 1.1e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 6e-08,
"output_cost_per_token_batches": 3e-08,
"output_cost_per_token_flex": 3e-08,
"output_cost_per_token_priority": 1.1e-07,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/eu-west-2/index.json",
"supports_audio_input": true,
"supports_system_messages": true,
"supports_native_structured_output": true
},
"bedrock/eu-west-2/mistral.voxtral-small-24b-2507": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 1.6e-07,
"input_cost_per_token_batches": 8e-08,
"input_cost_per_token_flex": 8e-08,
"input_cost_per_token_priority": 2.8e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 4.7e-07,
"output_cost_per_token_batches": 2.3e-07,
"output_cost_per_token_flex": 2.3e-07,
"output_cost_per_token_priority": 8.2e-07,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/eu-west-2/index.json",
"supports_audio_input": true,
"supports_system_messages": true,
"supports_native_structured_output": true
},
"mistral/codestral-2508": {
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 3e-07,
@ -61374,6 +61488,359 @@
"supports_tool_choice": true,
"supports_vision": true
},
"streamlake/deepseek-v4-flash-0731": {
"cache_read_input_token_cost": 1.4e-08,
"input_cost_per_token": 4.4e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 1048576,
"max_output_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 1.32e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/DeepSeek-V4-Pro-0813": {
"cache_read_input_token_cost": 4.4e-08,
"input_cost_per_token": 1.32e-06,
"litellm_provider": "streamlake",
"max_input_tokens": 1048566,
"max_output_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 3.96e-06,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/DeepSeek-V3": {
"cache_read_input_token_cost": 1.14e-07,
"input_cost_per_token": 2.86e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 56000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 1.143e-06,
"supports_function_calling": true,
"supports_reasoning": false,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/GLM-5.3": {
"cache_read_input_token_cost": 2.6e-07,
"input_cost_per_token": 1.4e-06,
"litellm_provider": "streamlake",
"max_input_tokens": 1048576,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 4.4e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_assistant_prefill": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/GLM-5.3-Flash": {
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 1048576,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 5e-07,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_assistant_prefill": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/Kimi-K2-Instruct-0905": {
"cache_read_input_token_cost": 1.43e-07,
"input_cost_per_token": 5.71e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 224000,
"max_output_tokens": 32000,
"mode": "chat",
"output_cost_per_token": 2.286e-06,
"supports_function_calling": true,
"supports_reasoning": false,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/deepseek-r1": {
"cache_read_input_token_cost": 2.29e-07,
"input_cost_per_token": 5.71e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 98304,
"max_output_tokens": 16000,
"mode": "chat",
"output_cost_per_token": 2.286e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/deepseek-v3.1": {
"cache_read_input_token_cost": 2.29e-07,
"input_cost_per_token": 5.71e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 98304,
"max_output_tokens": 32000,
"mode": "chat",
"output_cost_per_token": 1.714e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/deepseek-v4-flash": {
"cache_read_input_token_cost": 2.8e-08,
"input_cost_per_token": 1.4e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 1048576,
"max_output_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 2.8e-07,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/deepseek-v4-pro": {
"cache_read_input_token_cost": 1.45e-07,
"input_cost_per_token": 1.74e-06,
"litellm_provider": "streamlake",
"max_input_tokens": 1048576,
"max_output_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 3.48e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/deepseek-v4.1-flash-1017": {
"cache_read_input_token_cost": 6e-09,
"input_cost_per_token": 3e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 1048576,
"max_output_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_vision": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/deepseek-v4.1-flash-260910": {
"cache_read_input_token_cost": 6e-09,
"input_cost_per_token": 3e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 1048568,
"max_output_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_vision": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/glm-5": {
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 1e-06,
"litellm_provider": "streamlake",
"max_input_tokens": 202752,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3.2e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/glm-5.1": {
"cache_read_input_token_cost": 2.6e-07,
"input_cost_per_token": 1.4e-06,
"litellm_provider": "streamlake",
"max_input_tokens": 202752,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 4.4e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/glm-5.2": {
"cache_read_input_token_cost": 2.6e-07,
"input_cost_per_token": 1.4e-06,
"litellm_provider": "streamlake",
"max_input_tokens": 1048566,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 4.4e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/kimi-k2.6": {
"cache_read_input_token_cost": 1.6e-07,
"input_cost_per_token": 9.5e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 262144,
"max_output_tokens": 256000,
"mode": "chat",
"output_cost_per_token": 4e-06,
"supports_function_calling": true,
"supports_vision": true,
"supports_reasoning": false,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/kimi-k2.7-code": {
"cache_read_input_token_cost": 1.9e-07,
"input_cost_per_token": 9.5e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 262144,
"max_output_tokens": 32000,
"mode": "chat",
"output_cost_per_token": 4e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_vision": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/mimo-v2.6-flash": {
"cache_read_input_token_cost": 2.8e-09,
"input_cost_per_token": 1.4e-07,
"litellm_provider": "streamlake",
"mode": "chat",
"output_cost_per_token": 2.8e-07,
"supports_vision": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/mimo-v2.6-pro": {
"cache_read_input_token_cost": 3.6e-09,
"input_cost_per_token": 4.35e-07,
"litellm_provider": "streamlake",
"mode": "chat",
"output_cost_per_token": 8.7e-07,
"supports_vision": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/minimax-m2.5": {
"cache_creation_input_token_cost": 3.75e-07,
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 3e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 196608,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/minimax-m3": {
"cache_read_input_token_cost": 1.2e-07,
"cache_read_input_token_cost_above_512k_tokens": 2.4e-07,
"input_cost_per_token": 6e-07,
"input_cost_per_token_above_512k_tokens": 1.2e-06,
"litellm_provider": "streamlake",
"max_input_tokens": 1000000,
"max_output_tokens": 512000,
"mode": "chat",
"output_cost_per_token": 2.4e-06,
"output_cost_per_token_above_512k_tokens": 4.8e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_vision": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/qwen3.5-397b-a17b": {
"cache_read_input_token_cost": 1.2e-07,
"input_cost_per_token": 6e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 258048,
"max_output_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 3.6e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_vision": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/qwen3.6-27b": {
"input_cost_per_token": 6e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 229376,
"max_output_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 3.6e-06,
"supports_function_calling": true,
"supports_vision": true,
"supports_reasoning": false,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/qwen3.6-35b-a3b": {
"input_cost_per_token": 3.75e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 229376,
"max_output_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 2.25e-06,
"supports_function_calling": true,
"supports_vision": true,
"supports_reasoning": false,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/qwen3.8-2.4t-a95b": {
"cache_read_input_token_cost": 2.5e-07,
"input_cost_per_token": 2e-06,
"litellm_provider": "streamlake",
"max_input_tokens": 983616,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 6e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_vision": true,
"cache_creation_input_token_cost": 2.5e-06,
"explicit_cache_read_input_token_cost": 1.7e-07,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/qwen3.8-27b": {
"cache_read_input_token_cost": 1e-07,
"input_cost_per_token": 5e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 983616,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_vision": true,
"cache_creation_input_token_cost": 6.25e-07,
"explicit_cache_read_input_token_cost": 5e-08,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"deepseek-v4-flash": {
"cache_creation_input_token_cost": 0.0,
"cache_read_input_token_cost": 6e-09,
@ -62689,6 +63156,14 @@
"litellm_provider": "anthropic"
}
},
{
"name": "typesafe-jev-ids",
"pattern": "^jev-(?:latest|preview|1\\.13\\.0|1\\.14\\.0)$",
"description": "A bare Jev id that has a typesafe/ price entry (jev-latest, jev-preview, jev-1.13.0, jev-1.14.0), the name the /typesafe pass-through forwards to TypeSafe. Listed by name, matched case-insensitively like every rule here, so foo/jev-latest or a model group named like a Jev id TypeSafe does not serve, such as jev-1.0 or jev-internal-router, never resolves to typesafe. A new Jev version goes here together with its price entry. Provider inference feeds the proxy's provider/* wildcard access checks, so this rule is what lets typesafe/* on a key, team or project allowlist cover a jev-latest request.",
"model_info": {
"litellm_provider": "typesafe"
}
},
{
"name": "claude-family-baseline",
"pattern": "claude-[a-z]+-\\d+(?:[-.]\\d+)?(?:-\\d{8})?",

View file

@ -3785,6 +3785,32 @@
"generateContent": true
}
},
"streamlake": {
"display_name": "StreamLake (`streamlake`)",
"url": "https://docs.litellm.ai/docs/providers/openai_compatible",
"endpoints": {
"chat_completions": true,
"responses": true,
"text_completion": true,
"embeddings": false,
"image_generations": false,
"image_edits": false,
"audio_transcriptions": false,
"audio_speech": false,
"rerank": false,
"ocr": false,
"search": false,
"moderations": false,
"realtime": false,
"video_generations": false,
"batches": false,
"messages": true,
"interactions": true,
"systemone": false,
"a2a": true,
"generateContent": true
}
},
"tencent": {
"display_name": "Tencent TokenHub (`tencent`)",
"url": "https://docs.litellm.ai/docs/providers/tencent",

View file

@ -8075,7 +8075,6 @@ async def vector_store_access_check(
rag_ids: Final = (rag_vector_store_id,) if rag_vector_store_id is not None else ()
vector_store_ids_to_run: Final = tuple(dict.fromkeys((*registry_ids, *rag_ids)))
if not vector_store_ids_to_run:
verbose_proxy_logger.debug("Vector store to run not found, skipping vector store access check")
return True
#########################################################

View file

@ -2581,7 +2581,6 @@ class ProxyBaseLLMRequestProcessing:
# have mutated `self.data` in place, and the audit-trail snapshot taken in
# add_litellm_data_to_request predates that mutation.
refresh_proxy_server_request_body_snapshot(self.data, guardrails_applied=True)
verbose_proxy_logger.debug("receiving data: %s", self.data)
if "messages" in self.data and self.data["messages"]:
logging_obj.update_messages(self.data["messages"])

View file

@ -845,5 +845,3 @@ def _add_vector_store_id_from_path(request_data: dict, request: Request) -> None
"populate_request_with_path_params: Updated request_data with vector_store_ids=%s",
request_data.get("vector_store_ids"),
)
else:
verbose_proxy_logger.debug("populate_request_with_path_params: No vector_store_id present in path=%s", path)

View file

@ -62,6 +62,20 @@ def is_non_conversational_call_type(call_type: str) -> bool:
return call_type in NON_CONVERSATIONAL_CALL_TYPES
_PASSTHROUGH_EMBEDDINGS_ENDPOINT: Final = re.compile(r"/?(?:[\w-]+/)*embeddings/?")
def is_non_conversational_request(call_type: str, data: Mapping[str, object]) -> bool:
if is_non_conversational_call_type(call_type):
return True
endpoint: Final = data.get("endpoint")
return (
call_type == "allm_passthrough_route"
and isinstance(endpoint, str)
and _PASSTHROUGH_EMBEDDINGS_ENDPOINT.fullmatch(endpoint) is not None
)
TEXT_PART_TYPES: Final[frozenset[str]] = frozenset(
{"text", "input_text", "output_text", "summary_text", "reasoning_text"}
)

View file

@ -49,6 +49,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, bedrock_bearer_token, run_aws_signing
from litellm.llms.bedrock.guardrail_attachments import find_request_attachments
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
get_async_httpx_client,
httpxSpecialProvider,
)
@ -286,9 +287,12 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
streaming_sampling_rate: int | None = None,
streaming_end_of_stream_only: bool | None = None,
streaming_buffer_release_on_scan: bool | None = None,
async_handler: AsyncHTTPHandler | None = None,
**kwargs,
):
self.async_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.GuardrailCallback)
self.async_handler = async_handler or get_async_httpx_client(
llm_provider=httpxSpecialProvider.GuardrailCallback
)
self._set_streaming_params(
BedrockGuardrailStreamingParams.from_extras(
MappingProxyType(

View file

@ -6,20 +6,19 @@ model_list:
model: gpt-4
api_key: os.environ/OPENAI_API_KEY
litellm_settings:
guardrails:
- guardrail_name: "my-generic-guardrail"
litellm_params:
guardrail: generic_guardrail_api
mode: pre_call # Options: pre_call, post_call, during_call, [pre_call, post_call]
api_key: os.environ/GENERIC_GUARDRAIL_API_KEY # Optional if using Bearer auth
api_base: http://localhost:8080 # Required. Endpoint /beta/litellm_basic_guardrail_api is automatically appended
unreachable_fallback: fail_closed # Options: fail_closed (default, raise), fail_open (proceed if endpoint unreachable or upstream returns 502/503/504)
default_on: false # Set to true to apply to all requests by default
additional_provider_specific_params:
# Any additional parameters your guardrail API needs
api_version: "v1"
custom_param: "value"
guardrails:
- guardrail_name: "my-generic-guardrail"
litellm_params:
guardrail: generic_guardrail_api
mode: pre_call # Options: pre_call, post_call, during_call, [pre_call, post_call]
api_key: os.environ/GENERIC_GUARDRAIL_API_KEY # Optional if using Bearer auth
api_base: http://localhost:8080 # Required. Endpoint /beta/litellm_basic_guardrail_api is automatically appended
unreachable_fallback: fail_closed # Options: fail_closed (default, raise), fail_open (proceed if endpoint unreachable or upstream returns 502/503/504)
default_on: false # Set to true to apply to all requests by default
additional_provider_specific_params:
# Any additional parameters your guardrail API needs
api_version: "v1"
custom_param: "value"
# Usage examples:

View file

@ -30,6 +30,7 @@ from litellm.integrations.custom_guardrail import (
from litellm.llms.base_llm.guardrail_translation.utils import (
effective_scan_only_tool_results_for_guardrail,
effective_skip_system_message_for_guardrail,
effective_skip_tool_message_for_guardrail,
role_out_of_guardrail_scope,
)
from litellm.llms.custom_httpx.http_handler import (
@ -37,12 +38,16 @@ from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
)
from litellm.llms.openai.responses.guardrail_translation.handler import scannable_instructions
from litellm.llms.openai.responses.guardrail_translation.handler import (
scannable_instructions,
tool_output_texts,
)
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.common_utils.callback_utils import (
add_guardrail_scan_id,
add_guardrail_to_applied_guardrails_header,
)
from litellm.responses.litellm_completion_transformation.transformation import TOOL_CALL_OUTPUT_ITEM_TYPES
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.openai import AllMessageValues
@ -112,12 +117,16 @@ class _ResponsesInputItem(LiteLLMBaseModel):
type: str | None = None
role: str | None = None
content: str | tuple[_ResponsesContentPart, ...] | None = None
output: object = None
call_id: object = None
def text_count(self, *, skip_system: bool) -> int:
if role_out_of_guardrail_scope(
(self.role or "").lower(), skip_system_message=skip_system, skip_tool_message=False
):
def text_count(self, *, skip_system: bool, skip_tool: bool) -> int:
is_tool_output: Final = self.type in TOOL_CALL_OUTPUT_ITEM_TYPES
role: Final = "tool" if is_tool_output else (self.role or "").lower()
if role_out_of_guardrail_scope(role, skip_system_message=skip_system, skip_tool_message=skip_tool):
return 0
if is_tool_output:
return len(tool_output_texts(self))
if isinstance(self.content, str):
return 1
if self.content is None:
@ -1692,13 +1701,14 @@ class PanwPrismaAirsHandler(CustomGuardrail):
request_data: Mapping[str, object],
*,
skip_system: bool,
skip_tool: bool,
) -> frozenset[int] | None:
"""Return the ``texts`` indices flattened from Responses ``reasoning`` input items.
The Responses translation handler gives those model-authored items the default
``user`` role, so the latest-turn selection must not mistake one for a human turn.
Empty for requests without a Responses ``input`` item list; None when the raw items
(after the leading ``instructions`` text, both minus whatever ``skip_system`` drops)
(after the leading ``instructions`` text, both minus whatever ``skip_system`` and ``skip_tool`` drop)
do not account for every entry of ``texts``.
"""
try:
@ -1708,7 +1718,7 @@ class PanwPrismaAirsHandler(CustomGuardrail):
if not isinstance(raw_input, tuple):
return frozenset()
offset: Final = 0 if scannable_instructions(request_data, skip_system=skip_system) is None else 1
counts: Final = tuple(item.text_count(skip_system=skip_system) for item in raw_input)
counts: Final = tuple(item.text_count(skip_system=skip_system, skip_tool=skip_tool) for item in raw_input)
if offset + sum(counts) != len(texts):
return None
starts: Final = itertools.accumulate(counts, initial=offset)
@ -1738,7 +1748,10 @@ class PanwPrismaAirsHandler(CustomGuardrail):
if sources is None:
return None
reasoning: Final = self._reasoning_item_text_indices(
texts, request_data, skip_system=effective_skip_system_message_for_guardrail(self)
texts,
request_data,
skip_system=effective_skip_system_message_for_guardrail(self),
skip_tool=effective_skip_tool_message_for_guardrail(self),
)
if reasoning is None:
return None

View file

@ -3024,13 +3024,7 @@ async def add_litellm_data_to_request(
getattr(request.state, "litellm_roi_estimator", False) is True
)
verbose_proxy_logger.debug(
"[PROXY] returned data from litellm_pre_call_utils: %s",
data,
)
# Team/Project credential overrides from model_config
# Placed after the debug log to avoid leaking credential secrets in logs
_apply_credential_overrides_from_model_config(
data=data,
user_api_key_dict=user_api_key_dict,

View file

@ -50,6 +50,7 @@ from litellm.constants import (
from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
from litellm.integrations.custom_guardrail import guardrail_request_data_with_streaming, is_guardrail_intervention
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
from litellm.litellm_core_utils.fallback_generalizations import match_routing_generalization
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, merge_anthropic_beta_headers
from litellm.llms.azure.passthrough.transformation import (
foreign_azure_deployment,
@ -698,6 +699,20 @@ async def typesafe_proxy_route(
return await endpoint_func(request, fastapi_response, user_api_key_dict)
_COST_MAP_ENTRIES: Final = TypeAdapter(Mapping[str, object])
def typesafe_allowlist_model_names() -> tuple[str, ...]:
if not passthrough_endpoint_router.has_credentials(custom_llm_provider="typesafe"):
return ()
cost_map: Final = _COST_MAP_ENTRIES.validate_python(litellm.model_cost) # pyright: ignore[reportUnknownMemberType] # legacy registry is validated at this boundary
priced_names: Final = (name.removeprefix("typesafe/") for name in cost_map if name.startswith("typesafe/"))
jev_models_typesafe_serves: Final = sorted(
name for name in priced_names if match_routing_generalization(name) == "typesafe"
)
return ("typesafe/*", *jev_models_typesafe_serves)
@router.post(
"/laya/v1/systemone",
tags=["Laya Pass-through", "pass-through"],

View file

@ -1,4 +1,5 @@
import json
import os
from collections.abc import Callable
from typing import TYPE_CHECKING, Final
@ -88,6 +89,14 @@ class PassthroughEndpointRouter:
)
return _resolve_oidc_reference(get_secret_str(_env_variable_name))
def has_credentials(self, custom_llm_provider: str) -> bool:
if self._get_deployment_api_key(custom_llm_provider=custom_llm_provider, region_name=None) is not None:
return True
env_variable_name: Final = self._get_default_env_variable_name_passthrough_endpoint(
custom_llm_provider=custom_llm_provider,
)
return bool(os.environ.get(env_variable_name))
def _get_deployment_api_key(
self,
custom_llm_provider: str,

View file

@ -12390,6 +12390,12 @@ async def model_list(
hidden_names: Final = blocked_names | unhealthy_names
from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import typesafe_allowlist_model_names
asks_for_allowlist_names: Final = (
include_model_access_groups is True and return_wildcard_routes is True and only_model_access_groups is not True
)
# If scope=expand and user has admin privileges, return all proxy models
if should_expand_scope:
# Get all proxy models as if user is a proxy admin
@ -12410,7 +12416,10 @@ async def model_list(
all_models = get_complete_model_list(
key_models=[],
team_models=[],
proxy_model_list=proxy_model_list,
proxy_model_list=[
*proxy_model_list,
*(typesafe_allowlist_model_names() if asks_for_allowlist_names else ()),
],
user_model=None,
infer_model_from_keys=False,
return_wildcard_routes=include_wildcard_routes,
@ -12471,7 +12480,7 @@ async def model_list(
# Otherwise, use the normal behavior (current implementation)
# Get available models for the user
all_models = await get_available_models_for_user(
available_models: Final = await get_available_models_for_user(
user_api_key_dict=user_api_key_dict,
llm_router=llm_router,
general_settings=general_settings,
@ -12484,6 +12493,11 @@ async def model_list(
return_wildcard_routes=include_wildcard_routes,
user_api_key_cache=user_api_key_cache,
)
offers_typesafe_allowlist_names: Final = (
asks_for_allowlist_names and team_id is None and user_api_key_has_admin_view(user_api_key_dict)
)
typesafe_names_to_offer: Final = typesafe_allowlist_model_names() if offers_typesafe_allowlist_names else ()
all_models = [*available_models, *(name for name in typesafe_names_to_offer if name not in available_models)]
undiscoverable_names: Final = undiscoverable_model_names(
all_models, llm_router, user_api_key_dict, team_id or user_api_key_dict.team_id
@ -18437,7 +18451,6 @@ async def async_queue_request(
"body": copy.copy(data), # use copy instead of deepcopy
}
verbose_proxy_logger.debug("receiving data: %s", data)
data["model"] = (
general_settings.get("completion_model", None) # server default
or user_model # model name passed via cli args

View file

@ -3519,6 +3519,34 @@
],
"default_model_placeholder": "inferenceapis/deepseek-ai/DeepSeek-V4-Flash"
},
{
"provider": "StreamLake",
"provider_display_name": "StreamLake",
"litellm_provider": "streamlake",
"credential_fields": [
{
"key": "api_base",
"label": "API Base",
"placeholder": "https://vanchin.streamlake.ai/api/gateway/v1/endpoints",
"tooltip": null,
"required": false,
"field_type": "text",
"options": null,
"default_value": null
},
{
"key": "api_key",
"label": "API Key",
"placeholder": null,
"tooltip": null,
"required": true,
"field_type": "password",
"options": null,
"default_value": null
}
],
"default_model_placeholder": "streamlake/<model-or-endpoint>"
},
{
"provider": "Snowflake",
"provider_display_name": "Snowflake",

View file

@ -321,7 +321,7 @@ def _responses_file_part(item: Mapping[str, object]) -> _ResponsesFilePart:
)
def _decode_responses_media(item: Mapping[str, object]) -> _ResponsesMediaPart | None:
def decode_responses_media(item: Mapping[str, object]) -> _ResponsesMediaPart | None:
kind: Final = item.get("type")
if kind in ("input_image", "image_url", "computer_screenshot") and any(key in item for key in ("image_url", "url")):
image: Final = _responses_image_part(item)
@ -383,6 +383,52 @@ def _attribute_fields(value: object) -> dict[str, object]:
return dict(cast("Iterable[tuple[str, object]]", value)) # cast-ok: dict() raises on non-pair values, as before
def tool_output_to_chat_content(
output: object,
) -> str | list[ToolMessageContentPart]:
if output is None:
return ""
if isinstance(output, str):
return output
single_media: Final = decode_responses_media(output) if isinstance(output, Mapping) else None
if single_media is not None:
return [single_media]
if isinstance(output, list):
normalized_blocks: Final[list[ToolMessageContentPart]] = []
text_acc: Final[list[str]] = []
for part in output:
if not isinstance(part, dict):
continue
part_type = part.get("type")
if part_type in ("input_text", "output_text", "text"):
txt = part.get("text")
if isinstance(txt, str) and txt:
text_acc.append(txt)
normalized_blocks.append({"type": "text", "text": txt})
elif (media := decode_responses_media(part)) is not None:
normalized_blocks.append(media)
if any(b.get("type") != "text" for b in normalized_blocks):
return normalized_blocks
if text_acc:
return "".join(text_acc)
try:
return json.dumps(output)
except Exception:
return str(output)
try:
return json.dumps(output)
except Exception:
return str(output)
TOOL_CALL_OUTPUT_ITEM_TYPES: Final = frozenset(
{"function_call_output", "custom_tool_call_output", "computer_call_output", "tool_result"}
)
def _input_item_role(input_item: Mapping[str, object]) -> str:
return cast(str, input_item.get("role") or "user") # cast-ok: client-supplied role forwarded verbatim, unvalidated
@ -2371,12 +2417,7 @@ class LiteLLMCompletionResponsesConfig:
"""
Check if the input item is a tool call output
"""
return input_item.get("type") in [
"function_call_output",
"custom_tool_call_output",
"computer_call_output",
"tool_result", # Anthropic/MCP format
]
return input_item.get("type") in TOOL_CALL_OUTPUT_ITEM_TYPES
@staticmethod
def _is_input_item_function_call(input_item: Mapping[str, object]) -> bool:
@ -2400,58 +2441,7 @@ class LiteLLMCompletionResponsesConfig:
if not call_id:
return []
def _normalize_function_call_output_to_tool_content(
output: object,
) -> str | list[ToolMessageContentPart]:
"""
Preserve structured media for downstream chat adapters and token estimation.
Text-only outputs retain their existing string representation.
"""
if output is None:
return ""
if isinstance(output, str):
return output
single_media: Final = _decode_responses_media(output) if isinstance(output, Mapping) else None
if single_media is not None:
return [single_media]
# Some adapters represent tool output as a list of "input_*" parts
if isinstance(output, list):
normalized_blocks: Final[list[ToolMessageContentPart]] = []
text_acc: Final[list[str]] = []
for part in output:
if not isinstance(part, dict):
continue
part_type = part.get("type")
if part_type in ("input_text", "output_text", "text"):
txt = part.get("text")
if isinstance(txt, str) and txt:
text_acc.append(txt)
normalized_blocks.append({"type": "text", "text": txt})
elif (media := _decode_responses_media(part)) is not None:
normalized_blocks.append(media)
if any(b.get("type") != "text" for b in normalized_blocks):
return normalized_blocks
if text_acc:
return "".join(text_acc)
try:
# last resort: keep something meaningful for providers that require a string
import json as _json
return _json.dumps(output)
except Exception:
return str(output)
# Fallback for dict/number/etc.
try:
import json as _json
return _json.dumps(output)
except Exception:
return str(output)
tool_content: Final = _normalize_function_call_output_to_tool_content(tool_call_output.get("output"))
tool_content: Final = tool_output_to_chat_content(tool_call_output.get("output"))
output_cache_control: Final = _input_item_cache_control(tool_call_output)
tool_output_message: Final = (
ChatCompletionToolMessage(role="tool", content=tool_content, tool_call_id=str(call_id))
@ -2606,7 +2596,7 @@ class LiteLLMCompletionResponsesConfig:
content_list.append(item)
elif isinstance(item, dict):
if (
media := _decode_responses_media(
media := decode_responses_media(
fields := _validate_responses_media(_STR_KEY_DICT_ADAPTER, item)
)
) is not None:

View file

@ -39,7 +39,11 @@ from litellm.constants import (
)
from litellm.exceptions import MidStreamFallbackError, RateLimitError
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.core_helpers import process_response_headers
from litellm.litellm_core_utils.core_helpers import (
process_response_headers,
provider_reported_usage_cost,
set_response_cost_in_hidden_params,
)
from litellm.litellm_core_utils.exception_mapping_utils import with_deployment_api_base
from litellm.litellm_core_utils.internal_call_metadata import (
is_background_response,
@ -1825,20 +1829,32 @@ def _estimate_usage_safely(
def stamp_responses_usage_cost(
response_obj: ResponsesAPIResponse | None, logging_obj: LiteLLMLoggingObj | None
) -> None:
if response_obj is None or logging_obj is None:
"""The terminal response's ``usage.cost`` is the bill litellm writes to the spend row. It is priced on a copy
shaped the way the success handler shapes the spend row's result (the reported ``usage.cost`` in the
response-cost header, the usage in chat shape), so the client and the row read the same number."""
if response_obj is None:
return
usage_obj: Final[ResponseAPIUsage | None] = _usage_as_model(getattr(response_obj, "usage", None))
if usage_obj is None:
return
response_obj.usage = usage_obj # rebind-ok: the stamped cost has to ride on the response the client receives
if isinstance(getattr(usage_obj, "cost", None), (int, float)):
response_obj.usage = usage_obj # rebind-ok: the typed usage has to ride on the response the client receives
if logging_obj is None:
return
priced: Final = response_obj.model_copy(
update={"usage": ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(usage_obj)}
)
priced.hidden_params = {}
set_response_cost_in_hidden_params(
priced, provider_reported_usage_cost(usage_obj, logging_obj.model_call_details.get("custom_llm_provider"))
)
try:
cost: Final[float | None] = logging_obj.response_cost_calculator(result=response_obj)
except Exception:
bill: Final = logging_obj.response_cost_calculator(result=priced)
except Exception: # noqa: BLE001 # a pricing failure leaves the client usage as the upstream sent it
return
if isinstance(cost, (int, float)) and cost > 0:
setattr(usage_obj, "cost", cost)
if isinstance(bill, bool) or not isinstance(bill, (int, float)):
return
if bill > 0 or isinstance(getattr(usage_obj, "cost", None), (int, float)):
setattr(usage_obj, "cost", float(bill))
def opening_item_payload(payload: Mapping[str, object]) -> Mapping[str, object]:

View file

@ -766,6 +766,12 @@ class ResponsesAPIRequestUtils:
model_id=model_id,
response_id=decoded_response_id,
)
except ValueError:
return DecodedResponseId(
custom_llm_provider=None,
model_id=None,
response_id=response_id,
)
except Exception as e:
verbose_logger.debug("Error decoding response_id '%s': %s", response_id, e)
return DecodedResponseId(

View file

@ -4921,6 +4921,7 @@ class Router:
deployment=deployment, forwarded_keys=kwargs.pop(_ALIAS_MARKER_FORWARDED_PARAMS_KWARG, ())
):
kwargs.pop(key, None)
kwargs.pop("daily_reset_time", None) # pyright: ignore[reportUnknownMemberType] # kwargs is an untyped dict
defaults: Final = _LITELLM_PARAMS_ADAPTER.validate_python(self.default_litellm_params)
self._merge_tools_from_deployment(deployment=deployment, kwargs=kwargs, defaults=defaults)
@ -5668,7 +5669,7 @@ class Router:
def _image_generation(self, prompt: str, model: str, **kwargs):
model_name: Final = ""
try:
verbose_router_logger.debug("Inside _image_generation()- model: %s; kwargs: %s", model, kwargs)
verbose_router_logger.debug("Inside _image_generation()- model: %s", model)
deployment: Final = self.get_available_deployment(
model=model,
messages=[{"role": "user", "content": "prompt"}],
@ -5730,7 +5731,7 @@ class Router:
async def _aimage_generation(self, prompt: str, model: str, **kwargs):
model_name = model
try:
verbose_router_logger.debug("Inside _image_generation()- model: %s; kwargs: %s", model, kwargs)
verbose_router_logger.debug("Inside _image_generation()- model: %s", model)
parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs)
deployment: Final = await self.async_get_available_deployment(
model=model,
@ -5815,7 +5816,7 @@ class Router:
async def _atranscription(self, file: FileTypes, model: str, **kwargs):
model_name: Final = model
try:
verbose_router_logger.debug("Inside _atranscription()- model: %s; kwargs: %s", model, kwargs)
verbose_router_logger.debug("Inside _atranscription()- model: %s", model)
parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs)
deployment: Final = await self.async_get_available_deployment(
model=model,
@ -5908,7 +5909,7 @@ class Router:
async def _aspeech(self, model: str, input: str, voice: str | None = None, **kwargs):
model_name: Final = model
try:
verbose_router_logger.debug("Inside _aspeech()- model: %s; kwargs: %s", model, kwargs)
verbose_router_logger.debug("Inside _aspeech()- model: %s", model)
parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs)
deployment: Final = await self.async_get_available_deployment(
model=model,
@ -5970,7 +5971,7 @@ class Router:
async def _arerank(self, model: str, **kwargs):
model_name = None
try:
verbose_router_logger.debug("Inside _rerank()- model: %s; kwargs: %s", model, kwargs)
verbose_router_logger.debug("Inside _rerank()- model: %s", model)
deployment: Final = await self.async_get_available_deployment(
model=model,
specific_deployment=kwargs.pop("specific_deployment", None),
@ -6079,7 +6080,7 @@ class Router:
async def _atext_completion(self, model: str, prompt: str, **kwargs):
try:
verbose_router_logger.debug("Inside _atext_completion()- model: %s; kwargs: %s", model, kwargs)
verbose_router_logger.debug("Inside _atext_completion()- model: %s", model)
parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs)
deployment: Final = await self.async_get_available_deployment(
model=model,
@ -6149,7 +6150,7 @@ class Router:
async def _aadapter_completion(self, adapter_id: str, model: str, **kwargs):
try:
verbose_router_logger.debug("Inside _aadapter_completion()- model: %s; kwargs: %s", model, kwargs)
verbose_router_logger.debug("Inside _aadapter_completion()- model: %s", model)
parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs)
deployment: Final = await self.async_get_available_deployment(
model=model,
@ -6243,7 +6244,7 @@ class Router:
kwargs=kwargs,
metadata_variable_name="litellm_metadata",
)
verbose_router_logger.debug("Inside aguardrail() - guardrail_name: %s; kwargs: %s", guardrail_name, kwargs)
verbose_router_logger.debug("Inside aguardrail() - guardrail_name: %s", guardrail_name)
response: Final = await self.async_function_with_fallbacks(**kwargs)
return response
@ -6324,9 +6325,7 @@ class Router:
controls: Final = per_request_fallback_controls(kwargs)
kwargs[MID_STREAM_FALLBACK_CONTROLS_KEY] = controls # rebind-ok: forwarded to every hop
self._update_kwargs_before_fallbacks(model=model, kwargs=kwargs, metadata_variable_name="litellm_metadata")
verbose_router_logger.debug(
"Inside ageneric_api_call_with_fallbacks() - model: %s; kwargs: %s", model, kwargs
)
verbose_router_logger.debug("Inside ageneric_api_call_with_fallbacks() - model: %s", model)
response: Final = await self.async_function_with_fallbacks(**kwargs)
return response
except Exception as e:
@ -7146,9 +7145,7 @@ class Router:
handler_name: Final = original_function.__name__
metadata_variable_name: Final = get_router_metadata_variable_name(function_name="generic_api_call")
try:
verbose_router_logger.debug(
"Inside _generic_api_call() - handler: %s, model: %s; kwargs: %s", handler_name, model, kwargs
)
verbose_router_logger.debug("Inside _generic_api_call() - handler: %s, model: %s", handler_name, model)
self._update_kwargs_before_fallbacks(
model=model,
kwargs=kwargs,
@ -7223,7 +7220,7 @@ class Router:
def _embedding(self, input: str | list[object], model: str, **kwargs):
model_name = None
try:
verbose_router_logger.debug("Inside embedding()- model: %s; kwargs: %s", model, kwargs)
verbose_router_logger.debug("Inside embedding()- model: %s", model)
with embedding_deployment_selection():
deployment: Final = self.get_available_deployment(
model=model,
@ -7312,7 +7309,7 @@ class Router:
async def _aembedding(self, input: str | list[object], model: str, **kwargs):
model_name = None
try:
verbose_router_logger.debug("Inside _aembedding()- model: %s; kwargs: %s", model, kwargs)
verbose_router_logger.debug("Inside _aembedding()- model: %s", model)
parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs)
with embedding_deployment_selection():
deployment: Final = await self.async_get_available_deployment(
@ -7396,7 +7393,7 @@ class Router:
try:
from litellm.router_utils.common_utils import add_model_file_id_mappings
verbose_router_logger.debug("Inside _atext_completion()- model: %s; kwargs: %s", model, kwargs)
verbose_router_logger.debug("Inside _atext_completion()- model: %s", model)
parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs)
healthy_deployments: Final = await self.async_get_healthy_deployments(
model=model,
@ -7634,7 +7631,7 @@ class Router:
**kwargs,
) -> LiteLLMBatch:
try:
verbose_router_logger.debug("Inside _acreate_batch()- model: %s; kwargs: %s", model, kwargs)
verbose_router_logger.debug("Inside _acreate_batch()- model: %s", model)
parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs)
with batch_create_routing():
deployment: Final = await self.async_get_available_deployment(
@ -7833,7 +7830,7 @@ class Router:
**kwargs,
) -> LiteLLMBatch:
try:
verbose_router_logger.debug("Inside _acancel_batch()- model: %s; kwargs: %s", model, kwargs)
verbose_router_logger.debug("Inside _acancel_batch()- model: %s", model)
parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs)
deployment: Final = await self.async_get_available_deployment(
model=model,
@ -10470,7 +10467,7 @@ class Router:
@staticmethod
def _inherit_builtin_base_rates_for_scoped_pricing(
model_info: dict, # mutable-ok: cost-map entry filled in place
model_info: dict[str, object], # mutable-ok: cost-map entry filled in place
backend_model: str,
custom_llm_provider: str | None,
) -> None:

View file

@ -12,7 +12,7 @@ is logged the first time such a deployment is seen.
"""
import contextlib
from collections.abc import Callable
from collections.abc import Callable, Mapping
from datetime import datetime, timedelta
from typing import TYPE_CHECKING, Any, Final, Literal
@ -38,7 +38,7 @@ from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import (
)
from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET
from litellm.types.caching import RedisPipelineIncrementOperation
from litellm.types.router import RouterErrors
from litellm.types.router import RouterErrors, parse_daily_reset_time
from litellm.types.utils import StandardLoggingPayload
from litellm.utils import get_utc_datetime
@ -58,42 +58,49 @@ class RoutingArgs:
DAY_FORMAT: Final = "%Y-%m-%d"
def seconds_until_utc_midnight(now: datetime) -> int:
next_midnight: Final = (now + timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0)
return max(1, int((next_midnight - now).total_seconds()))
def daily_window_day(now: datetime, reset_offset: timedelta) -> str:
return (now - reset_offset).strftime(DAY_FORMAT)
def _first_set(*values: int | None) -> int | None:
return next((value for value in values if value is not None), None)
def seconds_until_daily_reset(now: datetime, reset_offset: timedelta = timedelta(0)) -> int:
window_start: Final = (now - reset_offset).replace(hour=0, minute=0, second=0, microsecond=0)
next_reset: Final = window_start + timedelta(days=1) + reset_offset
return max(1, int((next_reset - now).total_seconds()))
class _DailyLimitFields(BaseModel):
class _DailyLimitParams(BaseModel):
"""rpd, tpd and daily_reset_time live in litellm_params only."""
model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True)
model: str | None = None
tpd: int | None = None
rpd: int | None = None
daily_reset_time: str | None = None
def reset_offset(self) -> timedelta:
return parse_daily_reset_time(self.daily_reset_time)
class _DailyLimitParams(_DailyLimitFields):
model: str | None = None
class _DailyLimitModelInfo(_DailyLimitFields):
class _DailyLimitModelInfo(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True)
id: str | None = None
class _DailyLimitDeployment(_DailyLimitFields):
class _DailyLimitDeployment(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True)
model_name: str = ""
litellm_params: _DailyLimitParams = _DailyLimitParams()
model_info: _DailyLimitModelInfo = _DailyLimitModelInfo()
def limits(self) -> tuple[int | None, int | None]:
return (
_first_set(self.tpd, self.litellm_params.tpd, self.model_info.tpd),
_first_set(self.rpd, self.litellm_params.rpd, self.model_info.rpd),
)
return self.litellm_params.tpd, self.litellm_params.rpd
def cache_keys(self, current_day: str) -> tuple[str, str]:
def reset_offset(self) -> timedelta:
return self.litellm_params.reset_offset()
def cache_keys(self, now: datetime) -> tuple[str, str]:
prefix: Final = f"{self.model_info.id}:{self.litellm_params.model}"
current_day: Final = daily_window_day(now, self.reset_offset())
return f"{prefix}:tpd:{current_day}", f"{prefix}:rpd:{current_day}"
def limit_error(self, kind: Literal["tpd", "rpd"], limit: int, usage: int, now: datetime) -> litellm.RateLimitError:
@ -104,12 +111,19 @@ class _DailyLimitDeployment(_DailyLimitFields):
response=httpx.Response(
status_code=429,
content=f"{RouterErrors.user_defined_ratelimit_error.value} {kind} limit={limit}. current usage={usage}. id={self.model_info.id}, model_group={self.model_name}",
headers={"retry-after": str(seconds_until_utc_midnight(now))},
headers={"retry-after": str(seconds_until_daily_reset(now, self.reset_offset()))},
request=httpx.Request(method="model_rate_limit_check", url="https://github.com/BerriAI/litellm"),
),
)
def _reset_offset_from_kwargs(kwargs: Mapping[str, object]) -> timedelta:
litellm_params: Final = kwargs.get("litellm_params")
if litellm_params is None:
return timedelta(0)
return _DailyLimitParams.model_validate(litellm_params).reset_offset()
class ModelRateLimitingCheck(CustomLogger):
"""
Pre-call check that enforces TPM/RPM or ITPM/OTPM limits on model deployments.
@ -242,7 +256,7 @@ class ModelRateLimitingCheck(CustomLogger):
tpd_limit, rpd_limit = deployment.limits()
if tpd_limit is None and rpd_limit is None:
return
tpd_key, rpd_key = deployment.cache_keys(now.strftime(DAY_FORMAT))
tpd_key, rpd_key = deployment.cache_keys(now)
if tpd_limit is not None:
current_tpd: Final = self._get_current_tpm(tpd_key, tpd_limit)
if current_tpd is not None and current_tpd >= tpd_limit:
@ -394,7 +408,7 @@ class ModelRateLimitingCheck(CustomLogger):
current_minute: Final = dt.strftime("%H-%M")
tpm_key, rpm_key = self._get_cache_keys(deployment, current_minute)
tpd_key, rpd_key = daily.cache_keys(dt.strftime(DAY_FORMAT))
tpd_key, rpd_key = daily.cache_keys(dt)
model_id: Final = deployment.get("model_info", {}).get("id")
model_name: Final = deployment.get("litellm_params", {}).get("model")
@ -510,7 +524,8 @@ class ModelRateLimitingCheck(CustomLogger):
dt: Final = self.clock()
current_minute: Final = dt.strftime("%H-%M")
tpm_key: Final = f"{model_id}:{model}:tpm:{current_minute}"
tpd_key: Final = f"{model_id}:{model}:tpd:{dt.strftime(DAY_FORMAT)}"
reset_offset: Final = _reset_offset_from_kwargs(kwargs) # pyright: ignore[reportUnknownArgumentType] # CustomLogger passes an untyped kwargs dict
tpd_key: Final = f"{model_id}:{model}:tpd:{daily_window_day(dt, reset_offset)}"
verbose_router_logger.debug("[TPM TRACKING] Incrementing %s by %s", tpm_key, total_tokens)
@ -570,7 +585,8 @@ class ModelRateLimitingCheck(CustomLogger):
dt: Final = self.clock()
current_minute: Final = dt.strftime("%H-%M")
tpm_key: Final = f"{model_id}:{model}:tpm:{current_minute}"
tpd_key: Final = f"{model_id}:{model}:tpd:{dt.strftime(DAY_FORMAT)}"
reset_offset: Final = _reset_offset_from_kwargs(kwargs) # pyright: ignore[reportUnknownArgumentType] # CustomLogger passes an untyped kwargs dict
tpd_key: Final = f"{model_id}:{model}:tpd:{daily_window_day(dt, reset_offset)}"
tokens: Final = int(total_tokens)
self._increment(tpm_key, tokens, RoutingArgs.ttl)

View file

@ -155,10 +155,9 @@ class SearchAPIRouter:
available_search_tool_names: Final = [tool.get("search_tool_name") for tool in router_instance.search_tools]
verbose_router_logger.debug(
"Inside SearchAPIRouter.async_search_with_fallbacks() - search_tool_name: %s, Available Search Tools: %s, kwargs: %s",
"Inside SearchAPIRouter.async_search_with_fallbacks() - search_tool_name: %s, Available Search Tools: %s",
search_tool_name,
available_search_tool_names,
kwargs,
)
# Use the existing retry/fallback infrastructure

View file

@ -191,6 +191,7 @@ class DeploymentOptions:
otpm: int | None = None
rpd: int | None = None
tpd: int | None = None
daily_reset_time: str | None = None
kubernetes_pod_discovery: bool | None = None
default_api_key_rpm_limit: int | None = None
default_api_key_tpm_limit: int | None = None

View file

@ -4,6 +4,7 @@ litellm.Router Types - includes RouterConfig, UpdateRouterConfig, ModelInfo etc
import datetime
import enum
import re
from collections.abc import Container, Mapping, Sequence
from dataclasses import dataclass, field
from typing import (
@ -518,6 +519,19 @@ def holds_secret_pointer(param_name: str) -> bool:
_RESERVED_INIT_KEYS: Final = frozenset({"self", "params", "__class__"})
_DAILY_RESET_TIME_PATTERN: Final = re.compile(r"\d{2}:\d{2}")
def parse_daily_reset_time(value: str | None) -> datetime.timedelta:
"""`daily_reset_time` is "HH:MM" in UTC, when the rpd/tpd window starts over; unset means midnight UTC."""
if value is None:
return datetime.timedelta(0)
if _DAILY_RESET_TIME_PATTERN.fullmatch(value) is None:
raise ValueError(f"daily_reset_time must be HH:MM in UTC, got {value!r}")
parsed: Final = datetime.time.fromisoformat(value)
return datetime.timedelta(hours=parsed.hour, minutes=parsed.minute)
class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
"""
LiteLLM Params without 'model' arg (used across completion / assistants api)
@ -535,6 +549,7 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
otpm: int | None = None
tpd: int | None = None
rpd: int | None = None
daily_reset_time: str | None = None
timeout: float | str | httpx.Timeout | SDKTimeout | None = None # if str, pass in as os.environ/
# read timeout of a live stream, a faked stream takes the longer of this and timeout. If str, pass in as os.environ/
stream_timeout: float | str | None = None
@ -626,6 +641,13 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
valkey_text_field: str | None = None
valkey_embedding_field: str | None = None
@field_validator("daily_reset_time")
@classmethod
def _daily_reset_time_is_utc_hh_mm(cls, value: str | None) -> str | None:
if value is not None:
parse_daily_reset_time(value)
return value
@field_validator("timeout")
@classmethod
def normalize_timeout(
@ -742,6 +764,7 @@ class LiteLLMParamsTypedDict(TypedDict, total=False):
otpm: int | None
tpd: ReadOnly[int | None]
rpd: ReadOnly[int | None]
daily_reset_time: ReadOnly[str | None]
order: int | None
weight: int | None
max_parallel_requests: int | None

View file

@ -2879,7 +2879,7 @@ class TranscriptionUsageTokensObject(LiteLLMBaseModel):
input_tokens: int
output_tokens: int
total_tokens: int
input_token_details: TranscriptionUsageInputTokenDetailsObject
input_token_details: TranscriptionUsageInputTokenDetailsObject | None = None
class TranscriptionResponse(OpenAIObject):
@ -4595,6 +4595,7 @@ class LlmProviders(str, Enum):
META = "meta"
SAIL = "sail"
REKA = "reka"
STREAMLAKE = "streamlake"
LITELLM_AGENT = "litellm_agent"
CURSOR = "cursor"
BEDROCK_MANTLE = "bedrock_mantle"

View file

@ -304,7 +304,7 @@ except (ImportError, AttributeError, TypeError):
# Convert to str (if necessary)
claude_json_str = json.dumps(json_data)
import importlib.metadata
from collections.abc import AsyncIterator, Callable, Collection, Iterable, Iterator, Mapping, Sequence
from collections.abc import AsyncIterator, Callable, Collection, Iterable, Iterator, Mapping, Sequence, Sized
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypeVar, cast, runtime_checkable
from typing_extensions import assert_never
@ -606,9 +606,23 @@ def custom_llm_setup():
litellm.provider_list.append(custom_llm["provider"])
if custom_llm["provider"] not in litellm._custom_providers:
_warn_if_built_in_provider_shadows_custom_handler(custom_llm["provider"])
litellm._custom_providers.append(custom_llm["provider"])
def _warn_if_built_in_provider_shadows_custom_handler(provider: str) -> None:
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
if provider not in LlmProvidersSet and not JSONProviderRegistry.exists(provider):
return
verbose_logger.warning(
"custom_provider_map provider '%s' has the same name as a built-in LiteLLM provider, which handles "
"'%s/' models instead of your custom_handler. Rename the provider and its models to use your handler",
provider,
provider,
)
def add_custom_logger_callback_to_specific_event(callback: str, logging_event: Literal["success", "failure"]) -> None:
"""
Add a custom logger callback to the specific event
@ -4067,7 +4081,9 @@ def get_optional_params_transcription(
"timestamp_granularities": None,
}
non_default_params = {k: v for k, v in passed_params.items() if (k in default_params and v != default_params[k])}
non_default_params: Final = {
k: v for k, v in passed_params.items() if (k in default_params and v != default_params[k])
}
optional_params = {}
## raise exception if non-default value passed for non-openai/azure embedding calls
@ -4104,6 +4120,19 @@ def get_optional_params_transcription(
model=model,
drop_params=drop_params if drop_params is not None else False,
)
elif custom_llm_provider == "bedrock":
from litellm.llms.bedrock.audio_transcription import (
UnsupportedTranscriptionParams,
map_openai_transcription_params,
)
match map_openai_transcription_params(
non_default_params, drop_params=drop_params is True or litellm.drop_params is True
):
case UnsupportedTranscriptionParams(message=message):
raise UnsupportedParamsError(message=message, llm_provider="bedrock")
case mapped_params:
optional_params = dict(mapped_params)
elif provider_config is not None: # custom audio transcription config
supported_params = provider_config.get_supported_openai_params(model=model)
_check_valid_arg(supported_params=supported_params)
@ -4118,7 +4147,7 @@ def get_optional_params_transcription(
optional_params=optional_params,
passed_params=passed_params,
custom_llm_provider=custom_llm_provider,
openai_params=OPENAI_TRANSCRIPTION_PARAMS,
openai_params=list(default_params) if custom_llm_provider == "bedrock" else OPENAI_TRANSCRIPTION_PARAMS,
additional_drop_params=kwargs.get("additional_drop_params", None),
)
@ -5059,8 +5088,10 @@ def get_optional_params(
supported_params: List[str] - supported params from the litellm config
"""
verbose_logger.info("\nLiteLLM completion() model= %s; provider = %s", model, custom_llm_provider)
verbose_logger.debug("\nLiteLLM: Params passed to completion() %s", passed_params)
verbose_logger.debug("\nLiteLLM: Non-Default params passed to completion() %s", non_default_params)
if verbose_logger.isEnabledFor(logging.DEBUG):
verbose_logger.debug(
"\nLiteLLM: Non-Default params passed to completion() %s", _debug_params_repr(non_default_params)
)
unsupported_params: Final = {}
for k in non_default_params:
if k not in supported_params:
@ -5601,7 +5632,9 @@ def get_optional_params(
additional_drop_params=additional_drop_params,
)
if _print_verbose_is_active():
print_verbose(f"Final returned optional params: {redact_credentials_in_payload(optional_params)}")
print_verbose(
f"Final returned optional params: {_debug_params_repr(redact_credentials_in_payload(optional_params))}"
)
optional_params = apply_openai_param_overrides(
optional_params=optional_params,
non_default_params=non_default_params,
@ -6610,8 +6643,6 @@ def get_model_info_helper(
model=model, custom_llm_provider=custom_llm_provider or declared_authenticating_provider(model)
)
verbose_logger.debug("checking potential_model_names in litellm.model_cost: %s", potential_model_names)
combined_model_name: Final = potential_model_names["combined_model_name"]
stripped_model_name: Final = potential_model_names["stripped_model_name"]
combined_stripped_model_name: Final = potential_model_names["combined_stripped_model_name"]
@ -6714,6 +6745,7 @@ def get_model_info_helper(
key, _model_info = generalization
if _model_info is None or key is None:
verbose_logger.debug("model not found in litellm.model_cost, tried: %s", potential_model_names)
raise ModelNotMappedError(_model_not_mapped_message(model, custom_llm_provider))
_input_cost_per_token: float | None = _model_info.get("input_cost_per_token")
if _input_cost_per_token is None:
@ -8534,6 +8566,23 @@ def get_valid_models(
return [] # NON-Blocking
_MAX_INLINE_DEBUG_ARG_CHARS: Final = 500
def _debug_arg_repr(value: object) -> str:
"""Short form of a call argument, since the outgoing provider request log already prints the request body."""
rendered: Final = repr(value)
if len(rendered) <= _MAX_INLINE_DEBUG_ARG_CHARS:
return rendered
if isinstance(value, Sized):
return f"<{type(value).__name__} len={len(value)}>"
return f"<{type(value).__name__}>"
def _debug_params_repr(params: Mapping[str, object]) -> str:
return "{" + ", ".join(f"{key!r}: {_debug_arg_repr(value)}" for key, value in params.items()) + "}"
def print_args_passed_to_litellm(original_function, args, kwargs):
if not is_debugging_on():
return
@ -8554,9 +8603,9 @@ def print_args_passed_to_litellm(original_function, args, kwargs):
):
return
args_str: Final = ", ".join(map(repr, args))
args_str: Final = ", ".join(map(_debug_arg_repr, args))
redacted_kwargs: Final = redact_credentials_in_payload(kwargs)
kwargs_str: Final = ", ".join(f"{key}={value!r}" for key, value in redacted_kwargs.items())
kwargs_str: Final = ", ".join(f"{key}={_debug_arg_repr(value)}" for key, value in redacted_kwargs.items())
print_verbose(
"\n",
) # new line before

View file

@ -41939,6 +41939,120 @@
"supports_system_messages": true,
"supports_native_structured_output": true
},
"bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 4.12e-08,
"input_cost_per_token_batches": 2.06e-08,
"input_cost_per_token_flex": 2.06e-08,
"input_cost_per_token_priority": 7.21e-08,
"litellm_provider": "bedrock",
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 4.12e-08,
"output_cost_per_token_batches": 2.06e-08,
"output_cost_per_token_flex": 2.06e-08,
"output_cost_per_token_priority": 7.21e-08,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json",
"supports_audio_input": true,
"supports_system_messages": true,
"supports_native_structured_output": true
},
"bedrock/ap-southeast-2/mistral.voxtral-small-24b-2507": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 1.03e-07,
"input_cost_per_token_batches": 5.15e-08,
"input_cost_per_token_flex": 5.15e-08,
"input_cost_per_token_priority": 1.8025e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 3.09e-07,
"output_cost_per_token_batches": 1.545e-07,
"output_cost_per_token_flex": 1.545e-07,
"output_cost_per_token_priority": 5.4075e-07,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json",
"supports_audio_input": true,
"supports_system_messages": true,
"supports_native_structured_output": true
},
"bedrock/ap-northeast-1/mistral.voxtral-mini-3b-2507": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 5e-08,
"input_cost_per_token_batches": 3e-08,
"input_cost_per_token_flex": 3e-08,
"input_cost_per_token_priority": 9e-08,
"litellm_provider": "bedrock",
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 5e-08,
"output_cost_per_token_batches": 3e-08,
"output_cost_per_token_flex": 3e-08,
"output_cost_per_token_priority": 9e-08,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-northeast-1/index.json",
"supports_audio_input": true,
"supports_system_messages": true,
"supports_native_structured_output": true
},
"bedrock/ap-northeast-1/mistral.voxtral-small-24b-2507": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 1.2e-07,
"input_cost_per_token_batches": 6e-08,
"input_cost_per_token_flex": 6e-08,
"input_cost_per_token_priority": 2.1e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 3.6e-07,
"output_cost_per_token_batches": 1.8e-07,
"output_cost_per_token_flex": 1.8e-07,
"output_cost_per_token_priority": 6.3e-07,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-northeast-1/index.json",
"supports_audio_input": true,
"supports_system_messages": true,
"supports_native_structured_output": true
},
"bedrock/eu-west-2/mistral.voxtral-mini-3b-2507": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 6e-08,
"input_cost_per_token_batches": 3e-08,
"input_cost_per_token_flex": 3e-08,
"input_cost_per_token_priority": 1.1e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 6e-08,
"output_cost_per_token_batches": 3e-08,
"output_cost_per_token_flex": 3e-08,
"output_cost_per_token_priority": 1.1e-07,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/eu-west-2/index.json",
"supports_audio_input": true,
"supports_system_messages": true,
"supports_native_structured_output": true
},
"bedrock/eu-west-2/mistral.voxtral-small-24b-2507": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 1.6e-07,
"input_cost_per_token_batches": 8e-08,
"input_cost_per_token_flex": 8e-08,
"input_cost_per_token_priority": 2.8e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 4.7e-07,
"output_cost_per_token_batches": 2.3e-07,
"output_cost_per_token_flex": 2.3e-07,
"output_cost_per_token_priority": 8.2e-07,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/eu-west-2/index.json",
"supports_audio_input": true,
"supports_system_messages": true,
"supports_native_structured_output": true
},
"mistral/codestral-2508": {
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 3e-07,
@ -61374,6 +61488,359 @@
"supports_tool_choice": true,
"supports_vision": true
},
"streamlake/deepseek-v4-flash-0731": {
"cache_read_input_token_cost": 1.4e-08,
"input_cost_per_token": 4.4e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 1048576,
"max_output_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 1.32e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/DeepSeek-V4-Pro-0813": {
"cache_read_input_token_cost": 4.4e-08,
"input_cost_per_token": 1.32e-06,
"litellm_provider": "streamlake",
"max_input_tokens": 1048566,
"max_output_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 3.96e-06,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/DeepSeek-V3": {
"cache_read_input_token_cost": 1.14e-07,
"input_cost_per_token": 2.86e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 56000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 1.143e-06,
"supports_function_calling": true,
"supports_reasoning": false,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/GLM-5.3": {
"cache_read_input_token_cost": 2.6e-07,
"input_cost_per_token": 1.4e-06,
"litellm_provider": "streamlake",
"max_input_tokens": 1048576,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 4.4e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_assistant_prefill": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/GLM-5.3-Flash": {
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 1048576,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 5e-07,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_assistant_prefill": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/Kimi-K2-Instruct-0905": {
"cache_read_input_token_cost": 1.43e-07,
"input_cost_per_token": 5.71e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 224000,
"max_output_tokens": 32000,
"mode": "chat",
"output_cost_per_token": 2.286e-06,
"supports_function_calling": true,
"supports_reasoning": false,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/deepseek-r1": {
"cache_read_input_token_cost": 2.29e-07,
"input_cost_per_token": 5.71e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 98304,
"max_output_tokens": 16000,
"mode": "chat",
"output_cost_per_token": 2.286e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/deepseek-v3.1": {
"cache_read_input_token_cost": 2.29e-07,
"input_cost_per_token": 5.71e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 98304,
"max_output_tokens": 32000,
"mode": "chat",
"output_cost_per_token": 1.714e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/deepseek-v4-flash": {
"cache_read_input_token_cost": 2.8e-08,
"input_cost_per_token": 1.4e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 1048576,
"max_output_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 2.8e-07,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/deepseek-v4-pro": {
"cache_read_input_token_cost": 1.45e-07,
"input_cost_per_token": 1.74e-06,
"litellm_provider": "streamlake",
"max_input_tokens": 1048576,
"max_output_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 3.48e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/deepseek-v4.1-flash-1017": {
"cache_read_input_token_cost": 6e-09,
"input_cost_per_token": 3e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 1048576,
"max_output_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_vision": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/deepseek-v4.1-flash-260910": {
"cache_read_input_token_cost": 6e-09,
"input_cost_per_token": 3e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 1048568,
"max_output_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_vision": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/glm-5": {
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 1e-06,
"litellm_provider": "streamlake",
"max_input_tokens": 202752,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3.2e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/glm-5.1": {
"cache_read_input_token_cost": 2.6e-07,
"input_cost_per_token": 1.4e-06,
"litellm_provider": "streamlake",
"max_input_tokens": 202752,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 4.4e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/glm-5.2": {
"cache_read_input_token_cost": 2.6e-07,
"input_cost_per_token": 1.4e-06,
"litellm_provider": "streamlake",
"max_input_tokens": 1048566,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 4.4e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/kimi-k2.6": {
"cache_read_input_token_cost": 1.6e-07,
"input_cost_per_token": 9.5e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 262144,
"max_output_tokens": 256000,
"mode": "chat",
"output_cost_per_token": 4e-06,
"supports_function_calling": true,
"supports_vision": true,
"supports_reasoning": false,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/kimi-k2.7-code": {
"cache_read_input_token_cost": 1.9e-07,
"input_cost_per_token": 9.5e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 262144,
"max_output_tokens": 32000,
"mode": "chat",
"output_cost_per_token": 4e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_vision": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/mimo-v2.6-flash": {
"cache_read_input_token_cost": 2.8e-09,
"input_cost_per_token": 1.4e-07,
"litellm_provider": "streamlake",
"mode": "chat",
"output_cost_per_token": 2.8e-07,
"supports_vision": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/mimo-v2.6-pro": {
"cache_read_input_token_cost": 3.6e-09,
"input_cost_per_token": 4.35e-07,
"litellm_provider": "streamlake",
"mode": "chat",
"output_cost_per_token": 8.7e-07,
"supports_vision": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/minimax-m2.5": {
"cache_creation_input_token_cost": 3.75e-07,
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 3e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 196608,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/minimax-m3": {
"cache_read_input_token_cost": 1.2e-07,
"cache_read_input_token_cost_above_512k_tokens": 2.4e-07,
"input_cost_per_token": 6e-07,
"input_cost_per_token_above_512k_tokens": 1.2e-06,
"litellm_provider": "streamlake",
"max_input_tokens": 1000000,
"max_output_tokens": 512000,
"mode": "chat",
"output_cost_per_token": 2.4e-06,
"output_cost_per_token_above_512k_tokens": 4.8e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_vision": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/qwen3.5-397b-a17b": {
"cache_read_input_token_cost": 1.2e-07,
"input_cost_per_token": 6e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 258048,
"max_output_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 3.6e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_vision": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/qwen3.6-27b": {
"input_cost_per_token": 6e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 229376,
"max_output_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 3.6e-06,
"supports_function_calling": true,
"supports_vision": true,
"supports_reasoning": false,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/qwen3.6-35b-a3b": {
"input_cost_per_token": 3.75e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 229376,
"max_output_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 2.25e-06,
"supports_function_calling": true,
"supports_vision": true,
"supports_reasoning": false,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/qwen3.8-2.4t-a95b": {
"cache_read_input_token_cost": 2.5e-07,
"input_cost_per_token": 2e-06,
"litellm_provider": "streamlake",
"max_input_tokens": 983616,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 6e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_vision": true,
"cache_creation_input_token_cost": 2.5e-06,
"explicit_cache_read_input_token_cost": 1.7e-07,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/qwen3.8-27b": {
"cache_read_input_token_cost": 1e-07,
"input_cost_per_token": 5e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 983616,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_vision": true,
"cache_creation_input_token_cost": 6.25e-07,
"explicit_cache_read_input_token_cost": 5e-08,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"deepseek-v4-flash": {
"cache_creation_input_token_cost": 0.0,
"cache_read_input_token_cost": 6e-09,
@ -62689,6 +63156,14 @@
"litellm_provider": "anthropic"
}
},
{
"name": "typesafe-jev-ids",
"pattern": "^jev-(?:latest|preview|1\\.13\\.0|1\\.14\\.0)$",
"description": "A bare Jev id that has a typesafe/ price entry (jev-latest, jev-preview, jev-1.13.0, jev-1.14.0), the name the /typesafe pass-through forwards to TypeSafe. Listed by name, matched case-insensitively like every rule here, so foo/jev-latest or a model group named like a Jev id TypeSafe does not serve, such as jev-1.0 or jev-internal-router, never resolves to typesafe. A new Jev version goes here together with its price entry. Provider inference feeds the proxy's provider/* wildcard access checks, so this rule is what lets typesafe/* on a key, team or project allowlist cover a jev-latest request.",
"model_info": {
"litellm_provider": "typesafe"
}
},
{
"name": "claude-family-baseline",
"pattern": "claude-[a-z]+-\\d+(?:[-.]\\d+)?(?:-\\d{8})?",

View file

@ -3785,6 +3785,32 @@
"generateContent": true
}
},
"streamlake": {
"display_name": "StreamLake (`streamlake`)",
"url": "https://docs.litellm.ai/docs/providers/openai_compatible",
"endpoints": {
"chat_completions": true,
"responses": true,
"text_completion": true,
"embeddings": false,
"image_generations": false,
"image_edits": false,
"audio_transcriptions": false,
"audio_speech": false,
"rerank": false,
"ocr": false,
"search": false,
"moderations": false,
"realtime": false,
"video_generations": false,
"batches": false,
"messages": true,
"interactions": true,
"systemone": false,
"a2a": true,
"generateContent": true
}
},
"tencent": {
"display_name": "Tencent TokenHub (`tencent`)",
"url": "https://docs.litellm.ai/docs/providers/tencent",

View file

@ -8,6 +8,10 @@ the same tests against Google's live spec
Unit tests must not depend on the current time or execution speed. Use fixed timestamps for fixtures
and controlled clocks for time-dependent behavior. Incidental clock reads inside production code are allowed
A unix socket goes under the `socket_dir` or `socket_path` fixture, never `tmp_path`, whose paths run past macOS's
104-byte AF_UNIX limit and fail only on a Mac. Code that opens a socket in its own `tempfile` directory runs under
`socket_tempdir`
## What good looks like
```python

View file

@ -2,6 +2,8 @@ import asyncio
import base64
import importlib
import os
import shutil
import tempfile
from collections.abc import Callable, Coroutine, Iterator
from dataclasses import dataclass, field
from pathlib import Path
@ -217,6 +219,38 @@ def _flush_client_caches() -> None:
_reset_aws_auth_caches()
_MACOS_LONGEST_SOCKET_PATH: Final = 103
_LONGEST_SOCKET_SUFFIX: Final = len("/sock12345678/d/litellm-pgbouncer-12345678/.s.PGSQL.65535")
def _socket_root() -> str:
default: Final = tempfile.gettempdir()
fits: Final = len(os.fsencode(default)) + _LONGEST_SOCKET_SUFFIX <= _MACOS_LONGEST_SOCKET_PATH
return default if fits else "/tmp"
@pytest.fixture
def socket_dir(tmp_path: Path) -> Iterator[Path]:
"""``tmp_path`` behind a short link, so unix socket paths fit the sun_path limit that it exceeds on macOS."""
link_dir: Final = Path(tempfile.mkdtemp(prefix="sock", dir=_socket_root()))
link: Final = link_dir / "d"
link.symlink_to(tmp_path, target_is_directory=True)
yield link
shutil.rmtree(link_dir, ignore_errors=True)
@pytest.fixture
def socket_path(socket_dir: Path) -> Path:
return socket_dir / "spend.sock"
@pytest.fixture
def socket_tempdir(socket_dir: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
"""Point ``tempfile``'s default directory at ``socket_dir``, for code that picks its own socket directory."""
monkeypatch.setattr(tempfile, "tempdir", str(socket_dir))
return socket_dir
@pytest.fixture(autouse=True, scope="session")
def bundled_tiktoken_cache() -> None:
importlib.import_module("litellm.litellm_core_utils.default_encoding")

View file

@ -1,5 +1,5 @@
import unittest.mock as mock
from typing import cast
from typing import Final, cast
from unittest.mock import MagicMock, patch
import pytest
@ -197,6 +197,17 @@ class TestEnterpriseCallbackControls:
)
assert result is False
def test_no_disable_request_logs_nothing(self, caplog: pytest.LogCaptureFixture) -> None:
litellm_params: Final = {"proxy_server_request": {"url": "test", "headers": {"user-agent": "codex"}}}
with caplog.at_level("DEBUG", logger="LiteLLM"):
result: Final = EnterpriseCallbackControls.is_callback_disabled_dynamically(
DataDogLogger.__new__(DataDogLogger), litellm_params, StandardCallbackDynamicParams()
)
assert result is False
assert caplog.records == []
def test_callback_not_disabled_when_header_none(
self, mock_premium_user, mock_request_headers
):

View file

@ -140,7 +140,7 @@ class TestPoolDatabaseUrl:
assert "DATABASE_URL" in outcome.reason
def test_token_auth_hands_the_workers_the_pool_user_not_the_token(
self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path
self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, socket_tempdir: Path
):
monkeypatch.setenv("IAM_TOKEN_DB_AUTH", "true")
monkeypatch.setenv("AWS_REGION_NAME", "us-east-1")
@ -149,7 +149,7 @@ class TestPoolDatabaseUrl:
with _minted_iam_token("MINTED_TOKEN"):
outcome: Final = pool_database_url(
DatabaseURLSettings.from_env(),
PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pooler(tmp_path))),
PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pooler(socket_tempdir))),
environ,
)
assert isinstance(outcome, str), outcome
@ -160,12 +160,12 @@ class TestPoolDatabaseUrl:
class TestMain:
def test_workers_inherit_the_loopback_url_the_supervisor_installed(
self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path
self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, socket_tempdir: Path
):
port: Final = _free_port()
monkeypatch.setenv("LITELLM_PGBOUNCER_ENABLED", "true")
monkeypatch.setenv("LITELLM_PGBOUNCER_PORT", str(port))
monkeypatch.setenv("LITELLM_PGBOUNCER_BINARY", str(_fake_pooler(tmp_path)))
monkeypatch.setenv("LITELLM_PGBOUNCER_BINARY", str(_fake_pooler(socket_tempdir)))
monkeypatch.setenv("KEEPALIVE_TIMEOUT", "75")
served: Final[list[tuple[str, ...]]] = []
main(("--workers", "4"), serve=lambda argv: served.append(tuple(argv)))
@ -183,7 +183,7 @@ class TestMain:
assert _query(os.environ["DATABASE_URL"])["pgbouncer"] == "true"
def test_iam_workers_keep_the_loopback_url_instead_of_minting_their_own(
self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path
self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, socket_tempdir: Path
):
port: Final = _free_port()
monkeypatch.delenv("DATABASE_PASSWORD")
@ -191,7 +191,7 @@ class TestMain:
monkeypatch.setenv("AWS_REGION_NAME", "us-east-1")
monkeypatch.setenv("LITELLM_PGBOUNCER_ENABLED", "true")
monkeypatch.setenv("LITELLM_PGBOUNCER_PORT", str(port))
monkeypatch.setenv("LITELLM_PGBOUNCER_BINARY", str(_fake_pooler(tmp_path)))
monkeypatch.setenv("LITELLM_PGBOUNCER_BINARY", str(_fake_pooler(socket_tempdir)))
served: Final[list[tuple[str, ...]]] = []
with _minted_iam_token("SUPERVISOR_TOKEN"):
main(("--workers", "4"), serve=lambda argv: served.append(tuple(argv)))

View file

@ -905,10 +905,10 @@ async def test_a_failed_request_is_delivered_as_an_error_log_that_keeps_the_erro
@pytest.mark.asyncio
async def test_a_failing_redis_cache_is_delivered_to_datadog_as_redis_warnings(
delivery: tuple[DataDogLogger, respx.Route], monkeypatch: pytest.MonkeyPatch, tmp_path: Path
delivery: tuple[DataDogLogger, respx.Route], monkeypatch: pytest.MonkeyPatch, socket_dir: Path
) -> None:
datadog_logger, intake = delivery
absent_socket: Final = str(tmp_path / "absent.sock")
absent_socket: Final = str(socket_dir / "absent.sock")
redis_cache: Final = Cache(type="redis", url=f"unix://{absent_socket}")
monkeypatch.setattr(redis_cache.cache.service_logger_obj, "dd_logger", datadog_logger, raising=False)
monkeypatch.setattr(litellm, "service_callback", ["datadog"])

View file

@ -714,6 +714,48 @@ class TestClaudeModelPatternMatching:
litellm.get_llm_provider(model="claude-opus-4-9")
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
class TestTypesafeJevPatternMatching:
def test_every_priced_jev_id_routes_to_typesafe(
self, shipped_generalizations: None
) -> None:
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
priced_ids: Final = tuple(
key.removeprefix("typesafe/")
for key in GetModelCostMap.load_local_model_cost_map()
if key.startswith("typesafe/")
)
assert "jev-latest" in priced_ids
assert {
model: litellm.get_llm_provider(model=model)[1] for model in priced_ids
} == dict.fromkeys(priced_ids, "typesafe")
@pytest.mark.parametrize(
"model",
[
"foo/jev-latest",
"openjev-latest",
"jev-",
"jev-latest model",
"jev-internal-router",
"jev-2.0-internal",
"jev-1.0-acme.router",
"jev-1.0",
"jev-2.0",
"jev-1.13",
],
)
def test_namespaced_or_partial_jev_name_does_not_match_rule(
self, model: str, shipped_generalizations: None
) -> None:
from litellm.litellm_core_utils.fallback_generalizations import (
match_routing_generalization,
)
assert match_routing_generalization(model) is None
@pytest.mark.parametrize("prefix", ["openai", "anthropic", "custom_openai", "openai_like"])
def test_opencode_affinity_resolution_preserves_existing_transport(prefix: str) -> None:
api_base: Final = "https://opencode.ai/zen/go/v1"

View file

@ -6316,6 +6316,22 @@ def test_chunk_creator_preserves_hidden_provider_specific_fields_from_parsed_chu
assert assembled._hidden_params["provider_specific_fields"] == {"traffic_type": "ON_DEMAND_FLEX"}
def test_the_usage_chunk_carries_the_bill_the_builder_priced_for_the_spend_row() -> None:
from litellm.litellm_core_utils.streaming_handler import (
_stamp_usage_cost, # pyright: ignore[reportPrivateUsage] # the helper under test
)
complete: Final = ModelResponse(model="omni-model", usage=Usage(prompt_tokens=100, completion_tokens=20))
complete.hidden_params = {"response_cost": 0.00032}
chunk_usage: Final = Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120)
setattr(chunk_usage, "cost", 0.037)
usage_chunk: Final = ModelResponseStream(id="chunk-1", model="omni-model", choices=[], usage=chunk_usage)
_stamp_usage_cost(usage_chunk, complete)
assert usage_chunk.usage is not None and usage_chunk.usage.cost == 0.00032, "the client sees what the row bills"
def test_chunk_creator_keeps_provider_model_private_across_stream():
from litellm.router_utils.add_retry_fallback_headers import (
get_hidden_params_dict,

View file

@ -165,6 +165,44 @@ def test_explicit_v1_api_base_keeps_responses_api_version_selection(
@pytest.mark.serial
@pytest.mark.parametrize(
("api_base", "api_version", "expected_url"),
[
(
"https://x.openai.azure.com",
"2025-04-01-preview",
"https://x.openai.azure.com/openai/v1/responses/compact",
),
(
"https://x.openai.azure.com",
"preview",
"https://x.openai.azure.com/openai/v1/responses/compact?api-version=preview",
),
(
"https://gw.example.com/foundry/openai/openai/v1?tenant=example",
"v1",
"https://gw.example.com/foundry/openai/openai/v1/responses/compact?tenant=example&api-version=v1",
),
],
)
def test_compact_reaches_the_v1_route_and_drops_only_a_dated_api_version(
api_base: str, api_version: str, expected_url: str
) -> None:
config: Final = AzureOpenAIResponsesAPIConfig()
responses_url: Final = config.get_complete_url(api_base=api_base, litellm_params={"api_version": api_version})
url, _ = config.transform_compact_response_api_request(
model="gpt-5.4-mini",
input="hi",
response_api_optional_request_params={},
api_base=responses_url,
litellm_params=GenericLiteLLMParams(),
headers={},
)
assert url == expected_url
def test_response_id_path_requests_encode_response_id():
config = AzureOpenAIResponsesAPIConfig()
api_base = (

View file

@ -1658,9 +1658,9 @@ def test_get_azure_ad_token_fallback_to_default_azure_credential(
)
def test_is_azure_v1_api_version(api_version, expected):
"""
Test that _is_azure_v1_api_version correctly identifies v1 API versions.
Test that is_azure_v1_api_version correctly identifies v1 API versions.
"""
result = BaseAzureLLM._is_azure_v1_api_version(api_version=api_version)
result = BaseAzureLLM.is_azure_v1_api_version(api_version=api_version)
assert result == expected

View file

@ -7,6 +7,7 @@ import respx
import litellm
from litellm.llms.litellm_proxy.responses.transformation import LiteLLMProxyResponsesAPIConfig
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager
from tests.unit.proxy.conftest import httpx_transport
@ -73,3 +74,23 @@ def test_get_complete_url_requires_api_base(monkeypatch: pytest.MonkeyPatch) ->
monkeypatch.delenv("LITELLM_PROXY_API_BASE", raising=False)
with pytest.raises(ValueError, match="api_base not set"):
LiteLLMProxyResponsesAPIConfig().get_complete_url(api_base=None, litellm_params={})
@pytest.mark.parametrize("drop_params", [False, True])
def test_compact_request_forwards_metadata_and_user_to_the_downstream_proxy(drop_params: bool) -> None:
url, data = LiteLLMProxyResponsesAPIConfig().transform_compact_response_api_request(
model="gpt-5.5",
input="hi",
response_api_optional_request_params={"metadata": {"ticket": "qa-1"}, "user": "end-user-1", "temperature": 0.2},
api_base="https://my-proxy.example.com/responses",
litellm_params=GenericLiteLLMParams(drop_params=drop_params),
headers={},
)
assert url == "https://my-proxy.example.com/responses/compact"
assert {key: data[key] for key in data if key != "input"} == {
"model": "gpt-5.5",
"metadata": {"ticket": "qa-1"},
"user": "end-user-1",
"temperature": 0.2,
}

View file

@ -7,12 +7,14 @@ with guardrail transformations.
import copy
import json
import uuid
from collections.abc import Callable
from typing import Any, Final, List, Literal, Optional, Tuple
from unittest.mock import AsyncMock, MagicMock, patch
import logging
import httpx
import pytest
@ -33,6 +35,10 @@ from litellm.llms.openai.responses.guardrail_translation.handler import (
)
from litellm.llms.openai.responses.guardrail_translation.tool_merge import merge_guardrailed_tools
from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api import GenericGuardrailAPI
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import BedrockGuardrail
from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ContentFilterGuardrail
from litellm.types.guardrails import ContentFilterAction, ContentFilterPattern
from litellm.types.llms.openai import ChatCompletionToolCallChunk
from litellm.responses.litellm_completion_transformation.compaction_items import compaction_summary
from litellm.responses.litellm_completion_transformation.transformation import (
@ -83,6 +89,7 @@ class RecordingMaskingGuardrail(MockGuardrail):
super().__init__(guardrail_name=guardrail_name)
self.seen_texts: list[list[str]] = []
self.seen_message_contents: list[list[object]] = []
self.seen_tools: list[object] = []
async def apply_guardrail(
self,
@ -93,6 +100,7 @@ class RecordingMaskingGuardrail(MockGuardrail):
) -> GenericGuardrailAPIInputs:
self.seen_texts.append(list(inputs.get("texts", [])))
self.seen_message_contents.append([m["content"] for m in inputs.get("structured_messages") or []])
self.seen_tools.append(inputs.get("tools"))
return await super().apply_guardrail(inputs, request_data, input_type, logging_obj)
@ -588,6 +596,638 @@ class TestSkipSystemMessageScopesInstructions:
]
_TOOL_TURNS: Final = (
(
{"type": "function_call", "call_id": "call_1", "name": "get_weather", "arguments": '{"city": "Paris"}'},
"function_call_output",
),
(
{"type": "custom_tool_call", "call_id": "call_1", "name": "run_sql", "input": "select 1"},
"custom_tool_call_output",
),
)
def _tool_result_request(call_item: dict[str, str], output_type: str, output: object) -> dict[str, object]:
return {
"model": "gpt-5.4-mini",
"input": [
{"role": "user", "content": "Weather?"},
call_item,
{"type": output_type, "call_id": "call_1", "output": output},
],
}
def _bedrock_guardrail(
name: str, bedrock_api: Callable[[httpx.Request], httpx.Response], **settings: bool
) -> BedrockGuardrail:
return BedrockGuardrail(
guardrail_name=name,
guardrailIdentifier="test-guardrail",
guardrailVersion="DRAFT",
aws_access_key_id="test-access-key",
aws_secret_access_key="test-secret-key",
aws_region_name="us-east-1",
async_handler=AsyncHTTPHandler(transport=httpx.MockTransport(bedrock_api)),
**settings,
)
def _bedrock_block_texts(request: httpx.Request) -> list[str]:
return [block["text"]["text"] for block in json.loads(request.content)["content"]]
class TestToolResultItemsReachTextsGuardrails:
"""A Responses tool result is scanned and rewritten in its `output` field the way a chat `tool`
message's content is, and skip_tool_message_in_guardrail keeps it out of the scan the same way."""
@pytest.mark.asyncio
@pytest.mark.parametrize(("call_item", "output_type"), _TOOL_TURNS, ids=["function", "custom"])
async def test_string_output_is_scanned_and_rewritten_in_place(
self, call_item: dict[str, str], output_type: str
) -> None:
guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test")
result: Final = await OpenAIResponsesHandler().process_input_messages(
_tool_result_request(call_item, output_type, "SECRET forecast"), guardrail
)
assert guardrail.seen_texts == [["Weather?", "SECRET forecast"]]
assert result["input"] == [
{"role": "user", "content": "Weather? [GUARDRAILED]"},
call_item,
{"type": output_type, "call_id": "call_1", "output": "SECRET forecast [GUARDRAILED]"},
]
@pytest.mark.asyncio
async def test_content_part_output_is_scanned_and_rewritten_in_its_part(self) -> None:
guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test")
call_item, output_type = _TOOL_TURNS[0]
image_part: Final = {"type": "input_image", "image_url": "https://example.com/map.png"}
result: Final = await OpenAIResponsesHandler().process_input_messages(
_tool_result_request(
call_item, output_type, [{"type": "input_text", "text": "SECRET forecast"}, image_part]
),
guardrail,
)
assert guardrail.seen_texts == [["Weather?", "SECRET forecast"]]
assert result["input"][2]["output"] == [
{"type": "input_text", "text": "SECRET forecast [GUARDRAILED]"},
image_part,
]
@pytest.mark.asyncio
@pytest.mark.parametrize("part_type", ["input_text", "output_text"])
async def test_text_only_parts_are_scanned_and_rewritten_part_by_part(self, part_type: str) -> None:
guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test")
call_item, output_type = _TOOL_TURNS[0]
parts: Final = [
{"type": part_type, "text": "SECRET forecast: ", "annotations": []},
{"type": part_type, "text": "sunny"},
]
result: Final = await OpenAIResponsesHandler().process_input_messages(
_tool_result_request(call_item, output_type, parts), guardrail
)
assert guardrail.seen_texts == [["Weather?", "SECRET forecast: ", "sunny"]]
assert result["input"][2]["output"] == [
{"type": part_type, "text": "SECRET forecast: [GUARDRAILED]", "annotations": []},
{"type": part_type, "text": "sunny [GUARDRAILED]"},
]
@pytest.mark.asyncio
async def test_a_pattern_ending_at_a_part_boundary_blocks_like_it_does_on_chat(self) -> None:
access_key: Final = "AKIA" + "SYNTHETIC0000000"
content_filter: Final = ContentFilterGuardrail(
guardrail_name="secrets",
patterns=[
ContentFilterPattern(
pattern_type="prebuilt", pattern_name="aws_access_key", action=ContentFilterAction.BLOCK
)
],
)
call_item, output_type = _TOOL_TURNS[0]
parts: Final = [
{"type": "input_text", "text": f"key={access_key}"},
{"type": "input_text", "text": "region=us-east-1"},
]
with pytest.raises(HTTPException) as blocked:
await OpenAIResponsesHandler().process_input_messages(
_tool_result_request(call_item, output_type, parts), content_filter
)
assert blocked.value.status_code == 400
@pytest.mark.asyncio
@pytest.mark.parametrize("empty_output", ["", None], ids=["empty_string", "null"])
async def test_empty_tool_output_sends_no_text_and_stays_as_sent(self, empty_output: str | None) -> None:
guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test")
call_item, output_type = _TOOL_TURNS[0]
result: Final = await OpenAIResponsesHandler().process_input_messages(
_tool_result_request(call_item, output_type, empty_output), guardrail
)
assert guardrail.seen_texts == [["Weather?"]]
assert result["input"][2] == {"type": output_type, "call_id": "call_1", "output": empty_output}
@pytest.mark.asyncio
@pytest.mark.parametrize("latest_user_only", [False, True], ids=["all_texts", "latest_user_only"])
@pytest.mark.parametrize(
"tool_output",
[
{"type": "function_call_output", "call_id": "call_1", "output": ""},
{"type": "function_call_output", "call_id": "call_1", "output": None},
{
"type": "computer_call_output",
"call_id": "call_1",
"output": {"type": "computer_screenshot", "file_id": "file-screen"},
},
],
ids=["empty_string", "null", "file_id_screenshot"],
)
async def test_bedrock_masking_applies_to_a_request_with_a_textless_tool_output(
self, tool_output: dict[str, object], latest_user_only: bool
) -> None:
sent_texts: Final[list[list[str]]] = []
def bedrock_masking_emails(request: httpx.Request) -> httpx.Response:
texts: Final = _bedrock_block_texts(request)
sent_texts.append(texts)
outputs: Final = [{"text": text.replace("jane@example.com", "{EMAIL}")} for text in texts]
return httpx.Response(200, json={"action": "GUARDRAIL_INTERVENED", "outputs": outputs})
guardrail: Final = _bedrock_guardrail(
"bedrock-pre", bedrock_masking_emails, experimental_use_latest_role_message_only=latest_user_only
)
request: Final = {
"model": "gpt-5.4-mini",
"input": [
{"role": "user", "content": "Look up the order."},
{"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"},
copy.deepcopy(tool_output),
{"role": "user", "content": "Email jane@example.com the result."},
],
}
result: Final = await OpenAIResponsesHandler().process_input_messages(request, guardrail)
expected_texts: Final = (
[["Email jane@example.com the result."]]
if latest_user_only
else [["Look up the order.", "Email jane@example.com the result."]]
)
assert sent_texts == expected_texts
assert result["input"][3]["content"] == "Email {EMAIL} the result."
assert result["input"][2] == tool_output
@pytest.mark.asyncio
async def test_bedrock_latest_user_masking_applies_with_a_multi_part_tool_output(self) -> None:
sent_texts: Final[list[list[str]]] = []
def bedrock_masking_emails(request: httpx.Request) -> httpx.Response:
texts: Final = _bedrock_block_texts(request)
sent_texts.append(texts)
outputs: Final = [{"text": text.replace("jane@example.com", "{EMAIL}")} for text in texts]
return httpx.Response(200, json={"action": "GUARDRAIL_INTERVENED", "outputs": outputs})
guardrail: Final = _bedrock_guardrail(
"bedrock-pre", bedrock_masking_emails, experimental_use_latest_role_message_only=True
)
tool_output: Final = {
"type": "function_call_output",
"call_id": "call_1",
"output": [{"type": "input_text", "text": "order 42"}, {"type": "input_text", "text": "shipped"}],
}
request: Final = {
"model": "gpt-5.4-mini",
"input": [
{"role": "user", "content": "Look up the order."},
{"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"},
copy.deepcopy(tool_output),
{"role": "user", "content": "Email jane@example.com the result."},
],
}
result: Final = await OpenAIResponsesHandler().process_input_messages(request, guardrail)
assert sent_texts == [["Email jane@example.com the result."]]
assert result["input"][3]["content"] == "Email {EMAIL} the result."
assert result["input"][2] == tool_output
@pytest.mark.asyncio
async def test_bedrock_only_scan_new_messages_scans_only_the_new_tool_result(self) -> None:
sent_texts: Final[list[list[str]]] = []
def bedrock_allowing(request: httpx.Request) -> httpx.Response:
sent_texts.append(_bedrock_block_texts(request))
return httpx.Response(200, json={"action": "NONE", "outputs": []})
guardrail: Final = _bedrock_guardrail("bedrock-incremental", bedrock_allowing, only_scan_new_messages=True)
first_turn: Final = [
{"role": "user", "content": "Weather in Paris and Rome?"},
{"type": "function_call", "call_id": "call_1", "name": "get_weather", "arguments": '{"city": "Paris"}'},
{"type": "function_call_output", "call_id": "call_1", "output": "Paris: sunny"},
]
second_turn: Final = [
*first_turn,
{"type": "function_call", "call_id": "call_2", "name": "get_weather", "arguments": '{"city": "Rome"}'},
{"type": "function_call_output", "call_id": "call_2", "output": "Rome: rain"},
]
session: Final = {"litellm_session_id": f"session-{uuid.uuid4()}"}
for turn in (first_turn, second_turn):
await OpenAIResponsesHandler().process_input_messages(
{"model": "gpt-5.4-mini", "input": copy.deepcopy(turn), **session}, guardrail
)
assert sent_texts == [["Weather in Paris and Rome?", "Paris: sunny"], ["Rome: rain"]]
@pytest.mark.asyncio
async def test_file_id_screenshot_is_not_scanned_as_text_or_rewritten(self) -> None:
guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test")
screenshot_output: Final = {
"type": "computer_call_output",
"call_id": "cu_1",
"output": {"type": "computer_screenshot", "file_id": "file-SECRET-screen"},
}
request: Final = {
"model": "gpt-5.4-mini",
"input": [
{"role": "user", "content": "Open the page"},
{"type": "computer_call", "call_id": "cu_1", "id": "cu_1", "action": {"type": "screenshot"}},
copy.deepcopy(screenshot_output),
],
}
result: Final = await OpenAIResponsesHandler().process_input_messages(request, guardrail)
assert guardrail.seen_texts == [["Open the page"]]
assert result["input"][2] == screenshot_output
@pytest.mark.asyncio
async def test_text_parts_between_file_images_are_rewritten_in_place(self) -> None:
guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test")
call_item, output_type = _TOOL_TURNS[0]
logo_part: Final = {"type": "input_image", "file_id": "file-logo"}
image_part: Final = {"type": "input_image", "file_id": "file-map"}
parts: Final = [
logo_part,
{"type": "input_text", "text": "SECRET forecast: "},
image_part,
{"type": "input_text", "text": "sunny"},
]
result: Final = await OpenAIResponsesHandler().process_input_messages(
_tool_result_request(call_item, output_type, parts), guardrail
)
assert guardrail.seen_texts == [["Weather?", "SECRET forecast: ", "sunny"]]
assert result["input"][2]["output"] == [
logo_part,
{"type": "input_text", "text": "SECRET forecast: [GUARDRAILED]"},
image_part,
{"type": "input_text", "text": "sunny [GUARDRAILED]"},
]
@pytest.mark.asyncio
async def test_dict_output_is_rewritten_as_the_text_the_guardrail_returns(self) -> None:
guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test")
call_item, output_type = _TOOL_TURNS[0]
result: Final = await OpenAIResponsesHandler().process_input_messages(
_tool_result_request(call_item, output_type, {"result": "SECRET forecast"}), guardrail
)
assert result["input"][2]["output"] == '{"result": "SECRET forecast"} [GUARDRAILED]'
@pytest.mark.asyncio
async def test_dict_output_the_guardrail_leaves_unchanged_stays_a_dict(self) -> None:
guardrail: Final = TextsReplacingGuardrail(guardrail_name="test", texts=("[MASKED]", '{"result": "sunny"}'))
call_item, output_type = _TOOL_TURNS[0]
result: Final = await OpenAIResponsesHandler().process_input_messages(
_tool_result_request(call_item, output_type, {"result": "sunny"}), guardrail
)
assert result["input"][0]["content"] == "[MASKED]"
assert result["input"][2]["output"] == {"result": "sunny"}
@pytest.mark.asyncio
async def test_tool_result_without_call_id_is_left_out_like_its_structured_row(self) -> None:
guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test")
call_item, output_type = _TOOL_TURNS[0]
orphan: Final = {"type": output_type, "output": "SECRET orphan"}
request: Final = {
"model": "gpt-5.4-mini",
"input": [
{"role": "user", "content": "Weather?"},
call_item,
dict(orphan),
{"type": output_type, "call_id": "call_1", "output": "SECRET forecast"},
],
}
result: Final = await OpenAIResponsesHandler().process_input_messages(request, guardrail)
assert guardrail.seen_texts == [["Weather?", "SECRET forecast"]]
assert "SECRET orphan" not in guardrail.seen_message_contents[0]
assert result["input"][2] == orphan
assert result["input"][3]["output"] == "SECRET forecast [GUARDRAILED]"
@pytest.mark.asyncio
async def test_a_textless_output_sharing_a_call_id_keeps_the_text_output_in_structured_messages(self) -> None:
guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test")
call_item, output_type = _TOOL_TURNS[0]
request: Final = _tool_result_request(call_item, output_type, "")
request["input"].append({"type": output_type, "call_id": "call_1", "output": "SECRET forecast"})
result: Final = await OpenAIResponsesHandler().process_input_messages(request, guardrail)
assert guardrail.seen_texts == [["Weather?", "SECRET forecast"]]
assert "SECRET forecast" in guardrail.seen_message_contents[0]
assert "" not in guardrail.seen_message_contents[0]
assert result["input"][3]["output"] == "SECRET forecast [GUARDRAILED]"
@pytest.mark.asyncio
async def test_multi_part_outputs_sharing_a_call_id_each_keep_their_own_text_in_structured_messages(
self,
) -> None:
guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test")
call_item, output_type = _TOOL_TURNS[0]
request: Final = _tool_result_request(
call_item,
output_type,
[{"type": "input_text", "text": "first "}, {"type": "input_text", "text": "output"}],
)
request["input"].append(
{
"type": output_type,
"call_id": "call_1",
"output": [{"type": "input_text", "text": "second "}, {"type": "input_text", "text": "output"}],
}
)
result: Final = await OpenAIResponsesHandler().process_input_messages(request, guardrail)
assert guardrail.seen_texts == [["Weather?", "first ", "output", "second ", "output"]]
assert guardrail.seen_message_contents[0][2:] == [
[{"type": "text", "text": "first "}, {"type": "text", "text": "output"}],
[{"type": "text", "text": "second "}, {"type": "text", "text": "output"}],
]
assert [part["text"] for part in result["input"][3]["output"]] == [
"second [GUARDRAILED]",
"output [GUARDRAILED]",
]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"textless_output",
[
{"type": "function_call_output", "call_id": "call_1", "output": ""},
{"type": "function_call_output", "call_id": "call_1", "output": None},
{
"type": "computer_call_output",
"call_id": "call_1",
"output": {"type": "computer_screenshot", "file_id": "file-screen"},
},
],
ids=["empty_string", "null", "file_id_screenshot"],
)
async def test_multi_part_output_sharing_a_call_id_with_a_textless_output_lists_its_parts_in_structured_messages(
self, textless_output: dict[str, object]
) -> None:
guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test")
call_item, output_type = _TOOL_TURNS[0]
request: Final = _tool_result_request(
call_item,
output_type,
[{"type": "input_text", "text": "first "}, {"type": "input_text", "text": "output"}],
)
request["input"].append(copy.deepcopy(textless_output))
await OpenAIResponsesHandler().process_input_messages(request, guardrail)
assert guardrail.seen_texts == [["Weather?", "first ", "output"]]
assert guardrail.seen_message_contents[0][2:] == [
[{"type": "text", "text": "first "}, {"type": "text", "text": "output"}],
]
@pytest.mark.asyncio
async def test_tool_result_item_is_scanned_like_function_call_output(self) -> None:
guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test")
call_item, _ = _TOOL_TURNS[0]
result: Final = await OpenAIResponsesHandler().process_input_messages(
_tool_result_request(call_item, "tool_result", "SECRET forecast"), guardrail
)
assert guardrail.seen_texts == [["Weather?", "SECRET forecast"]]
assert guardrail.seen_message_contents[0][2] == "SECRET forecast"
assert result["input"][2]["output"] == "SECRET forecast [GUARDRAILED]"
@pytest.mark.asyncio
async def test_text_only_parts_the_guardrail_leaves_unchanged_keep_their_parts(self) -> None:
call_item, output_type = _TOOL_TURNS[0]
parts: Final = [{"type": "input_text", "text": "SECRET forecast: "}, {"type": "input_text", "text": "sunny"}]
result: Final = await OpenAIResponsesHandler().process_input_messages(
_tool_result_request(call_item, output_type, copy.deepcopy(parts)), CustomGuardrail(guardrail_name="test")
)
assert result["input"][2]["output"] == parts
@pytest.mark.asyncio
async def test_text_parts_a_structured_guardrail_returns_unchanged_keep_their_parts(self) -> None:
class StructuredEchoGuardrail(CustomGuardrail):
async def apply_guardrail(
self,
inputs: GenericGuardrailAPIInputs,
request_data: dict[str, object],
input_type: Literal["request", "response"],
logging_obj: LiteLLMLoggingObj | None = None,
) -> GenericGuardrailAPIInputs:
return {**inputs, "structured_messages": copy.deepcopy(inputs.get("structured_messages") or [])}
call_item, output_type = _TOOL_TURNS[0]
parts: Final = [
{"type": "output_text", "text": "SECRET forecast: ", "annotations": []},
{"type": "output_text", "text": "sunny"},
]
result: Final = await OpenAIResponsesHandler().process_input_messages(
_tool_result_request(call_item, output_type, copy.deepcopy(parts)),
StructuredEchoGuardrail(guardrail_name="t"),
)
assert result["input"][2]["output"] == parts
@pytest.mark.asyncio
@pytest.mark.parametrize(
("output", "expected_texts"),
[
pytest.param("SECRET forecast", ["SECRET forecast"], id="string"),
pytest.param([{"type": "input_text", "text": "SECRET forecast"}], ["SECRET forecast"], id="one_part"),
pytest.param(
[{"type": "input_text", "text": "SECRET "}, {"type": "output_text", "text": "forecast"}],
["SECRET ", "forecast"],
id="parts",
),
pytest.param(
[
{"type": "input_text", "text": "SECRET "},
{"type": "input_image", "image_url": "https://example.com/map.png"},
{"type": "input_text", "text": ""},
{"type": "input_text", "text": "forecast"},
],
["SECRET ", "forecast"],
id="parts_with_image",
),
pytest.param(
[
{"type": "input_text", "text": "SECRET forecast"},
{"type": "summary_text", "text": "not tool output"},
],
["SECRET forecast"],
id="non_text_part_with_text_field",
),
pytest.param(
["SECRET forecast", {"type": "input_text", "text": "visible"}],
["visible"],
id="non_dict_part",
),
pytest.param([], ["[]"], id="empty"),
pytest.param({"result": "SECRET forecast"}, ['{"result": "SECRET forecast"}'], id="dict"),
pytest.param(
[{"type": "input_image", "file_id": "file-map"}],
['[{"type": "input_image", "file_id": "file-map"}]'],
id="no_text_parts",
),
pytest.param({"type": "input_image", "image_url": "https://example.com/map.png"}, [], id="single_image"),
],
)
async def test_tool_result_texts_match_the_structured_tool_message(
self, output: object, expected_texts: list[str]
) -> None:
guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test")
call_item, output_type = _TOOL_TURNS[0]
await OpenAIResponsesHandler().process_input_messages(
_tool_result_request(call_item, output_type, output), guardrail
)
tool_content: Final = guardrail.seen_message_contents[0][2]
structured_texts: Final = (
[tool_content]
if isinstance(tool_content, str)
else [part["text"] for part in tool_content if part.get("type") == "text"]
)
tool_texts: Final = guardrail.seen_texts[0][1:]
assert tool_texts == expected_texts
assert tool_texts == structured_texts
@pytest.mark.asyncio
@pytest.mark.parametrize(("call_item", "output_type"), _TOOL_TURNS, ids=["function", "custom"])
async def test_skip_tool_message_keeps_the_tool_result_out_of_texts_and_rows(
self, call_item: dict[str, str], output_type: str
) -> None:
guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test")
guardrail.skip_tool_message_in_guardrail = True
tool_result: Final = {"type": output_type, "call_id": "call_1", "output": "SECRET forecast"}
result: Final = await OpenAIResponsesHandler().process_input_messages(
_tool_result_request(call_item, output_type, "SECRET forecast"), guardrail
)
assert guardrail.seen_texts == [["Weather?"]]
assert "SECRET forecast" not in guardrail.seen_message_contents[0]
assert result["input"][0] == {"role": "user", "content": "Weather? [GUARDRAILED]"}
assert result["input"][2] == tool_result
@pytest.mark.asyncio
@pytest.mark.parametrize(("call_item", "output_type"), _TOOL_TURNS, ids=["function", "custom"])
async def test_scan_only_tool_results_sends_only_the_tool_result(
self, call_item: dict[str, str], output_type: str
) -> None:
guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test")
guardrail.scan_only_tool_results = True
tool: Final = {"type": "function", "name": "get_weather", "parameters": {"type": "object", "properties": {}}}
request: Final = {
**_tool_result_request(call_item, output_type, "SECRET forecast"),
"instructions": "Answer briefly",
"tools": [tool],
}
result: Final = await OpenAIResponsesHandler().process_input_messages(request, guardrail)
assert guardrail.seen_texts == [["SECRET forecast"]]
assert guardrail.seen_message_contents == [["SECRET forecast"]]
assert guardrail.seen_tools == [None]
assert result["instructions"] == "Answer briefly"
assert result["tools"] == [tool]
assert result["input"] == [
{"role": "user", "content": "Weather?"},
call_item,
{"type": output_type, "call_id": "call_1", "output": "SECRET forecast [GUARDRAILED]"},
]
@pytest.mark.asyncio
async def test_scan_only_tool_results_appends_tools_the_guardrail_adds(self) -> None:
added_tool: Final = {
"type": "function",
"function": {"name": "retrieve", "description": "Fetch the full text", "parameters": {"type": "object"}},
}
renamed_original: Final = {
"type": "function",
"function": {"name": "get_weather", "description": "rewritten", "parameters": {"type": "object"}},
}
class ToolAddingGuardrail(RecordingMaskingGuardrail):
async def apply_guardrail(
self,
inputs: GenericGuardrailAPIInputs,
request_data: dict[str, object],
input_type: Literal["request", "response"],
logging_obj: LiteLLMLoggingObj | None = None,
) -> GenericGuardrailAPIInputs:
masked: Final = await super().apply_guardrail(inputs, request_data, input_type, logging_obj)
return {**masked, "tools": [renamed_original, added_tool]}
guardrail: Final = ToolAddingGuardrail(guardrail_name="test")
guardrail.scan_only_tool_results = True
call_item, output_type = _TOOL_TURNS[0]
tool: Final = {"type": "function", "name": "get_weather", "parameters": {"type": "object", "properties": {}}}
request: Final = {**_tool_result_request(call_item, output_type, "SECRET forecast"), "tools": [tool]}
result: Final = await OpenAIResponsesHandler().process_input_messages(request, guardrail)
assert guardrail.seen_tools == [None]
assert result["tools"][0] == tool
assert [(t["type"], t["name"]) for t in result["tools"]] == [
("function", "get_weather"),
("function", "retrieve"),
]
assert result["tools"][1]["description"] == "Fetch the full text"
@pytest.mark.asyncio
async def test_scan_only_tool_results_leaves_a_request_without_tool_results_unscanned(self) -> None:
guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test")
guardrail.scan_only_tool_results = True
request: Final = {"model": "gpt-5.4-mini", "instructions": "Answer briefly", "input": "Weather?"}
result: Final = await OpenAIResponsesHandler().process_input_messages(dict(request), guardrail)
assert guardrail.seen_texts == []
assert result == request
class TestOpenAIResponsesHandlerOutputProcessing:
"""Test output processing functionality"""

View file

@ -12,8 +12,11 @@ from pydantic import JsonValue, TypeAdapter
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig
from litellm.llms.azure_ai.responses.transformation import AzureAIResponsesAPIConfig
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
from litellm.llms.hosted_vllm.responses.transformation import HostedVLLMResponsesAPIConfig
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.llms.xai.responses.transformation import XAIResponsesAPIConfig
from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig
from litellm.types.llms.openai import (
ImageGenerationPartialImageEvent,
@ -4277,3 +4280,116 @@ def test_dict_responses_input_filters_unset_reasoning_fields() -> None:
function_call_item: Final = validated_input[2]
assert function_call_item["type"] == "function_call"
assert function_call_item["status"] == "completed"
_COMPACT_CONFIGS: Final = (
pytest.param(OpenAIResponsesAPIConfig(), "https://api.openai.com/v1/responses", id="openai"),
pytest.param(
AzureOpenAIResponsesAPIConfig(),
"https://x.openai.azure.com/openai/responses?api-version=2025-04-01-preview",
id="azure",
),
pytest.param(AzureAIResponsesAPIConfig(), "https://x.services.ai.azure.com/openai/v1/responses", id="azure_ai"),
)
@pytest.mark.parametrize(("config", "api_base"), _COMPACT_CONFIGS)
def test_compact_request_omits_metadata_and_user_and_keeps_the_rest_without_drop_params(
config: OpenAIResponsesAPIConfig, api_base: str
) -> None:
_, data = config.transform_compact_response_api_request(
model="gpt-5.4-mini",
input="hi",
response_api_optional_request_params={
"metadata": {"ticket": "qa-1"},
"user": "end-user-1",
"instructions": "be brief",
"previous_response_id": "resp_1",
"reasoning": {"effort": "low"},
"temperature": 0.2,
},
api_base=api_base,
litellm_params=GenericLiteLLMParams(drop_params=False),
headers={},
)
assert {key: data[key] for key in data if key != "input"} == {
"model": "gpt-5.4-mini",
"instructions": "be brief",
"previous_response_id": "resp_1",
"reasoning": {"effort": "low"},
"temperature": 0.2,
}
@pytest.mark.parametrize(("config", "api_base"), _COMPACT_CONFIGS)
def test_compact_request_drops_params_compact_rejects_under_drop_params(
config: OpenAIResponsesAPIConfig, api_base: str
) -> None:
_, data = config.transform_compact_response_api_request(
model="gpt-5.4-mini",
input="hi",
response_api_optional_request_params={
"metadata": {"ticket": "qa-1"},
"user": "end-user-1",
"temperature": 0.2,
"store": False,
"truncation": "auto",
"max_output_tokens": 100,
"tool_choice": "auto",
"reasoning": {"effort": "low"},
"include": ["reasoning.encrypted_content"],
"text": {"format": {"type": "text"}},
},
api_base=api_base,
litellm_params=GenericLiteLLMParams(drop_params=True),
headers={},
)
assert {key: data[key] for key in data if key != "input"} == {
"model": "gpt-5.4-mini",
"reasoning": {"effort": "low"},
"include": ["reasoning.encrypted_content"],
"text": {"format": {"type": "text"}},
}
@pytest.mark.parametrize(
("config", "api_base"),
(
pytest.param(XAIResponsesAPIConfig(), "https://api.x.ai/v1/responses", id="xai"),
pytest.param(HostedVLLMResponsesAPIConfig(), "http://vllm.example.com/v1/responses", id="hosted_vllm"),
),
)
def test_compact_request_forwards_every_param_for_providers_not_known_to_reject_them(
config: OpenAIResponsesAPIConfig, api_base: str
) -> None:
_, data = config.transform_compact_response_api_request(
model="some-model",
input="hi",
response_api_optional_request_params={"metadata": {"ticket": "qa-1"}, "user": "end-user-1", "temperature": 0.2},
api_base=api_base,
litellm_params=GenericLiteLLMParams(drop_params=True),
headers={},
)
assert {key: data[key] for key in data if key != "input"} == {
"model": "some-model",
"metadata": {"ticket": "qa-1"},
"user": "end-user-1",
"temperature": 0.2,
}
@pytest.mark.parametrize(("config", "api_base"), _COMPACT_CONFIGS)
def test_compact_request_keeps_stream_under_drop_params(config: OpenAIResponsesAPIConfig, api_base: str) -> None:
_, data = config.transform_compact_response_api_request(
model="gpt-5.4-mini",
input="hi",
response_api_optional_request_params={"stream": True, "stream_options": {"include_obfuscation": False}},
api_base=api_base,
litellm_params=GenericLiteLLMParams(drop_params=True),
headers={},
)
assert {key: data[key] for key in data if key != "input"} == {"model": "gpt-5.4-mini", "stream": True}

View file

@ -0,0 +1,402 @@
import json
from pathlib import Path
from typing import Final
import httpx
import pytest
import respx
import litellm
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.llms.openai_like.dynamic_config import create_config_class
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
from litellm.types.utils import ModelResponse, Usage
STREAMLAKE_BASE_URL: Final = "https://vanchin.streamlake.ai/api/gateway/v1/endpoints"
STREAMLAKE_CHAT_URL: Final = f"{STREAMLAKE_BASE_URL}/chat/completions"
REPO_ROOT: Final = Path(__file__).parents[4]
class TestStreamlakeProviderConfig:
def test_streamlake_provider_resolution(self) -> None:
model, provider, api_key, api_base = get_llm_provider(
model="streamlake/ep-test-endpoint",
custom_llm_provider=None,
api_base=None,
api_key="sk-test",
)
assert model == "ep-test-endpoint"
assert provider == "streamlake"
assert api_base == STREAMLAKE_BASE_URL
assert api_key == "sk-test"
def test_streamlake_api_base_override(self) -> None:
model, provider, api_key, api_base = get_llm_provider(
model="streamlake/ep-test-endpoint",
custom_llm_provider=None,
api_base="https://custom.streamlake.example/v1",
api_key="sk-test",
)
assert model == "ep-test-endpoint"
assert provider == "streamlake"
assert api_base == "https://custom.streamlake.example/v1"
assert api_key == "sk-test"
def test_streamlake_url_autodetection(self) -> None:
_, provider, _, api_base = get_llm_provider(
model="some-public-model",
custom_llm_provider=None,
api_base=STREAMLAKE_BASE_URL,
api_key=None,
)
assert provider == "streamlake"
assert api_base == STREAMLAKE_BASE_URL
def test_streamlake_api_key_resolved_from_env(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("STREAMLAKE_API_KEY", "sk-env-key")
_, provider, api_key, _ = get_llm_provider(
model="streamlake/ep-test-endpoint",
custom_llm_provider=None,
api_base=None,
api_key=None,
)
assert provider == "streamlake"
assert api_key == "sk-env-key"
def test_streamlake_validate_environment_sets_bearer(self) -> None:
provider: Final = JSONProviderRegistry.get("streamlake")
assert provider is not None
config: Final = create_config_class(provider)()
headers: Final = config.validate_environment(
headers={},
model="ep-test-endpoint",
messages=[{"role": "user", "content": "hi"}],
optional_params={},
litellm_params={},
api_key="sk-resolved-key",
api_base=None,
)
assert headers["Authorization"] == "Bearer sk-resolved-key"
def test_streamlake_complete_url_appends_chat_completions(self) -> None:
provider: Final = JSONProviderRegistry.get("streamlake")
assert provider is not None
config: Final = create_config_class(provider)()
url: Final = config.get_complete_url(
api_base=None,
api_key="sk-test",
model="ep-test-endpoint",
optional_params={},
litellm_params={},
)
assert url == STREAMLAKE_CHAT_URL
@pytest.mark.parametrize("cap_param", ("max_tokens", "max_completion_tokens"))
def test_streamlake_token_cap_sent_as_max_completion_tokens(self, cap_param: str) -> None:
provider: Final = JSONProviderRegistry.get("streamlake")
assert provider is not None
config: Final = create_config_class(provider)()
optional_params: Final = config.map_openai_params(
non_default_params={cap_param: 1024},
optional_params={},
model="ep-test-endpoint",
drop_params=False,
)
assert optional_params == {"max_completion_tokens": 1024}
def test_streamlake_chat_params_pass_through(self) -> None:
provider: Final = JSONProviderRegistry.get("streamlake")
assert provider is not None
config: Final = create_config_class(provider)()
optional_params: Final = config.map_openai_params(
non_default_params={"temperature": 0.2, "top_p": 0.9, "stream": True},
optional_params={},
model="ep-test-endpoint",
drop_params=False,
)
assert optional_params == {"temperature": 0.2, "top_p": 0.9, "stream": True}
@pytest.mark.asyncio
@pytest.mark.parametrize(
"is_async, api_base",
(
(False, None),
(False, "https://custom.streamlake.example/v1"),
(False, "https://custom.streamlake.example/v1/"),
(True, None),
(True, "https://custom.streamlake.example/v1"),
(True, "https://custom.streamlake.example/v1/"),
),
ids=(
"sync-default",
"sync-override",
"sync-override-trailing-slash",
"async-default",
"async-override",
"async-override-trailing-slash",
),
)
async def test_streamlake_completion_uses_openai_compatible_http(
is_async: bool, api_base: str | None, monkeypatch: pytest.MonkeyPatch
) -> None:
response_body: Final[dict[str, object]] = {
"id": "chatcmpl-streamlake-test",
"object": "chat.completion",
"created": 1710000000,
"model": "ep-test-endpoint",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "StreamLake response"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5},
}
expected_url: Final[str] = f"{(api_base or STREAMLAKE_BASE_URL).rstrip('/')}/chat/completions"
expected_request_body: Final[dict[str, object]] = {
"messages": [{"role": "user", "content": "hello"}],
"model": "ep-test-endpoint",
"max_completion_tokens": 512,
}
def handler(request: httpx.Request) -> httpx.Response:
assert request.method == "POST"
assert str(request.url) == expected_url
assert request.headers["authorization"] == "Bearer sk-streamlake-test"
assert request.headers["content-type"] == "application/json"
assert json.loads(request.content) == expected_request_body
return httpx.Response(200, json=response_body, request=request)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
with respx.mock(assert_all_called=True) as respx_mock:
route: Final = respx_mock.post(expected_url).mock(side_effect=handler)
if is_async:
response: Final = await litellm.acompletion(
model="streamlake/ep-test-endpoint",
messages=[{"role": "user", "content": "hello"}],
api_key="sk-streamlake-test",
api_base=api_base,
max_tokens=512,
)
else:
response: Final = litellm.completion(
model="streamlake/ep-test-endpoint",
messages=[{"role": "user", "content": "hello"}],
api_key="sk-streamlake-test",
api_base=api_base,
max_tokens=512,
)
assert route.call_count == 1
assert len(response.choices) == 1
assert response.choices[0].message.role == "assistant"
assert response.choices[0].message.content == "StreamLake response"
assert response.usage is not None
assert response.usage.prompt_tokens == 3
assert response.usage.completion_tokens == 2
assert response.usage.total_tokens == 5
@pytest.mark.usefixtures("local_model_cost_map")
def test_endpoint_id_call_forwards_tools_and_reasoning_effort_to_streamlake(monkeypatch: pytest.MonkeyPatch) -> None:
tools: Final = [
{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {"type": "object", "properties": {"city": {"type": "string"}}},
},
}
]
response_body: Final[dict[str, object]] = {
"id": "chatcmpl-streamlake-tools",
"object": "chat.completion",
"created": 1710000000,
"model": "glm-5.3",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4},
}
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
with respx.mock(assert_all_called=True) as respx_mock:
route: Final = respx_mock.post(STREAMLAKE_CHAT_URL).mock(return_value=httpx.Response(200, json=response_body))
litellm.completion(
model="streamlake/ep-test-endpoint",
messages=[{"role": "user", "content": "weather in Paris?"}],
api_key="sk-streamlake-test",
tools=tools,
tool_choice="auto",
reasoning_effort="low",
)
sent: Final = json.loads(route.calls.last.request.content)
assert sent["tools"] == tools
assert sent["tool_choice"] == "auto"
assert sent["reasoning_effort"] == "low"
def _streamlake_keys(cost_map: dict[str, object]) -> tuple[str, ...]:
return tuple(key for key in cost_map if key.startswith("streamlake/"))
def test_root_and_backup_cost_maps_carry_the_same_streamlake_rows() -> None:
root: Final = json.loads((REPO_ROOT / "model_prices_and_context_window.json").read_text())
backup: Final = json.loads((REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json").read_text())
assert _streamlake_keys(root)
assert {key: root[key] for key in _streamlake_keys(root)} == {key: backup[key] for key in _streamlake_keys(backup)}
@pytest.mark.usefixtures("local_model_cost_map")
def test_endpoint_id_call_is_priced_from_the_lowercase_model_streamlake_returns() -> None:
keys: Final = _streamlake_keys(litellm.model_cost)
assert keys
for key in keys:
info: Final = litellm.get_model_info(model=key)
response: Final = ModelResponse(
model=key.split("/", 1)[1].lower(),
usage=Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500),
)
cost: Final = litellm.completion_cost(
completion_response=response,
model="streamlake/ep-test-endpoint",
custom_llm_provider="streamlake",
)
expected: Final = 1000 * info["input_cost_per_token"] + 500 * info["output_cost_per_token"]
assert cost == pytest.approx(expected), key
@pytest.mark.usefixtures("local_model_cost_map")
def test_named_model_params_follow_the_row_and_fall_back_to_the_provider_default() -> None:
provider: Final = JSONProviderRegistry.get("streamlake")
assert provider is not None
config: Final = create_config_class(provider)()
keys: Final = _streamlake_keys(litellm.model_cost)
assert keys
for key in keys:
row: Final = litellm.model_cost[key]
supported: Final = config.get_supported_openai_params(model=key.split("/", 1)[1].lower())
assert ("tools" in supported) is (row.get("supports_function_calling") is not False), key
assert ("reasoning_effort" in supported) is (row.get("supports_reasoning") is not False), key
@pytest.mark.usefixtures("local_model_cost_map")
def test_streamlake_wildcard_expands_to_every_streamlake_row() -> None:
expanded: Final = get_known_models_from_wildcard("streamlake/*")
assert set(expanded) == set(_streamlake_keys(litellm.model_cost))
@pytest.mark.usefixtures("local_model_cost_map")
def test_cache_control_marks_reach_streamlake(monkeypatch: pytest.MonkeyPatch) -> None:
system_block: Final = {"type": "text", "text": "You are terse.", "cache_control": {"type": "ephemeral"}}
tool: Final = {
"type": "function",
"function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}},
"cache_control": {"type": "ephemeral"},
}
response_body: Final[dict[str, object]] = {
"id": "chatcmpl-streamlake-cache",
"object": "chat.completion",
"created": 1710000000,
"model": "qwen3.8-27b",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4},
}
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
with respx.mock(assert_all_called=True) as respx_mock:
route: Final = respx_mock.post(STREAMLAKE_CHAT_URL).mock(return_value=httpx.Response(200, json=response_body))
litellm.completion(
model="streamlake/qwen3.8-27b",
messages=[{"role": "system", "content": [system_block]}, {"role": "user", "content": "hi"}],
api_key="sk-streamlake-test",
tools=[tool],
)
sent: Final = json.loads(route.calls.last.request.content)
assert sent["messages"][0]["content"] == [system_block]
assert sent["tools"][0]["cache_control"] == {"type": "ephemeral"}
@pytest.mark.usefixtures("local_model_cost_map")
@pytest.mark.parametrize("model", ("kimi-k2.6", "qwen3.6-27b", "qwen3.6-35b-a3b"))
def test_reasoning_effort_is_dropped_for_models_streamlake_lists_without_reasoning(
model: str, monkeypatch: pytest.MonkeyPatch
) -> None:
# StreamLake's Models Overview (https://www.streamlake.ai/document/DOC/mh1gbfvrdn6hpbzxixv, read 2026-10-28)
# gives these models no Reasoning tag, and kimi-k2.6 answers 400 to reasoning_effort under an 8193 token cap
response_body: Final[dict[str, object]] = {
"id": "chatcmpl-streamlake-reasoning",
"object": "chat.completion",
"created": 1710000000,
"model": model,
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4},
}
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
with respx.mock(assert_all_called=True) as respx_mock:
route: Final = respx_mock.post(STREAMLAKE_CHAT_URL).mock(return_value=httpx.Response(200, json=response_body))
litellm.completion(
model=f"streamlake/{model}",
messages=[{"role": "user", "content": "hi"}],
api_key="sk-streamlake-test",
reasoning_effort="low",
drop_params=True,
)
assert "reasoning_effort" not in json.loads(route.calls.last.request.content)
with pytest.raises(litellm.UnsupportedParamsError):
litellm.completion(
model=f"streamlake/{model}",
messages=[{"role": "user", "content": "hi"}],
api_key="sk-streamlake-test",
reasoning_effort="low",
)
@pytest.mark.usefixtures("local_model_cost_map")
def test_streamlake_rows_with_a_cache_read_price_declare_prompt_caching() -> None:
cached_rows: Final = tuple(
key for key in _streamlake_keys(litellm.model_cost) if "cache_read_input_token_cost" in litellm.model_cost[key]
)
assert cached_rows
assert [key for key in cached_rows if litellm.model_cost[key].get("supports_prompt_caching") is not True] == []
@pytest.mark.usefixtures("local_model_cost_map")
@pytest.mark.parametrize("model", ("qwen3.8-27b", "qwen3.8-2.4t-a95b"))
@pytest.mark.parametrize("cache_type", ("ephemeral", None))
def test_qwen3_8_explicit_cache_hits_and_prefix_hits_use_their_own_read_price(
model: str, cache_type: str | None
) -> None:
# StreamLake's price page (https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce, read 2026-10-28) lists
# separate PrefixCache and Cache Read prices for Qwen3.8, and its usage tags only explicit hits with cache_type
rate_key: Final = "explicit_cache_read_input_token_cost" if cache_type else "cache_read_input_token_cost"
row: Final = litellm.model_cost[f"streamlake/{model}"]
details: Final = {"cached_tokens": 4000} | ({"cache_type": cache_type} if cache_type else {})
response: Final = ModelResponse(
model=model,
usage=Usage(prompt_tokens=5000, completion_tokens=100, total_tokens=5100, prompt_tokens_details=details),
)
cost: Final = litellm.completion_cost(
completion_response=response, model=f"streamlake/{model}", custom_llm_provider="streamlake"
)
expected: Final = 4000 * row[rate_key] + 1000 * row["input_cost_per_token"] + 100 * row["output_cost_per_token"]
assert cost == pytest.approx(expected)

View file

@ -12,7 +12,7 @@ load_dotenv()
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
from copy import deepcopy
from types import SimpleNamespace
from typing import Final
from typing import Final, Literal
from unittest.mock import AsyncMock, MagicMock, patch
import pytest, litellm
@ -721,6 +721,51 @@ async def test_can_key_call_model_wildcard_access(key_models, model, expect_to_w
)
@pytest.fixture
def shipped_routing_rules() -> Iterator[None]:
from litellm.litellm_core_utils.fallback_generalizations import (
get_fallback_generalization_rules,
set_fallback_generalizations,
)
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
previous_rules: Final = list(get_fallback_generalization_rules())
set_fallback_generalizations(GetModelCostMap.load_local_model_cost_map()["fallback_generalizations"]["rules"])
yield
set_fallback_generalizations(previous_rules)
@pytest.mark.parametrize("object_type", ["team", "project"])
@pytest.mark.parametrize(
"allowed_models, model, expect_to_work",
[
(["typesafe/*"], "jev-latest", True),
(["typesafe/*"], "jev-1.14.0", True),
(["jev-latest"], "jev-latest", True),
(["openai/*"], "jev-latest", False),
(["typesafe/*"], "foo/jev-latest", False),
(["typesafe/*"], "jev-internal-router", False),
(["typesafe/*"], "jev-2.0-internal", False),
(["typesafe/*"], "jev-1.0", False),
],
)
def test_typesafe_provider_wildcard_covers_bare_jev_model(
shipped_routing_rules: None,
object_type: Literal["team", "project"],
allowed_models: list[str],
model: str,
expect_to_work: bool,
) -> None:
from litellm.proxy._types import ProxyException
from litellm.proxy.auth.auth_checks import can_object_call_model
if expect_to_work:
assert can_object_call_model(model=model, llm_router=None, models=allowed_models, object_type=object_type)
return
with pytest.raises(ProxyException):
can_object_call_model(model=model, llm_router=None, models=allowed_models, object_type=object_type)
@pytest.mark.parametrize(
"key_models, model, expect_to_work",
[

View file

@ -277,7 +277,7 @@ def _free_port() -> int:
def _fake_pooler(
tmp_path: Path,
directory: Path,
port: int,
exit_immediately: bool = False,
port_file: Path | None = None,
@ -288,13 +288,13 @@ def _fake_pooler(
"""An executable that listens like PgBouncer: on the TCP port first, then on ``.s.PGSQL.<port>`` in the socket dir.
Port and socket dir come from the ini it is given, else from ``port`` and
``tmp_path``. With ``port_file`` each start reads the port from that file
``directory``. With ``port_file`` each start reads the port from that file
instead. ``bind_delay_seconds`` holds the bind back, like a slow start.
``--version`` prints ``version_banner``. With ``auth_log`` it appends the
``auth_file`` it reads at startup and on every SIGHUP, one line per read,
like PgBouncer loading its credentials.
"""
script: Final = tmp_path / "fake-pgbouncer"
script: Final = directory / "fake-pgbouncer"
script.write_text(
textwrap.dedent(
f"""\
@ -316,7 +316,7 @@ def _fake_pooler(
port = ini.getint("pgbouncer", "listen_port", fallback={port})
if not {port_file is None!r}:
port = int(pathlib.Path({str(port_file)!r}).read_text())
socket_dir = ini.get("pgbouncer", "unix_socket_dir", fallback={str(tmp_path)!r})
socket_dir = ini.get("pgbouncer", "unix_socket_dir", fallback={str(directory)!r})
socket_path = f"{{socket_dir}}/.s.PGSQL.{{port}}"
time.sleep({bind_delay_seconds!r})
listener = socket.socket()
@ -357,10 +357,10 @@ def _wait_until(condition: Callable[[], bool], timeout_seconds: float = 5.0) ->
class TestPgBouncerProcess:
def test_start_waits_for_the_listener_and_stop_ends_it(self, tmp_path: Path):
def test_start_waits_for_the_listener_and_stop_ends_it(self, socket_dir: Path):
port: Final = _free_port()
pooler: Final = PgBouncerProcess(
argv=(str(_fake_pooler(tmp_path, port)),), port=port, socket_path=unix_socket_path(tmp_path, port)
argv=(str(_fake_pooler(socket_dir, port)),), port=port, socket_path=unix_socket_path(socket_dir, port)
)
assert pooler.start() is None
assert _listening(port)
@ -371,12 +371,12 @@ class TestPgBouncerProcess:
with pytest.raises(ProcessLookupError):
os.kill(pid, 0)
def test_a_crashed_pooler_is_restarted_with_a_new_pid(self, tmp_path: Path):
def test_a_crashed_pooler_is_restarted_with_a_new_pid(self, socket_dir: Path):
port: Final = _free_port()
pooler: Final = PgBouncerProcess(
argv=(str(_fake_pooler(tmp_path, port)),),
argv=(str(_fake_pooler(socket_dir, port)),),
port=port,
socket_path=unix_socket_path(tmp_path, port),
socket_path=unix_socket_path(socket_dir, port),
restart_delay_seconds=0.1,
)
assert pooler.start() is None
@ -388,17 +388,17 @@ class TestPgBouncerProcess:
assert _wait_until(lambda: not _listening(port))
def test_a_failed_restart_is_retried_until_the_pooler_is_back(
self, tmp_path: Path, caplog: pytest.LogCaptureFixture
self, socket_dir: Path, caplog: pytest.LogCaptureFixture
):
port: Final = _free_port()
script: Final = _fake_pooler(tmp_path, port)
script: Final = _fake_pooler(socket_dir, port)
pooler: Final = PgBouncerProcess(
argv=(str(script),), port=port, socket_path=unix_socket_path(tmp_path, port), restart_delay_seconds=0.1
argv=(str(script),), port=port, socket_path=unix_socket_path(socket_dir, port), restart_delay_seconds=0.1
)
assert pooler.start() is None
first_pid: Final = pooler.pid
assert first_pid is not None
hidden: Final = script.rename(tmp_path / "hidden")
hidden: Final = script.rename(socket_dir / "hidden")
with caplog.at_level(logging.ERROR, logger=verbose_proxy_logger.name):
os.kill(first_pid, signal.SIGKILL)
assert _wait_until(lambda: any("could not be restarted" in record.message for record in caplog.records))
@ -408,15 +408,17 @@ class TestPgBouncerProcess:
pooler.stop()
assert _wait_until(lambda: not _listening(port))
def test_a_replacement_that_never_listens_is_replaced_again(self, tmp_path: Path, caplog: pytest.LogCaptureFixture):
def test_a_replacement_that_never_listens_is_replaced_again(
self, socket_dir: Path, caplog: pytest.LogCaptureFixture
):
port: Final = _free_port()
port_file: Final = tmp_path / "port"
port_file: Final = socket_dir / "port"
port_file.write_text(str(port))
script: Final = _fake_pooler(tmp_path, port, port_file=port_file)
script: Final = _fake_pooler(socket_dir, port, port_file=port_file)
pooler: Final = PgBouncerProcess(
argv=(str(script),),
port=port,
socket_path=unix_socket_path(tmp_path, port),
socket_path=unix_socket_path(socket_dir, port),
restart_delay_seconds=0.1,
ready_timeout_seconds=2.0,
)
@ -435,12 +437,12 @@ class TestPgBouncerProcess:
pooler.stop()
assert _wait_until(lambda: not _listening(port))
def test_stopping_during_the_restart_delay_leaves_no_pooler_behind(self, tmp_path: Path):
def test_stopping_during_the_restart_delay_leaves_no_pooler_behind(self, socket_dir: Path):
port: Final = _free_port()
pooler: Final = PgBouncerProcess(
argv=(str(_fake_pooler(tmp_path, port)),),
argv=(str(_fake_pooler(socket_dir, port)),),
port=port,
socket_path=unix_socket_path(tmp_path, port),
socket_path=unix_socket_path(socket_dir, port),
restart_delay_seconds=0.3,
)
assert pooler.start() is None
@ -453,12 +455,12 @@ class TestPgBouncerProcess:
assert not _listening(port)
assert pooler.pid == first_pid
def test_a_stopped_pooler_is_not_restarted(self, tmp_path: Path, caplog: pytest.LogCaptureFixture):
def test_a_stopped_pooler_is_not_restarted(self, socket_dir: Path, caplog: pytest.LogCaptureFixture):
port: Final = _free_port()
pooler: Final = PgBouncerProcess(
argv=(str(_fake_pooler(tmp_path, port)),),
argv=(str(_fake_pooler(socket_dir, port)),),
port=port,
socket_path=unix_socket_path(tmp_path, port),
socket_path=unix_socket_path(socket_dir, port),
restart_delay_seconds=0.1,
)
assert pooler.start() is None
@ -468,31 +470,31 @@ class TestPgBouncerProcess:
assert not _listening(port)
assert caplog.records == []
def test_a_pooler_that_exits_during_startup_is_reported(self, tmp_path: Path):
def test_a_pooler_that_exits_during_startup_is_reported(self, socket_dir: Path):
port: Final = _free_port()
pooler: Final = PgBouncerProcess(
argv=(str(_fake_pooler(tmp_path, port, exit_immediately=True)),),
argv=(str(_fake_pooler(socket_dir, port, exit_immediately=True)),),
port=port,
socket_path=unix_socket_path(tmp_path, port),
socket_path=unix_socket_path(socket_dir, port),
)
outcome: Final = pooler.start()
assert isinstance(outcome, PgBouncerError)
assert "status 3" in outcome.reason
def test_a_missing_binary_is_reported(self, tmp_path: Path):
def test_a_missing_binary_is_reported(self, socket_dir: Path):
outcome: Final = PgBouncerProcess(
argv=("/nonexistent/pgbouncer",), port=_free_port(), socket_path=tmp_path / "sock"
argv=("/nonexistent/pgbouncer",), port=_free_port(), socket_path=socket_dir / "sock"
).start()
assert isinstance(outcome, PgBouncerError)
assert "/nonexistent/pgbouncer" in outcome.reason
def test_a_port_owned_by_someone_else_is_refused_before_spawning(self, tmp_path: Path):
def test_a_port_owned_by_someone_else_is_refused_before_spawning(self, socket_dir: Path):
with socket.socket() as squatter:
squatter.bind(("127.0.0.1", 0))
squatter.listen()
port: Final = _bound_port(squatter)
pooler: Final = PgBouncerProcess(
argv=(str(_fake_pooler(tmp_path, port)),), port=port, socket_path=unix_socket_path(tmp_path, port)
argv=(str(_fake_pooler(socket_dir, port)),), port=port, socket_path=unix_socket_path(socket_dir, port)
)
outcome: Final = pooler.start()
assert isinstance(outcome, PgBouncerError)
@ -500,13 +502,13 @@ class TestPgBouncerProcess:
assert pooler.pid is None
def test_a_replacement_waits_until_a_squatter_leaves_the_port(
self, tmp_path: Path, caplog: pytest.LogCaptureFixture
self, socket_dir: Path, caplog: pytest.LogCaptureFixture
):
port: Final = _free_port()
pooler: Final = PgBouncerProcess(
argv=(str(_fake_pooler(tmp_path, port)),),
argv=(str(_fake_pooler(socket_dir, port)),),
port=port,
socket_path=unix_socket_path(tmp_path, port),
socket_path=unix_socket_path(socket_dir, port),
restart_delay_seconds=0.5,
)
assert pooler.start() is None
@ -524,12 +526,12 @@ class TestPgBouncerProcess:
pooler.stop()
assert _wait_until(lambda: not _listening(port))
def test_a_listener_that_grabs_the_port_after_the_spawn_is_not_taken_for_the_pooler(self, tmp_path: Path):
def test_a_listener_that_grabs_the_port_after_the_spawn_is_not_taken_for_the_pooler(self, socket_dir: Path):
port: Final = _free_port()
pooler: Final = PgBouncerProcess(
argv=(str(_fake_pooler(tmp_path, port, bind_delay_seconds=0.5)),),
argv=(str(_fake_pooler(socket_dir, port, bind_delay_seconds=0.5)),),
port=port,
socket_path=unix_socket_path(tmp_path, port),
socket_path=unix_socket_path(socket_dir, port),
ready_timeout_seconds=3.0,
)
with socket.socket() as squatter, ThreadPoolExecutor(max_workers=1) as starter:
@ -542,12 +544,12 @@ class TestPgBouncerProcess:
assert isinstance(outcome, PgBouncerError)
assert "exited with status 1" in outcome.reason
def test_a_port_served_by_a_stranger_while_the_pooler_is_still_starting_is_reported(self, tmp_path: Path):
def test_a_port_served_by_a_stranger_while_the_pooler_is_still_starting_is_reported(self, socket_dir: Path):
port: Final = _free_port()
pooler: Final = PgBouncerProcess(
argv=(str(_fake_pooler(tmp_path, port, bind_delay_seconds=30.0)),),
argv=(str(_fake_pooler(socket_dir, port, bind_delay_seconds=30.0)),),
port=port,
socket_path=unix_socket_path(tmp_path, port),
socket_path=unix_socket_path(socket_dir, port),
ready_timeout_seconds=0.5,
)
with socket.socket() as squatter, ThreadPoolExecutor(max_workers=1) as starter:
@ -564,12 +566,12 @@ class TestPgBouncerProcess:
with pytest.raises(ProcessLookupError):
os.kill(pid, 0)
def test_a_pooler_that_never_listens_times_out(self, tmp_path: Path):
def test_a_pooler_that_never_listens_times_out(self, socket_dir: Path):
port: Final = _free_port()
pooler: Final = PgBouncerProcess(
argv=(str(_fake_pooler(tmp_path, _free_port())),),
argv=(str(_fake_pooler(socket_dir, _free_port())),),
port=port,
socket_path=unix_socket_path(tmp_path, port),
socket_path=unix_socket_path(socket_dir, port),
ready_timeout_seconds=0.5,
)
outcome: Final = pooler.start()
@ -704,10 +706,10 @@ def _runtime_dir_listening_on(port: int) -> Path:
class TestStartInContainerPgBouncer:
def test_returns_the_loopback_url_once_the_pooler_listens(self, tmp_path: Path):
def test_returns_the_loopback_url_once_the_pooler_listens(self, socket_tempdir: Path):
port: Final = _free_port()
auth_log: Final = tmp_path / "auth.log"
binary: Final = _fake_pooler(tmp_path, port, auth_log=auth_log)
auth_log: Final = socket_tempdir / "auth.log"
binary: Final = _fake_pooler(socket_tempdir, port, auth_log=auth_log)
settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(binary))
pooled: Final = start_in_container_pgbouncer(settings, "postgresql://app:pw@db/litellm?connection_limit=5")
assert isinstance(pooled, str), pooled
@ -723,9 +725,9 @@ class TestStartInContainerPgBouncer:
assert auth_log.read_text() == repr(f'"app" "pw"\n"litellm_pgbouncer" "{parsed.password}"\n') + "\n"
@pytest.mark.filterwarnings("ignore:This process .* is multi-threaded:DeprecationWarning")
def test_a_forked_worker_exiting_leaves_the_pooler_and_its_files_to_the_parent(self, tmp_path: Path):
def test_a_forked_worker_exiting_leaves_the_pooler_and_its_files_to_the_parent(self, socket_tempdir: Path):
port: Final = _free_port()
settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pooler(tmp_path, port)))
settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pooler(socket_tempdir, port)))
exit_hooks: Final[list[Callable[[], None]]] = []
pooled: Final = start_in_container_pgbouncer(
settings, "postgresql://app:pw@db/litellm", register_exit_hook=exit_hooks.append
@ -766,10 +768,10 @@ class TestStartInContainerPgBouncer:
assert "IAM_TOKEN_DB_AUTH" in outcome.reason
assert not _listening(port)
def test_token_auth_mints_the_first_token_into_the_auth_file_before_the_pooler_starts(self, tmp_path: Path):
def test_token_auth_mints_the_first_token_into_the_auth_file_before_the_pooler_starts(self, socket_tempdir: Path):
port: Final = _free_port()
auth_log: Final = tmp_path / "auth.log"
binary: Final = _fake_pooler(tmp_path, port, auth_log=auth_log)
auth_log: Final = socket_tempdir / "auth.log"
binary: Final = _fake_pooler(socket_tempdir, port, auth_log=auth_log)
settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(binary))
token: Final = _entra_jwt(datetime.now(tz=timezone.utc) + timedelta(hours=1))
pooled: Final = start_in_container_pgbouncer(
@ -798,25 +800,25 @@ class TestStartInContainerPgBouncer:
assert "no Azure credential" in outcome.reason
assert not _listening(port)
def test_a_renewed_token_is_written_and_picked_up_by_the_running_and_by_a_restarted_pooler(self, tmp_path: Path):
def test_a_renewed_token_is_written_and_picked_up_by_the_running_and_by_a_restarted_pooler(self, socket_dir: Path):
port: Final = _free_port()
auth_log: Final = tmp_path / "auth.log"
auth_log: Final = socket_dir / "auth.log"
plan: Final = plan_pgbouncer(
"postgresql://app@db/litellm", PgBouncerSettings(enabled=True, port=port), tmp_path, None
"postgresql://app@db/litellm", PgBouncerSettings(enabled=True, port=port), socket_dir, None
)
assert isinstance(plan, PgBouncerPlan), plan
ini_path: Final = write_pgbouncer_ini(plan, tmp_path, None)
write_userlist(plan.userlist("first"), tmp_path, None)
ini_path: Final = write_pgbouncer_ini(plan, socket_dir, None)
write_userlist(plan.userlist("first"), socket_dir, None)
pooler: Final = PgBouncerProcess(
argv=(str(_fake_pooler(tmp_path, port, auth_log=auth_log)), str(ini_path)),
argv=(str(_fake_pooler(socket_dir, port, auth_log=auth_log)), str(ini_path)),
port=port,
socket_path=unix_socket_path(tmp_path, port),
socket_path=unix_socket_path(socket_dir, port),
restart_delay_seconds=0.1,
)
assert pooler.start() is None
first_pid: Final = pooler.pid
assert first_pid is not None
install_pgbouncer_token(plan, tmp_path, None, pooler, "second")
install_pgbouncer_token(plan, socket_dir, None, pooler, "second")
assert _wait_until(lambda: auth_log.read_text().count("\n") == 2)
os.kill(first_pid, signal.SIGKILL)
assert _wait_until(lambda: pooler.pid not in (None, first_pid) and _listening(port))
@ -837,9 +839,9 @@ class TestStartInContainerPgBouncer:
assert "1.19" in outcome.reason
assert not _listening(port)
def test_the_first_version_that_dies_on_a_failed_tcp_bind_is_accepted(self, tmp_path: Path):
def test_the_first_version_that_dies_on_a_failed_tcp_bind_is_accepted(self, socket_tempdir: Path):
port: Final = _free_port()
binary: Final = _fake_pooler(tmp_path, port, version_banner="PgBouncer 1.19.0")
binary: Final = _fake_pooler(socket_tempdir, port, version_banner="PgBouncer 1.19.0")
settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(binary))
pooled: Final = start_in_container_pgbouncer(settings, "postgresql://app:pw@db/litellm")
assert isinstance(pooled, str), pooled

View file

@ -4834,6 +4834,83 @@ class TestPanwAirsLatestRoleMessageOnlyEveryRequestShape:
None,
id="function_call_output",
),
pytest.param(
(
{"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"},
{
"type": "function_call_output",
"call_id": "call_1",
"output": [{"type": "input_text", "text": "tool "}, {"type": "input_text", "text": "result"}],
},
),
None,
id="function_call_output_text_parts",
),
pytest.param(
(
{"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"},
{
"type": "function_call_output",
"call_id": "call_1",
"output": [
{"type": "input_text", "text": "tool "},
{"type": "input_image", "image_url": "https://example.com/chart.png"},
{"type": "input_text", "text": "result"},
],
},
),
None,
id="function_call_output_text_and_image_parts",
),
pytest.param(
(
{"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"},
{"type": "function_call_output", "output": "orphan tool result"},
{"type": "function_call_output", "call_id": "call_1", "output": "tool result"},
),
None,
id="function_call_output_without_call_id",
),
pytest.param(
(
{"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"},
{"type": "tool_result", "call_id": "call_1", "output": "tool result"},
),
None,
id="tool_result",
),
pytest.param(
(
{"type": "computer_call", "call_id": "cu_1", "id": "cu_1", "action": {"type": "screenshot"}},
{
"type": "computer_call_output",
"call_id": "cu_1",
"output": {"type": "computer_screenshot", "image_url": "https://example.com/screen.png"},
},
),
None,
id="computer_call_output",
),
pytest.param(
(
{"type": "computer_call", "call_id": "cu_1", "id": "cu_1", "action": {"type": "screenshot"}},
{
"type": "computer_call_output",
"call_id": "cu_1",
"output": {"type": "computer_screenshot", "file_id": "file-screen"},
},
),
None,
id="computer_call_output_file_id",
),
pytest.param(
(
{"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"},
{"type": "function_call_output", "call_id": "call_1", "output": ""},
),
None,
id="function_call_output_empty",
),
pytest.param(
({"type": "reasoning", "id": "rs_1", "summary": [{"type": "summary_text", "text": "thinking"}]},),
None,
@ -4898,6 +4975,25 @@ class TestPanwAirsLatestRoleMessageOnlyEveryRequestShape:
assert [call.kwargs["content"] for call in mock_api.call_args_list] == [self.LATEST]
@pytest.mark.asyncio
async def test_flag_true_with_skip_tool_still_scans_only_the_latest_turn_on_responses(self) -> None:
from litellm.llms.openai.responses.guardrail_translation.handler import (
OpenAIResponsesHandler,
)
handler = make_handler(experimental_use_latest_role_message_only=True)
handler.skip_tool_message_in_guardrail = True
request_data = self._responses_request(
{"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"},
{"type": "function_call_output", "call_id": "call_1", "output": "tool result"},
{"role": "user", "content": self.LATEST},
)
patcher, mock_api = self._scan(handler)
with patcher:
await OpenAIResponsesHandler().process_input_messages(data=request_data, guardrail_to_apply=handler)
assert [call.kwargs["content"] for call in mock_api.call_args_list] == [self.LATEST]
@pytest.mark.asyncio
async def test_flag_false_responses_scans_instructions_and_full_history(self) -> None:
from litellm.llms.openai.responses.guardrail_translation.handler import (

View file

@ -12,6 +12,7 @@ from litellm.proxy.guardrails._content_utils import (
build_inspection_messages,
has_non_string_content,
is_non_conversational_call_type,
is_non_conversational_request,
is_string_batch_input,
iter_message_text,
iter_tool_json_text,
@ -819,6 +820,30 @@ def test_is_non_conversational_call_type_defaults_to_inspecting_unknown_call_typ
assert is_non_conversational_call_type("some_future_call_type") is False
@pytest.mark.parametrize(
("call_type", "endpoint", "expected"),
[
("aembedding", None, True),
("allm_passthrough_route", "embeddings", True),
("allm_passthrough_route", "/v1/embeddings/", True),
("allm_passthrough_route", "chat/completions", False),
("allm_passthrough_route", "model/amazon.titan-embed-text-v2:0/invoke", False),
("allm_passthrough_route", "fine_tuned_embeddings_jobs", False),
("allm_passthrough_route", "chat/completions#embeddings", False),
("allm_passthrough_route", "chat/completions?x=embeddings", False),
("allm_passthrough_route", "chat/completions;embeddings", False),
("allm_passthrough_route", None, False),
("acompletion", "embeddings", False),
],
)
def test_is_non_conversational_request_skips_only_embeddings_requests(
call_type: str, endpoint: str | None, expected: bool
):
data: Final[dict[str, object]] = {"endpoint": endpoint} if endpoint is not None else {}
assert is_non_conversational_request(call_type, data) is expected
@pytest.mark.parametrize(
"tool_input",
(

View file

@ -12,17 +12,19 @@ inspection by the relevant guardrail hook:
not just ``choices[0]``.
"""
from types import SimpleNamespace
from typing import Any, Dict, Final
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from fastapi import HTTPException
from httpx import Request, Response
import litellm
from litellm import DualCache
from litellm.constants import DEFAULT_OPENAI_MODERATIONS_MODEL
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.utils import Choices, Message, ModelResponse
from litellm.types.utils import CallTypesLiteral, Choices, Message, ModelResponse
@pytest.fixture
@ -905,3 +907,78 @@ async def test_google_text_moderation_inspects_multimodal_content(user_api_key):
)
assert seen_documents == ["hello world"]
class _FinanceFlaggingLanguageClient:
def __init__(self) -> None:
self.moderated_texts: Final[list[str]] = [] # mutable-ok: records what reached Google
def moderate_text(self, request: SimpleNamespace) -> SimpleNamespace:
self.moderated_texts.append(request.document.content)
return SimpleNamespace(moderation_categories=[SimpleNamespace(name="Finance", confidence=0.885)])
def _google_text_moderation_flagging_finance():
from enterprise.enterprise_hooks.google_text_moderation import (
ENTERPRISE_GoogleTextModeration,
)
guard: Final = ENTERPRISE_GoogleTextModeration.__new__(ENTERPRISE_GoogleTextModeration)
guard.client = _FinanceFlaggingLanguageClient()
guard.language_document = lambda content, type_: SimpleNamespace(content=content)
guard.moderate_text_request = lambda document: SimpleNamespace(document=document)
guard.document_type = "PLAIN_TEXT"
guard.finance_confidence_threshold = 0.8
return guard
@pytest.mark.asyncio
@pytest.mark.parametrize(
("call_type", "data"),
[
("aembedding", {"model": "text-embedding-ada-002", "input": "stocks"}),
("embedding", {"model": "text-embedding-ada-002", "input": ["stocks", "mortgage"]}),
("allm_passthrough_route", {"model": "gigachat-embeddings", "endpoint": "embeddings", "input": ["stocks"]}),
],
)
async def test_google_text_moderation_leaves_embedding_input_unmoderated(
user_api_key: UserAPIKeyAuth, call_type: CallTypesLiteral, data: dict[str, object]
):
guard: Final = _google_text_moderation_flagging_finance()
await guard.async_moderation_hook(data=data, user_api_key_dict=user_api_key, call_type=call_type)
assert guard.client.moderated_texts == []
@pytest.mark.asyncio
@pytest.mark.parametrize(
("call_type", "data"),
[
("acompletion", {"model": "gpt-4o", "messages": [{"role": "user", "content": "stocks"}]}),
("anthropic_messages", {"model": "claude", "messages": [{"role": "user", "content": "stocks"}]}),
("aresponses", {"model": "gpt-4o", "input": "stocks"}),
(
"allm_passthrough_route",
{"model": "gigachat", "endpoint": "chat/completions", "messages": [{"role": "user", "content": "stocks"}]},
),
(
"allm_passthrough_route",
{
"model": "gigachat",
"endpoint": "chat/completions#embeddings",
"messages": [{"role": "user", "content": "stocks"}],
},
),
],
)
async def test_google_text_moderation_still_blocks_conversations(
user_api_key: UserAPIKeyAuth, call_type: CallTypesLiteral, data: dict[str, object]
):
guard: Final = _google_text_moderation_flagging_finance()
with pytest.raises(HTTPException) as exc_info:
await guard.async_moderation_hook(data=data, user_api_key_dict=user_api_key, call_type=call_type)
assert exc_info.value.status_code == 400
assert guard.client.moderated_texts == ["stocks"]

View file

@ -3,6 +3,7 @@ import json
import logging
from collections.abc import Mapping
from datetime import datetime, timedelta, timezone
from pathlib import Path
from types import SimpleNamespace
from typing import Final, cast
from unittest.mock import AsyncMock, MagicMock, create_autospec, patch
@ -3062,10 +3063,10 @@ async def _no_fallback(line: bytes) -> None:
@pytest.mark.asyncio
async def test_async_log_success_event_hands_the_sidecar_a_compact_event_and_skips_the_pipeline(tmp_path):
async def test_async_log_success_event_hands_the_sidecar_a_compact_event_and_skips_the_pipeline(socket_path: Path):
handler = _RecordingHandler()
consumer = SpendEventConsumer(handler)
address = UnixAddress(path=str(tmp_path / "spend.sock"))
address = UnixAddress(path=str(socket_path))
server = await consumer.serve(address)
producer = SpendEventProducer(
address=address, on_unavailable="fallback", buffer_size=10, connect_timeout=1.0, fallback=_no_fallback

View file

@ -11,11 +11,14 @@ import contextlib
import os
import sys
from pathlib import Path
from typing import Any, AsyncIterator, Callable, Dict, Iterator, List, Optional
from typing import TYPE_CHECKING, Any, AsyncIterator, Callable, Dict, Final, Iterator, List, Optional
from unittest.mock import AsyncMock, MagicMock
import pytest
if TYPE_CHECKING:
from litellm import Router
# Repo root, anchored to this file (not CWD) so the path is correct no
# matter where pytest is invoked from. With the project installed via
# uv this is defensive — `litellm` already resolves through site-packages
@ -536,3 +539,25 @@ def reset_login_throttle(monkeypatch):
_drop_throttle_keys()
yield _drop_throttle_keys
_drop_throttle_keys()
@pytest.fixture
def openai_only_router(monkeypatch: pytest.MonkeyPatch) -> Router:
from litellm import Router
from litellm.proxy import proxy_server
router: Final = Router(
model_list=[
{
"model_name": "gpt-4",
"litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-fake"},
"model_info": {"id": "gpt-4-id"},
}
]
)
monkeypatch.setattr(proxy_server, "llm_router", router)
monkeypatch.setattr(proxy_server, "llm_model_list", router.model_list)
monkeypatch.setattr(proxy_server, "prisma_client", None)
monkeypatch.setattr(proxy_server, "general_settings", {})
monkeypatch.setattr(proxy_server, "user_model", None)
return router

View file

@ -9,15 +9,20 @@ Pins (PR2):
from __future__ import annotations
from typing import Final
from unittest.mock import MagicMock
import pytest
import litellm
from litellm import Router
from litellm.integrations.custom_secret_manager import CustomSecretManager
from litellm.proxy import proxy_server
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy import utils as proxy_utils
from litellm.proxy.utils import create_model_info_response
from litellm.types.router import DeploymentModelListingInfo
from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem
from .conftest import normalize # type: ignore[import-not-found]
@ -237,6 +242,7 @@ def test_anthropic_display_name_resolved_via_internal_team_key(
return_value=[
{
"model_name": internal_name,
"litellm_params": {"model": "openai/gpt-4"},
"model_info": {
"team_id": "team-1",
"team_public_model_name": "gpt-4-team",
@ -320,6 +326,7 @@ def test_anthropic_format_returns_public_team_model_name(
return_value=[
{
"model_name": internal_name,
"litellm_params": {"model": "openai/gpt-4"},
"model_info": {
"team_id": "team-1",
"team_public_model_name": "gpt-4-team",
@ -494,7 +501,11 @@ def test_codex_service_tiers_resolved_via_internal_team_key(client, auth_as, pat
patched_models.get_model_list = MagicMock(
return_value=[
{"model_name": internal_name, "model_info": {"team_id": "team-1", "team_public_model_name": "gpt-4-team"}}
{
"model_name": internal_name,
"litellm_params": {"model": "openai/gpt-4"},
"model_info": {"team_id": "team-1", "team_public_model_name": "gpt-4-team"},
}
]
)
patched_models.get_model_names = MagicMock(return_value=[internal_name])
@ -514,3 +525,206 @@ def test_codex_service_tiers_resolved_via_internal_team_key(client, auth_as, pat
assert response.status_code == 200
(entry,) = response.json()["models"]
assert (entry["slug"], [tier["id"] for tier in entry["service_tiers"]]) == ("gpt-4-team", ["ultrafast"])
def _priced_jev_models() -> frozenset[str]:
return frozenset(name.removeprefix("typesafe/") for name in litellm.model_cost if name.startswith("typesafe/"))
async def _admin_picker_models() -> set[str]:
admin: Final = UserAPIKeyAuth(api_key="sk-test", user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN)
response: Final = await proxy_server.model_list(
user_api_key_dict=admin, return_wildcard_routes=True, include_model_access_groups=True, scope="expand"
)
return {model["id"] for model in response["data"]}
@pytest.mark.asyncio
async def test_scope_expand_offers_typesafe_models_when_typesafe_is_configured(
openai_only_router: Router, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake")
jev_models: Final = _priced_jev_models()
picker_models: Final = await _admin_picker_models()
assert jev_models
assert {"gpt-4", "typesafe/*", *jev_models} <= picker_models
@pytest.mark.asyncio
async def test_scope_expand_still_lists_models_when_the_typesafe_key_reference_cannot_resolve(
openai_only_router: Router, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.delenv("LITELLM_QA_UNSET_TYPESAFE_TOKEN", raising=False)
monkeypatch.setenv("TYPESAFE_API_KEY", "oidc/env/LITELLM_QA_UNSET_TYPESAFE_TOKEN")
picker_models: Final = await _admin_picker_models()
assert {"gpt-4", "typesafe/*"} <= picker_models
@pytest.mark.asyncio
async def test_scope_expand_offers_no_typesafe_models_without_typesafe_credentials(
openai_only_router: Router, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.delenv("TYPESAFE_API_KEY", raising=False)
picker_models: Final = await _admin_picker_models()
assert "gpt-4" in picker_models
assert picker_models.isdisjoint({"typesafe/*", *_priced_jev_models()})
@pytest.mark.asyncio
async def test_scope_expand_does_not_read_the_secret_manager_for_typesafe_credentials(
openai_only_router: Router, monkeypatch: pytest.MonkeyPatch
) -> None:
secret_manager: Final = MagicMock(spec=CustomSecretManager)
secret_manager.sync_read_secret.return_value = None
monkeypatch.delenv("TYPESAFE_API_KEY", raising=False)
monkeypatch.setattr(litellm, "secret_manager_client", secret_manager)
monkeypatch.setattr(litellm, "_key_management_system", KeyManagementSystem.CUSTOM)
monkeypatch.setattr(litellm, "_key_management_settings", KeyManagementSettings(access_mode="read_only"))
picker_models: Final = await _admin_picker_models()
secrets_read: Final = {
call.kwargs["secret_name"]
for call in (*secret_manager.sync_read_secret.call_args_list, *secret_manager.async_read_secret.call_args_list)
}
assert "gpt-4" in picker_models
assert "TYPESAFE_API_KEY" not in secrets_read
@pytest.mark.asyncio
async def test_default_model_list_does_not_offer_typesafe_models(
openai_only_router: Router, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake")
admin: Final = UserAPIKeyAuth(api_key="sk-test", user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN)
response: Final = await proxy_server.model_list(user_api_key_dict=admin)
assert {model["id"] for model in response["data"]}.isdisjoint({"typesafe/*", *_priced_jev_models()})
async def _key_dialog_models(user_role: LitellmUserRoles, team_id: str | None) -> set[str]:
caller: Final = UserAPIKeyAuth(api_key="sk-test", user_id="u", user_role=user_role)
response: Final = await proxy_server.model_list(
user_api_key_dict=caller, return_wildcard_routes=True, include_model_access_groups=True, team_id=team_id
)
return {model["id"] for model in response["data"]}
@pytest.mark.asyncio
async def test_key_dialog_without_a_team_offers_typesafe_models_to_an_admin(
openai_only_router: Router, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake")
dialog_models: Final = await _key_dialog_models(LitellmUserRoles.PROXY_ADMIN, team_id=None)
assert {"gpt-4", "typesafe/*", *_priced_jev_models()} <= dialog_models
@pytest.mark.asyncio
@pytest.mark.parametrize("scope", ["expand", None])
async def test_pickers_offer_typesafe_names_once_and_no_bare_name_for_registered_typesafe_deployments(
monkeypatch: pytest.MonkeyPatch, scope: str | None
) -> None:
monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake")
jev_models: Final = _priced_jev_models()
monkeypatch.setattr(litellm, "model_cost", {**litellm.model_cost})
router: Final = Router(
model_list=[
{"model_name": "typesafe/*", "litellm_params": {"model": "typesafe/*", "api_key": "sk-typesafe-fake"}},
{
"model_name": "ticket-router",
"litellm_params": {"model": "typesafe/my-classifier", "api_key": "sk-typesafe-fake"},
},
]
)
monkeypatch.setattr(proxy_server, "llm_router", router)
monkeypatch.setattr(proxy_server, "llm_model_list", router.model_list)
monkeypatch.setattr(proxy_server, "prisma_client", None)
monkeypatch.setattr(proxy_server, "general_settings", {})
monkeypatch.setattr(proxy_server, "user_model", None)
admin: Final = UserAPIKeyAuth(api_key="sk-test", user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN)
response: Final = await proxy_server.model_list(
user_api_key_dict=admin, return_wildcard_routes=True, include_model_access_groups=True, scope=scope
)
listed: Final = [model["id"] for model in response["data"]]
assert {"ticket-router", *jev_models} <= set(listed)
assert listed.count("typesafe/*") == 1
assert set(listed).isdisjoint({"*", "my-classifier"})
@pytest.mark.asyncio
async def test_key_dialog_offers_no_typesafe_models_to_an_internal_user(
openai_only_router: Router, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake")
dialog_models: Final = await _key_dialog_models(LitellmUserRoles.INTERNAL_USER, team_id=None)
assert "gpt-4" in dialog_models
assert dialog_models.isdisjoint({"typesafe/*", *_priced_jev_models()})
@pytest.mark.asyncio
async def test_key_dialog_for_a_team_offers_no_typesafe_models_the_team_lacks(
openai_only_router: Router, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake")
dialog_models: Final = await _key_dialog_models(LitellmUserRoles.PROXY_ADMIN, team_id="team-without-typesafe")
assert dialog_models.isdisjoint({"typesafe/*", *_priced_jev_models()})
@pytest.mark.asyncio
async def test_scope_expand_lists_no_typesafe_models_to_a_client_not_asking_for_allowlist_names(
openai_only_router: Router, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake")
admin: Final = UserAPIKeyAuth(api_key="sk-test", user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN)
response: Final = await proxy_server.model_list(user_api_key_dict=admin, scope="expand")
listed: Final = {model["id"] for model in response["data"]}
assert "gpt-4" in listed
assert listed.isdisjoint({"typesafe/*", *_priced_jev_models()})
@pytest.mark.asyncio
@pytest.mark.parametrize(
("scope", "return_wildcard_routes", "only_model_access_groups"),
[
pytest.param(None, False, True, id="access-group-dropdown"),
pytest.param(None, None, None, id="dashboard-list-without-wildcard-routes"),
pytest.param("expand", None, None, id="expanded-list-without-wildcard-routes"),
pytest.param(None, True, True, id="access-groups-only-with-wildcard-routes"),
],
)
async def test_dashboard_lists_not_asking_for_allowlist_names_offer_no_typesafe_models(
openai_only_router: Router,
monkeypatch: pytest.MonkeyPatch,
scope: str | None,
return_wildcard_routes: bool | None,
only_model_access_groups: bool | None,
) -> None:
monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake")
admin: Final = UserAPIKeyAuth(api_key="sk-test", user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN)
response: Final = await proxy_server.model_list(
user_api_key_dict=admin,
include_model_access_groups=True,
return_wildcard_routes=return_wildcard_routes,
only_model_access_groups=only_model_access_groups,
scope=scope,
)
assert {model["id"] for model in response["data"]}.isdisjoint({"typesafe/*", *_priced_jev_models()})

View file

@ -1141,6 +1141,25 @@ def test_get_supported_endpoints_chat_completions_present(reset_endpoints_cache)
assert len(chat["providers"]) > 0
def test_get_supported_endpoints_lists_streamlake_on_chat_messages_and_responses(
reset_endpoints_cache: None,
) -> None:
app: Final[FastAPI] = FastAPI()
app.include_router(router)
client: Final[TestClient] = TestClient(app)
with client:
response: Final = client.get("/public/endpoints")
assert response.status_code == 200, response.text
streamlake_endpoints: Final = {
endpoint["key"]
for endpoint in response.json()["endpoints"]
if any(provider["slug"] == "streamlake" for provider in endpoint["providers"])
}
assert {"chat_completions", "messages", "responses"} <= streamlake_endpoints
assert "embeddings" not in streamlake_endpoints
def test_get_supported_endpoints_display_names_have_no_slug_suffix(
reset_endpoints_cache,
):

View file

@ -177,9 +177,9 @@ def test_settings_read_the_documented_env(monkeypatch):
@pytest.mark.asyncio
async def test_events_reach_the_sidecar_once_and_in_order(tmp_path: Path):
async def test_events_reach_the_sidecar_once_and_in_order(socket_path: Path):
fallback: Final = _Fallback()
async with _Sidecar(tmp_path / "spend.sock") as sidecar:
async with _Sidecar(socket_path) as sidecar:
producer: Final = _producer(sidecar.path, fallback)
outcomes: Final = [await producer.publish(f"event-{i}\n".encode()) for i in range(20)]
await producer.close(drain_timeout=5.0)
@ -193,9 +193,9 @@ async def test_events_reach_the_sidecar_once_and_in_order(tmp_path: Path):
@pytest.mark.asyncio
async def test_unreachable_sidecar_falls_back_in_process_and_backs_off(tmp_path: Path):
async def test_unreachable_sidecar_falls_back_in_process_and_backs_off(socket_path: Path):
fallback: Final = _Fallback()
producer: Final = _producer(tmp_path / "missing.sock", fallback)
producer: Final = _producer(socket_path.with_name("missing.sock"), fallback)
first: Final = await producer.publish(b"event-1\n")
await asyncio.sleep(0.05)
second: Final = await producer.publish(b"event-2\n")
@ -210,11 +210,11 @@ async def test_unreachable_sidecar_falls_back_in_process_and_backs_off(tmp_path:
@pytest.mark.parametrize("loop_factory", [asyncio.new_event_loop, uvloop.new_event_loop], ids=["asyncio", "uvloop"])
def test_sidecar_hang_up_falls_back_instead_of_losing_events(
tmp_path: Path, loop_factory: Callable[[], asyncio.AbstractEventLoop]
socket_path: Path, loop_factory: Callable[[], asyncio.AbstractEventLoop]
):
async def scenario() -> tuple[list[bytes], list[bytes], tuple[int, int, int]]:
fallback: Final = _Fallback()
sidecar: Final = _Sidecar(tmp_path / "spend.sock")
sidecar: Final = _Sidecar(socket_path)
async with sidecar:
producer: Final = _producer(sidecar.path, fallback)
await producer.publish(b"event-1\n")
@ -236,7 +236,7 @@ def test_sidecar_hang_up_falls_back_instead_of_losing_events(
@pytest.mark.parametrize("loop_factory", [asyncio.new_event_loop, uvloop.new_event_loop], ids=["asyncio", "uvloop"])
def test_mid_stream_crash_never_bills_an_event_on_both_sides(
tmp_path: Path, loop_factory: Callable[[], asyncio.AbstractEventLoop]
socket_path: Path, loop_factory: Callable[[], asyncio.AbstractEventLoop]
):
"""Events large enough to straddle the kernel buffer, a sidecar that reads some and then drops the socket: a
failed write may only fall back when the sidecar cannot have read the whole line."""
@ -244,7 +244,7 @@ def test_mid_stream_crash_never_bills_an_event_on_both_sides(
async def scenario() -> tuple[list[bytes], list[bytes], tuple[int, int, int]]:
fallback: Final = _Fallback()
sidecar: Final = _CrashingSidecar(tmp_path / "spend.sock", lines_before_crash=3)
sidecar: Final = _CrashingSidecar(socket_path, lines_before_crash=3)
async with sidecar:
producer: Final = _producer(sidecar.path, fallback)
for event in events:
@ -266,11 +266,11 @@ def test_mid_stream_crash_never_bills_an_event_on_both_sides(
@pytest.mark.asyncio
async def test_drain_timeout_hands_the_in_flight_event_to_fallback(tmp_path: Path):
async def test_drain_timeout_hands_the_in_flight_event_to_fallback(socket_path: Path):
"""A sidecar that stops reading leaves one event half-written; cancelling the writer must not lose it."""
fallback: Final = _Fallback()
stuck: Final = b"x" * (4 * 1024 * 1024) + b"\n"
async with _Sidecar(tmp_path / "spend.sock", reads=False) as sidecar:
async with _Sidecar(socket_path, reads=False) as sidecar:
producer: Final = _producer(sidecar.path, fallback)
assert await producer.publish(stuck) == "queued"
await asyncio.sleep(0.1)
@ -282,10 +282,10 @@ async def test_drain_timeout_hands_the_in_flight_event_to_fallback(tmp_path: Pat
@pytest.mark.asyncio
async def test_shutdown_lets_the_writer_finish_a_fallback_already_in_progress(tmp_path: Path):
async def test_shutdown_lets_the_writer_finish_a_fallback_already_in_progress(socket_path: Path):
"""Cancelling the writer while it runs the pipeline in-process must neither lose nor repeat that event."""
fallback: Final = _GatedFallback()
producer: Final = _producer(tmp_path / "missing.sock", fallback)
producer: Final = _producer(socket_path.with_name("missing.sock"), fallback)
assert await producer.publish(b"event-1\n") == "queued"
await asyncio.wait_for(fallback.started.wait(), 5.0)
closing: Final = asyncio.ensure_future(producer.close(drain_timeout=0.05))
@ -299,10 +299,10 @@ async def test_shutdown_lets_the_writer_finish_a_fallback_already_in_progress(tm
@pytest.mark.asyncio
async def test_shutdown_does_not_replay_an_event_the_kernel_already_took(tmp_path: Path):
async def test_shutdown_does_not_replay_an_event_the_kernel_already_took(socket_path: Path):
"""Cancelling a drain whose bytes already left the process must not run the event a second time in-process."""
fallback: Final = _Fallback()
async with _Sidecar(tmp_path / "spend.sock") as sidecar:
async with _Sidecar(socket_path) as sidecar:
producer: Final = _producer(sidecar.path, fallback, open_connection=_open_with_stalled_drain)
assert await producer.publish(b"event-1\n") == "queued"
await asyncio.sleep(0.1)
@ -316,9 +316,9 @@ async def test_shutdown_does_not_replay_an_event_the_kernel_already_took(tmp_pat
@pytest.mark.asyncio
async def test_drop_policy_counts_instead_of_running_in_process(tmp_path: Path):
async def test_drop_policy_counts_instead_of_running_in_process(socket_path: Path):
fallback: Final = _Fallback()
producer: Final = _producer(tmp_path / "missing.sock", fallback, on_unavailable="drop")
producer: Final = _producer(socket_path.with_name("missing.sock"), fallback, on_unavailable="drop")
await producer.publish(b"event-1\n")
await producer.close(drain_timeout=5.0)
assert await producer.publish(b"event-2\n") == "dropped"
@ -328,9 +328,9 @@ async def test_drop_policy_counts_instead_of_running_in_process(tmp_path: Path):
@pytest.mark.asyncio
async def test_full_buffer_applies_the_unavailable_policy_immediately(tmp_path: Path):
async def test_full_buffer_applies_the_unavailable_policy_immediately(socket_path: Path):
fallback: Final = _Fallback()
async with _Sidecar(tmp_path / "spend.sock") as sidecar:
async with _Sidecar(socket_path) as sidecar:
producer: Final = _producer(sidecar.path, fallback, buffer_size=2)
outcomes: Final = [await producer.publish(f"event-{i}\n".encode()) for i in range(3)]
await producer.close(drain_timeout=5.0)
@ -342,9 +342,9 @@ async def test_full_buffer_applies_the_unavailable_policy_immediately(tmp_path:
@pytest.mark.asyncio
async def test_close_flushes_buffered_events_then_refuses_new_ones(tmp_path: Path):
async def test_close_flushes_buffered_events_then_refuses_new_ones(socket_path: Path):
fallback: Final = _Fallback()
async with _Sidecar(tmp_path / "spend.sock") as sidecar:
async with _Sidecar(socket_path) as sidecar:
producer: Final = _producer(sidecar.path, fallback)
for i in range(50):
await producer.publish(f"event-{i}\n".encode())

View file

@ -49,14 +49,14 @@ class _Fallback:
@pytest.mark.asyncio
@pytest.mark.parametrize("transport", ["unix", "tcp"])
async def test_consumer_handles_each_producer_line_once_in_order(tmp_path: Path, transport: str):
async def test_consumer_handles_each_producer_line_once_in_order(socket_path: Path, transport: str):
handler: Final = _Handler(fail_on=b"event-3\n")
consumer: Final = SpendEventConsumer(handler)
server: Final = await consumer.serve(
UnixAddress(path=str(tmp_path / "spend.sock")) if transport == "unix" else TcpAddress("127.0.0.1", 0)
UnixAddress(path=str(socket_path)) if transport == "unix" else TcpAddress("127.0.0.1", 0)
)
address: Final = (
UnixAddress(path=str(tmp_path / "spend.sock"))
UnixAddress(path=str(socket_path))
if transport == "unix"
else TcpAddress("127.0.0.1", server.sockets[0].getsockname()[1])
)
@ -74,10 +74,10 @@ async def test_consumer_handles_each_producer_line_once_in_order(tmp_path: Path,
@pytest.mark.asyncio
async def test_consumer_discards_a_truncated_trailing_event(tmp_path: Path):
async def test_consumer_discards_a_truncated_trailing_event(socket_path: Path):
handler: Final = _Handler()
consumer: Final = SpendEventConsumer(handler)
address: Final = UnixAddress(path=str(tmp_path / "spend.sock"))
address: Final = UnixAddress(path=str(socket_path))
server: Final = await consumer.serve(address)
_, writer = await open_collector_connection(address, timeout=1.0)
writer.write(b"whole\npartial-without-newline")
@ -93,9 +93,9 @@ async def test_consumer_discards_a_truncated_trailing_event(tmp_path: Path):
@pytest.mark.asyncio
async def test_drain_reports_producers_still_connected_after_the_timeout(tmp_path: Path):
async def test_drain_reports_producers_still_connected_after_the_timeout(socket_path: Path):
consumer: Final = SpendEventConsumer(_Handler())
address: Final = UnixAddress(path=str(tmp_path / "spend.sock"))
address: Final = UnixAddress(path=str(socket_path))
server: Final = await consumer.serve(address)
_, writer = await open_collector_connection(address, timeout=1.0)
await asyncio.sleep(0.05)
@ -108,11 +108,11 @@ async def test_drain_reports_producers_still_connected_after_the_timeout(tmp_pat
@pytest.mark.asyncio
async def test_graceful_stop_hands_the_producer_over_to_its_fallback_without_losing_events(tmp_path: Path):
async def test_graceful_stop_hands_the_producer_over_to_its_fallback_without_losing_events(socket_path: Path):
handler: Final = _Handler()
fallback: Final = _Fallback()
consumer: Final = SpendEventConsumer(handler)
address: Final = UnixAddress(path=str(tmp_path / "spend.sock"))
address: Final = UnixAddress(path=str(socket_path))
server: Final = await consumer.serve(address)
producer: Final = SpendEventProducer(
address=address, on_unavailable="fallback", buffer_size=100, connect_timeout=1.0, fallback=fallback
@ -134,7 +134,7 @@ async def test_graceful_stop_hands_the_producer_over_to_its_fallback_without_los
@pytest.mark.parametrize("loop_factory", [asyncio.new_event_loop, uvloop.new_event_loop], ids=["asyncio", "uvloop"])
def test_drain_still_hands_over_live_producers_when_another_connection_already_died(
tmp_path: Path, loop_factory: Callable[[], asyncio.AbstractEventLoop]
socket_path: Path, loop_factory: Callable[[], asyncio.AbstractEventLoop]
):
"""A transport the loop force-closed under a busy handler must not abort the half-close of the others."""
@ -145,7 +145,7 @@ def test_drain_still_hands_over_live_producers_when_another_connection_already_d
await release.wait()
consumer: Final = SpendEventConsumer(slow_handler)
address: Final = UnixAddress(path=str(tmp_path / "spend.sock"))
address: Final = UnixAddress(path=str(socket_path))
server: Final = await consumer.serve(address)
_, dead = await open_collector_connection(address, timeout=1.0)
dead.write(b"stuck\n")

View file

@ -22,7 +22,9 @@ import json
import asyncio
import logging
import httpx
import pytest
import respx
import litellm
from litellm import RateLimitError, Timeout, completion, completion_cost, embedding
@ -3857,3 +3859,104 @@ async def test_model_list_return_wildcard_routes_defaults_to_general_setting(
assert "gpt-6.1-sol" in listed, listed
assert ("openai/*" in listed) is wildcard_listed, listed
_DEBUG_LOG_SENTINEL: Final = "sentinel-request-body-7f3a"
_LONG_PROMPT: Final = f"{_DEBUG_LOG_SENTINEL} " + "Keep edits minimal and explain them briefly. " * 20
_TOOL_PARAMETERS: Final = {"type": "object", "properties": {"cmd": {"type": "string"}}}
class _MessageCollector(logging.Handler):
def __init__(self) -> None:
super().__init__(level=logging.DEBUG)
self.messages: list[str] = [] # mutable-ok: a logging.Handler collects records as they are emitted
def emit(self, record: logging.LogRecord) -> None:
self.messages.append(record.getMessage())
@pytest.mark.parametrize(
"route, path_suffix, body, upstream_json",
[
(
"/v1/chat/completions",
"/chat/completions",
{
"model": "test_openai_models",
"messages": [{"role": "user", "content": _LONG_PROMPT}],
"tools": [
{
"type": "function",
"function": {"name": "shell", "description": _LONG_PROMPT, "parameters": _TOOL_PARAMETERS},
}
],
},
{
"id": "chatcmpl-1",
"object": "chat.completion",
"created": 1,
"model": "gpt-3.5-turbo",
"choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "hi"}}],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
},
),
(
"/v1/responses",
"/responses",
{
"model": "test_openai_models",
"instructions": _LONG_PROMPT,
"input": "hi",
"tools": [
{"type": "function", "name": "shell", "description": _LONG_PROMPT, "parameters": _TOOL_PARAMETERS}
],
},
{
"id": "resp_1",
"object": "response",
"created_at": 1,
"status": "completed",
"model": "gpt-3.5-turbo",
"output": [
{
"type": "message",
"id": "msg_1",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "hi", "annotations": []}],
}
],
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
},
),
],
)
def test_detailed_debug_logs_request_body_only_as_received_and_as_sent(
client_no_auth: TestClient,
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
route: str,
path_suffix: str,
body: Mapping[str, object],
upstream_json: Mapping[str, object],
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
respx_mock.post(url__regex=rf".*{path_suffix}$").mock(return_value=httpx.Response(200, json=upstream_json))
collector: Final = _MessageCollector()
loggers: Final = tuple(logging.getLogger(name) for name in ("LiteLLM", "LiteLLM Router", "LiteLLM Proxy"))
original_levels: Final = tuple(lg.level for lg in loggers)
for lg in loggers:
lg.addHandler(collector)
lg.setLevel(logging.DEBUG)
try:
response: Final = client_no_auth.post(route, json=body)
finally:
for lg, level in zip(loggers, original_levels):
lg.removeHandler(collector)
lg.setLevel(level)
assert response.status_code == 200, response.text
body_dumps: Final = [message for message in collector.messages if _DEBUG_LOG_SENTINEL in message]
assert len(body_dumps) == 2, [message[:120] for message in body_dumps]
assert body_dumps[0].startswith("Request received by LiteLLM:")
assert "POST Request Sent from LiteLLM" in body_dumps[1]

View file

@ -111,6 +111,8 @@ def test_media_normalization_preserves_formats_references_and_metadata(kind: str
"keep",
),
([{"type": "input_image", "file_id": "file-only"}], '[{"type": "input_image", "file_id": "file-only"}]'),
({"ids": {7}}, "{'ids': {7}}"),
([{"type": "unknown", "ids": {7}}], "[{'type': 'unknown', 'ids': {7}}]"),
],
)
def test_tool_output_without_usable_media_keeps_its_existing_fallback(output: object, expected: str) -> None:

View file

@ -10,6 +10,7 @@ before response.completed, and that every event of a bridged stream carries the
spend tracking stores, so a follow-up previous_response_id still finds the conversation.
"""
import itertools
import json
from collections.abc import Callable
from typing import Final
@ -23,6 +24,7 @@ from litellm.responses.litellm_completion_transformation.streaming_iterator impo
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.llms.openai import (
BaseLiteLLMOpenAIResponseObject,
OutputTextDeltaEvent,
ResponsesAPIStreamEvents,
)
from litellm.types.responses.main import build_web_search_call
@ -1658,3 +1660,34 @@ async def test_a_compaction_block_without_its_summary_streams_no_compaction_item
assert [event["item"]["type"] for event in events if "item" in event] == ["message", "message"]
assert [item["type"] for item in events[-1]["response"]["output"]] == ["message"]
def test_the_partial_response_billed_on_disconnect_carries_the_logging_objects_bill_as_hidden_response_cost() -> None:
from litellm.litellm_core_utils.hidden_params import get_hidden_params
wrapper: Final = _FakeStreamWrapper([_chunk("Hello"), _chunk(" world", finish_reason="stop")])
wrapper.logging_obj.response_cost_calculator.return_value = 0.00032
iterator: Final = LiteLLMCompletionStreamingIterator(
model="claude-haiku-4-5",
litellm_custom_stream_wrapper=wrapper, # pyright: ignore[reportArgumentType] # the fake stands in for CustomStreamWrapper, as in _build_iterator
request_input="What is the weather in San Francisco?",
responses_api_request={},
custom_llm_provider="anthropic",
litellm_metadata={},
)
relayed: Final = tuple(
itertools.takewhile(lambda event: not isinstance(event, OutputTextDeltaEvent), iterator)
)
assert relayed, "the stream must relay at least one chunk before the client goes away"
partial: Final = iterator.partial_response_on_disconnect()
assert partial is not None
hidden: Final = get_hidden_params(partial.response)
assert hidden is not None and hidden["response_cost"] == 0.00032
priced: Final = wrapper.logging_obj.response_cost_calculator.call_args.kwargs["result"]
assert priced.id == partial.response.id
assert (priced.usage.prompt_tokens, priced.usage.completion_tokens) == (
partial.response.usage.input_tokens,
partial.response.usage.output_tokens,
), "the bill is priced on the partial's own usage"

View file

@ -24,7 +24,7 @@ from litellm.exceptions import MidStreamFallbackError
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
from litellm.responses.streaming_iterator import(
from litellm.responses.streaming_iterator import (
CachedResponsesAPIStreamingIterator,
MockResponsesAPIStreamingIterator,
ResponsesAPIStreamingIterator,
@ -845,25 +845,33 @@ def test_stamp_responses_usage_cost_stamps_computed_cost():
response = _responses_api_response_with_usage()
logging_obj = Mock(spec=LiteLLMLoggingObj)
logging_obj.model_call_details = {"custom_llm_provider": "openai"}
logging_obj.response_cost_calculator.return_value = 0.000704
stamp_responses_usage_cost(response, logging_obj)
assert getattr(response.usage, "cost", None) == pytest.approx(0.000704)
logging_obj.response_cost_calculator.assert_called_once_with(result=response)
priced = logging_obj.response_cost_calculator.call_args.kwargs["result"]
assert priced.usage.prompt_tokens == response.usage.input_tokens, (
"priced in the chat usage shape the spend row uses"
)
def test_stamp_responses_usage_cost_keeps_provider_reported_cost():
def test_stamp_responses_usage_cost_bills_the_spend_row_price_over_the_provider_reported_cost():
from litellm.litellm_core_utils.core_helpers import response_cost_in_hidden_params
from litellm.responses.streaming_iterator import stamp_responses_usage_cost
response = _responses_api_response_with_usage()
setattr(response.usage, "cost", 0.5)
logging_obj = Mock(spec=LiteLLMLoggingObj)
logging_obj.model_call_details = {"custom_llm_provider": "openai"}
logging_obj.response_cost_calculator.return_value = 0.00032
stamp_responses_usage_cost(response, logging_obj)
assert getattr(response.usage, "cost", None) == pytest.approx(0.5)
logging_obj.response_cost_calculator.assert_not_called()
assert getattr(response.usage, "cost", None) == pytest.approx(0.00032), "the client sees what the row bills"
priced = logging_obj.response_cost_calculator.call_args.kwargs["result"]
assert response_cost_in_hidden_params(priced) == pytest.approx(0.5), "the reported cost reaches the calculator"
def _unvalidated_response_with_dict_usage(usage: dict) -> ResponsesAPIResponse:
@ -881,6 +889,7 @@ def _unvalidated_response_with_dict_usage(usage: dict) -> ResponsesAPIResponse:
def test_stamp_responses_usage_cost_keeps_provider_cost_from_dict_usage():
from litellm.responses.streaming_iterator import stamp_responses_usage_cost
response = _unvalidated_response_with_dict_usage(
{
"input_tokens": 29,
@ -891,26 +900,29 @@ def test_stamp_responses_usage_cost_keeps_provider_cost_from_dict_usage():
}
)
logging_obj = Mock(spec=LiteLLMLoggingObj)
logging_obj.model_call_details = {"custom_llm_provider": "openai"}
logging_obj.response_cost_calculator.return_value = 3e-05
stamp_responses_usage_cost(response, logging_obj)
assert isinstance(response.usage, ResponseAPIUsage)
assert response.usage.cost == pytest.approx(3e-05)
assert response.usage.output_tokens_details.reasoning_tokens == 117
logging_obj.response_cost_calculator.assert_not_called()
def test_stamp_responses_usage_cost_computes_cost_for_dict_usage_without_cost():
from litellm.responses.streaming_iterator import stamp_responses_usage_cost
response = _unvalidated_response_with_dict_usage({"input_tokens": 29, "output_tokens": 120, "total_tokens": 149})
logging_obj = Mock(spec=LiteLLMLoggingObj)
logging_obj.model_call_details = {"custom_llm_provider": "openai"}
logging_obj.response_cost_calculator.return_value = 0.000704
stamp_responses_usage_cost(response, logging_obj)
assert isinstance(response.usage, ResponseAPIUsage)
assert response.usage.cost == pytest.approx(0.000704)
logging_obj.response_cost_calculator.assert_called_once_with(result=response)
logging_obj.response_cost_calculator.assert_called_once()
def test_stamp_responses_usage_cost_survives_calculator_failure():
@ -918,6 +930,7 @@ def test_stamp_responses_usage_cost_survives_calculator_failure():
response = _responses_api_response_with_usage()
logging_obj = Mock(spec=LiteLLMLoggingObj)
logging_obj.model_call_details = {"custom_llm_provider": "openai"}
logging_obj.response_cost_calculator.side_effect = RuntimeError("cost map unavailable")
stamp_responses_usage_cost(response, logging_obj)
@ -1207,9 +1220,7 @@ async def test_completed_event_survives_a_failing_usage_estimate():
falls back to usage None."""
malformed_input: Final = [{"type": "message", "role": "user", "content": 42}]
with pytest.raises(ValueError, match="Invalid content type"):
_estimate_usage_from_text(
"gpt-4o-mini", malformed_input, {"input": malformed_input}, "hello world"
)
_estimate_usage_from_text("gpt-4o-mini", malformed_input, {"input": malformed_input}, "hello world")
response = _responses_api_response_without_usage()
iterator = _make_iterator(
@ -1305,7 +1316,14 @@ async def test_completed_event_with_a_dict_response_is_typed_and_billed():
assert usage.input_tokens > 0
assert usage.output_tokens > 0
assert usage.cost == pytest.approx(0.000704)
logging_obj.response_cost_calculator.assert_any_call(result=completed_response)
priced: Final = [
call.kwargs["result"]
for call in logging_obj.response_cost_calculator.call_args_list
if isinstance(call.kwargs.get("result"), ResponsesAPIResponse)
]
assert [(r.id, r.usage.prompt_tokens) for r in priced] == [(completed_response.id, usage.input_tokens)], (
"priced once, in the chat usage shape the spend row uses"
)
def test_billed_terminal_response_keeps_a_response_that_already_has_usage():
@ -1375,6 +1393,7 @@ def _vcr_outcome_gate(request, vcr):
yield
record_vcr_outcome(request, vcr)
@pytest.fixture(scope="function")
def setup_and_teardown():
"""
@ -1392,6 +1411,7 @@ def setup_and_teardown():
loop.close()
asyncio.set_event_loop(None)
class _FakeLoggingObj:
def __init__(self):
self.success_calls = 0
@ -1432,10 +1452,12 @@ class _FakeLoggingObj:
self.completion_start_time = completion_start_time
self.model_call_details["completion_start_time"] = completion_start_time
async def _await_tasks_spawned_since(existing: set["asyncio.Task[object]"]) -> None:
spawned: Final = asyncio.all_tasks() - existing
await asyncio.wait_for(asyncio.gather(*spawned, return_exceptions=True), timeout=5)
def _make_completed_response(response_id: str = "resp_test") -> ResponseCompletedEvent:
return ResponseCompletedEvent(
type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED,
@ -1463,6 +1485,7 @@ def _make_completed_response(response_id: str = "resp_test") -> ResponseComplete
),
)
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
@pytest.mark.asyncio
async def test_log_background_task_failure_logs_task_exceptions(monkeypatch):
@ -1485,6 +1508,7 @@ async def test_log_background_task_failure_logs_task_exceptions(monkeypatch):
task.exception(),
)
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
@pytest.mark.asyncio
async def test_log_background_task_failure_ignores_cancelled_tasks(monkeypatch):
@ -1500,6 +1524,7 @@ async def test_log_background_task_failure_ignores_cancelled_tasks(monkeypatch):
error_logger.assert_not_called()
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
def test_content_part_done_event_supports_refusal_and_reasoning_text():
refusal_event = streaming_module._build_content_part_done_event(
@ -1527,6 +1552,7 @@ def test_content_part_done_event_supports_refusal_and_reasoning_text():
assert reasoning_event.part.reasoning == "because"
assert unsupported_event is None
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
def test_dump_response_object_handles_model_and_unknown_values():
response = ResponsesAPIResponse(
@ -1542,6 +1568,7 @@ def test_dump_response_object_handles_model_and_unknown_values():
assert streaming_module._dump_response_object({"type": "message"}) == {"type": "message"}
assert streaming_module._dump_response_object(object()) == {}
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
@pytest.mark.asyncio
async def test_responses_streaming_triggers_hooks(monkeypatch):
@ -1598,6 +1625,7 @@ async def test_responses_streaming_triggers_hooks(monkeypatch):
assert seen["request_data"].get("litellm_params") is not None
assert seen["call_type"] == CallTypes.responses
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
@pytest.mark.asyncio
async def test_responses_streaming_calls_post_streaming_deployment_hook(monkeypatch):
@ -1638,6 +1666,7 @@ async def test_responses_streaming_calls_post_streaming_deployment_hook(monkeypa
# reset callbacks
litellm.callbacks = original_callbacks
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
@pytest.mark.asyncio
async def test_responses_streaming_failure_triggers_failure_handlers():
@ -1667,6 +1696,7 @@ async def test_responses_streaming_failure_triggers_failure_handlers():
assert logging_obj.failure_calls >= 1
assert logging_obj.async_failure_calls >= 1
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
def test_process_chunk_requires_provider_config():
iterator = ResponsesAPIStreamingIterator(
@ -1681,6 +1711,7 @@ def test_process_chunk_requires_provider_config():
with pytest.raises(ValueError, match="responses_api_provider_config is required"):
iterator._process_chunk(json.dumps({"type": "response.completed"}))
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
def test_process_chunk_wraps_encrypted_content_with_model_id():
openai_types = streaming_module._get_openai_response_types()
@ -1715,6 +1746,7 @@ def test_process_chunk_wraps_encrypted_content_with_model_id():
assert event.item.encrypted_content.startswith("litellm_enc:")
assert event.item.encrypted_content.endswith(";ciphertext")
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
def test_process_chunk_completed_response_updates_id_and_usage_cost(monkeypatch):
original_include_cost = litellm.include_cost_in_streaming_usage
@ -1768,6 +1800,7 @@ def test_process_chunk_completed_response_updates_id_and_usage_cost(monkeypatch)
assert event.response.usage.cost == 1.23
completion_handler.assert_called_once()
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
def test_process_chunk_failed_response_triggers_failure_logging(monkeypatch):
openai_types = streaming_module._get_openai_response_types()
@ -1803,6 +1836,7 @@ def test_process_chunk_failed_response_triggers_failure_logging(monkeypatch):
assert iterator.completed_response is event
failure_handler.assert_called_once()
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
@pytest.mark.asyncio
async def test_handle_logging_failed_response_uses_response_error_message():
@ -1835,6 +1869,7 @@ async def test_handle_logging_failed_response_uses_response_error_message():
assert logging_obj.failure_calls == 1
assert logging_obj.async_failure_calls == 1
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
def test_process_chunk_returns_none_for_invalid_json_and_non_dict_payload():
class _NoopConfig:
@ -1853,6 +1888,7 @@ def test_process_chunk_returns_none_for_invalid_json_and_non_dict_payload():
assert iterator._process_chunk("not-json") is None
assert iterator._process_chunk(json.dumps(["not", "a", "dict"])) is None
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
def test_process_chunk_cost_annotation_failure_is_nonfatal(monkeypatch):
original_include_cost = litellm.include_cost_in_streaming_usage
@ -1900,6 +1936,7 @@ def test_process_chunk_cost_annotation_failure_is_nonfatal(monkeypatch):
assert event.response.usage.cost is None
completion_handler.assert_called_once()
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
def test_get_completed_response_object_accepts_direct_response():
logging_obj = _FakeLoggingObj()
@ -1916,6 +1953,7 @@ def test_get_completed_response_object_accepts_direct_response():
assert iterator._get_completed_response_object() is direct_response
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
@pytest.mark.asyncio
async def test_responses_streaming_completed_event_persists_async_cache():
@ -1964,6 +2002,7 @@ async def test_responses_streaming_completed_event_persists_async_cache():
assert json.loads(litellm.cache.async_add_cache.call_args.args[0])["id"] == iterator.completed_response.response.id
litellm.cache = original_cache
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
def test_responses_streaming_completed_event_persists_sync_cache():
logging_obj = _FakeLoggingObj()
@ -2009,6 +2048,7 @@ def test_responses_streaming_completed_event_persists_sync_cache():
assert json.loads(litellm.cache.add_cache.call_args.args[0])["id"] == iterator.completed_response.response.id
litellm.cache = original_cache
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
def test_log_completed_response_sync_direct_path(monkeypatch):
hook_calls = {"post_call": 0, "metadata": 0}
@ -2050,6 +2090,7 @@ def test_log_completed_response_sync_direct_path(monkeypatch):
assert hook_calls["post_call"] == 1
assert hook_calls["metadata"] == 1
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
def test_log_completed_response_falls_back_when_model_validate_fails(monkeypatch):
class _BadSerializableResponse:
@ -2079,6 +2120,7 @@ def test_log_completed_response_falls_back_when_model_validate_fails(monkeypatch
assert logging_obj.success_calls == 1
assert logging_obj.async_success_calls == 1
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
@pytest.mark.parametrize(
"scenario",
@ -2143,6 +2185,7 @@ def test_persist_completed_response_to_cache_guard_branches(monkeypatch, scenari
expected_cached_flag = scenario == "already_cached"
assert iterator._completed_response_cached is expected_cached_flag
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
def test_build_synthetic_response_events_covers_annotations_function_calls_and_refusals():
original_include_cost = litellm.include_cost_in_streaming_usage
@ -2201,6 +2244,7 @@ def test_build_synthetic_response_events_covers_annotations_function_calls_and_r
assert "response.function_call_arguments.done" in event_types
assert event_types[-1] == "response.completed"
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
@pytest.mark.asyncio
async def test_mock_responses_streaming_iterator_async_iteration_logs_completion(
@ -2251,6 +2295,7 @@ async def test_mock_responses_streaming_iterator_async_iteration_logs_completion
assert hook_calls["post_call"] == 1
assert hook_calls["metadata"] == 1
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
def test_mock_responses_streaming_iterator_sync_iteration_logs_completion(monkeypatch):
hook_calls = {"post_call": 0, "metadata": 0}
@ -2296,6 +2341,7 @@ def test_mock_responses_streaming_iterator_sync_iteration_logs_completion(monkey
assert hook_calls["post_call"] == 1
assert hook_calls["metadata"] == 1
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
@pytest.mark.asyncio
async def test_cached_responses_stream_async_hit_triggers_success_callbacks(
@ -2355,6 +2401,7 @@ async def test_cached_responses_stream_async_hit_triggers_success_callbacks(
litellm.cache.add_cache.assert_not_called()
litellm.cache = original_cache
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
def test_cached_responses_stream_sync_hit_triggers_success_callbacks(monkeypatch):
hook_calls = {"post_call": 0, "metadata": 0}
@ -2453,7 +2500,9 @@ def test_faked_stream_bridged_to_chat_sends_tool_arguments_once():
tool_call_deltas: Final = tuple(
tool_call
for chunk in chunks
for tool_call in (chunk.choices[0].delta.tool_calls or ()) # comprehension-ok: flattening each chunk's tool calls
for tool_call in (
chunk.choices[0].delta.tool_calls or ()
) # comprehension-ok: flattening each chunk's tool calls
)
joined_arguments: Final = "".join(tool_call.function.arguments or "" for tool_call in tool_call_deltas)
joined_text: Final = "".join(chunk.choices[0].delta.content or "" for chunk in chunks)
@ -2509,7 +2558,6 @@ def test_faked_stream_opens_items_in_progress_and_empty_like_a_real_stream():
assert [part.text for part in added_parts] == [""]
def _answer_with_reasoning_refusal_and_custom_tool_call() -> ResponsesAPIResponse:
return ResponsesAPIResponse(
id="resp_synthetic",
@ -2643,7 +2691,9 @@ async def test_shutdown_drain_waits_for_a_pending_logging_task():
@pytest.mark.asyncio
async def test_shutdown_drain_gives_up_on_a_stuck_logging_task_at_its_timeout():
stuck: Final = streaming_module.spawn_logging_task(asyncio.get_running_loop(), asyncio.Event().wait(), task_name="stuck")
stuck: Final = streaming_module.spawn_logging_task(
asyncio.get_running_loop(), asyncio.Event().wait(), task_name="stuck"
)
await asyncio.wait_for(streaming_module.drain_logging_tasks(timeout=0.05), timeout=1)

View file

@ -87,7 +87,7 @@ def assert_native_request(
def native_response(status: int) -> bytes:
if status == 429:
return b'{"error":"native-rate-limit"}'
return b'{"output":{"message":{"content":[{"text":"native-transcription"}]}}}'
return b'{"output":{"message":{"content":[{"text":"native-transcription"}]}},"usage":{"inputTokens":390,"outputTokens":60}}'
def load_native(native_path: Path) -> object:
@ -130,6 +130,9 @@ def assert_success(route: str, response: object) -> None:
raise TypeError(f"{route} returned {type(response).__name__}, expected dict")
if response["text"] != "native-transcription":
raise AssertionError(f"{route} returned {response['text']!r}, expected 'native-transcription'")
expected_usage: Final = {"type": "tokens", "input_tokens": 390, "output_tokens": 60, "total_tokens": 450}
if response.get("usage") != expected_usage:
raise AssertionError(f"{route} returned usage {response.get('usage')!r}, expected {expected_usage!r}")
def assert_rate_limit(route: str, error: BaseException) -> None:

View file

@ -11,6 +11,7 @@ from litellm.rust_bridge import configuration
from litellm.rust_bridge._native import RustBridgeDeclined, RustUpstreamError
from litellm.rust_bridge.public_call import NativeCall
from litellm.rust_bridge.transcription.native import NATIVE_ATRANSCRIPTION, NATIVE_TRANSCRIPTION
from litellm.types.utils import TranscriptionUsageTokensObject
MODEL: Final = "bedrock/mistral.voxtral-mini-3b-2507"
AUDIO_FILE: Final = ("audio.wav", b"audio", "audio/wav")
@ -59,6 +60,14 @@ class AsyncBridge:
return {"text": "async rust"}
class UsageBridge:
def __call__(self, call: NativeCall) -> dict[str, object]:
return {
"text": "rust",
"usage": {"type": "tokens", "input_tokens": 390, "output_tokens": 60, "total_tokens": 450},
}
def dispatch_sync() -> litellm.TranscriptionResponse:
return BedrockAudioTranscriptionRustDispatch().audio_transcriptions(
model=MODEL,
@ -196,3 +205,99 @@ async def test_async_missing_native_binding_raises_without_python_fallback() ->
optional_params={},
timeout=None,
)
@pytest.mark.parametrize(
("region_kwargs", "region_env", "priced_model"),
[
({}, {}, MODEL),
({"aws_region_name": "ap-southeast-2"}, {}, "bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507"),
({}, {"AWS_REGION_NAME": "eu-west-2"}, "bedrock/eu-west-2/mistral.voxtral-mini-3b-2507"),
({}, {"AWS_REGION": "ap-northeast-1"}, "bedrock/ap-northeast-1/mistral.voxtral-mini-3b-2507"),
(
{"aws_region_name": "ap-southeast-2"},
{"AWS_REGION_NAME": "eu-west-2"},
"bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507",
),
],
)
def test_bedrock_transcription_bills_the_converse_token_usage(
region_kwargs: dict[str, str],
region_env: dict[str, str],
priced_model: str,
monkeypatch: pytest.MonkeyPatch,
) -> None:
for name in ("AWS_REGION_NAME", "AWS_REGION"):
monkeypatch.delenv(name, raising=False)
for name, value in region_env.items():
monkeypatch.setenv(name, value)
NATIVE_TRANSCRIPTION.override(UsageBridge())
response: Final = litellm.transcription(model=MODEL, file=AUDIO_FILE, **region_kwargs)
model_info: Final = litellm.get_model_info(priced_model)
expected_cost: Final = 390 * (model_info.get("input_cost_per_token") or 0.0) + 60 * (
model_info.get("output_cost_per_token") or 0.0
)
assert response.usage == TranscriptionUsageTokensObject(
type="tokens", input_tokens=390, output_tokens=60, total_tokens=450
)
assert expected_cost > 0
assert litellm.completion_cost(
completion_response=response, model=MODEL, call_type="transcription"
) == pytest.approx(expected_cost)
def test_bedrock_transcription_forwards_language_and_text_formats_to_rust() -> None:
bridge: Final = SyncBridge()
NATIVE_TRANSCRIPTION.override(bridge)
litellm.transcription(model=MODEL, file=AUDIO_FILE, language="fr", response_format="text", prompt="Names")
forwarded: Final = bridge.calls[0].resolved["optional_params"]
assert {key: forwarded.get(key) for key in ("language", "response_format", "prompt")} == {
"language": "fr",
"response_format": "text",
"prompt": "Names",
}
@pytest.mark.parametrize(("sent", "expected"), [("0.2", 0.2), ("0", 0.0), (0.5, 0.5), (1, 1.0)])
def test_bedrock_transcription_forwards_form_temperature_as_a_number(sent: object, expected: float) -> None:
bridge: Final = SyncBridge()
NATIVE_TRANSCRIPTION.override(bridge)
litellm.transcription(model=MODEL, file=AUDIO_FILE, temperature=sent)
forwarded: Final = bridge.calls[0].resolved["optional_params"]["temperature"]
assert (type(forwarded), forwarded) == (float, expected)
@pytest.mark.parametrize(
"unsupported",
[
{"response_format": "srt"},
{"response_format": "verbose_json"},
{"timestamp_granularities": ["word"]},
{"temperature": "warm"},
{"temperature": "inf"},
{"temperature": True},
],
)
def test_bedrock_transcription_rejects_params_it_cannot_honor(unsupported: dict[str, object]) -> None:
bridge: Final = SyncBridge()
NATIVE_TRANSCRIPTION.override(bridge)
with pytest.raises(litellm.BadRequestError) as raised:
litellm.transcription(model=MODEL, file=AUDIO_FILE, language="fr", **unsupported)
assert raised.value.status_code == 400
assert next(iter(unsupported)) in str(raised.value)
assert bridge.calls == ()
dropped: Final = litellm.transcription(model=MODEL, file=AUDIO_FILE, language="fr", drop_params=True, **unsupported)
assert dropped.text == "rust"
forwarded: Final = bridge.calls[0].resolved["optional_params"]
assert forwarded["language"] == "fr"
assert forwarded.keys().isdisjoint(unsupported)

View file

@ -7,9 +7,11 @@ regardless of the routing strategy being used.
import asyncio
from datetime import datetime, timedelta, timezone
from typing import Final
from unittest.mock import AsyncMock, MagicMock
import pytest
from pydantic import TypeAdapter
import litellm
from litellm import Router
@ -724,3 +726,159 @@ class TestDailyLimits:
router.discard()
assert served == ["a", "b", "b", "b"]
_LOGGED_TOKENS: Final = TypeAdapter[float | None](float | None)
async def _tpd_tokens_once_logged(cache: DualCache, key: str) -> int:
for _ in range(200):
logged: Final = _LOGGED_TOKENS.validate_python(
cache.in_memory_cache.get_cache(key) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # InMemoryCache.get_cache is untyped
)
if logged is not None:
return int(logged)
await asyncio.sleep(0.01)
pytest.fail(f"the success callback never logged tpd tokens under {key}")
RESET_AT_EIGHT = datetime(2026, 10, 28, 8, 0, 0, tzinfo=timezone.utc)
DAILY_RESET_DEPLOYMENT = {
**RPD_DEPLOYMENT,
"litellm_params": {"model": "gpt-4.1-mini", "rpd": 2, "daily_reset_time": "08:00"},
}
class TestDailyResetTime:
def test_rpd_window_starts_over_at_daily_reset_time_not_midnight(self):
clock = Clock(RESET_AT_EIGHT - timedelta(hours=9))
check = ModelRateLimitingCheck(dual_cache=DualCache(), clock=clock)
assert _pre_call(check, DAILY_RESET_DEPLOYMENT) == DAILY_RESET_DEPLOYMENT
clock.now = RESET_AT_EIGHT - timedelta(hours=7)
assert _pre_call(check, DAILY_RESET_DEPLOYMENT) == DAILY_RESET_DEPLOYMENT
clock.now = RESET_AT_EIGHT - timedelta(minutes=1)
with pytest.raises(litellm.RateLimitError, match="RPD limit=2, current usage=3") as exc_info:
_pre_call(check, DAILY_RESET_DEPLOYMENT)
assert exc_info.value.response.headers["retry-after"] == "60"
clock.now = RESET_AT_EIGHT
assert _pre_call(check, DAILY_RESET_DEPLOYMENT) == DAILY_RESET_DEPLOYMENT
@pytest.mark.asyncio
async def test_async_rpd_window_starts_over_at_daily_reset_time_not_midnight(self):
clock = Clock(RESET_AT_EIGHT - timedelta(hours=9))
check = ModelRateLimitingCheck(dual_cache=DualCache(), clock=clock)
assert await _async_pre_call(check, DAILY_RESET_DEPLOYMENT) == DAILY_RESET_DEPLOYMENT
clock.now = RESET_AT_EIGHT - timedelta(hours=7)
assert await _async_pre_call(check, DAILY_RESET_DEPLOYMENT) == DAILY_RESET_DEPLOYMENT
clock.now = RESET_AT_EIGHT - timedelta(minutes=1)
with pytest.raises(litellm.RateLimitError, match="RPD limit=2, current usage=3") as exc_info:
await _async_pre_call(check, DAILY_RESET_DEPLOYMENT)
assert exc_info.value.response.headers["retry-after"] == "60"
clock.now = RESET_AT_EIGHT
assert await _async_pre_call(check, DAILY_RESET_DEPLOYMENT) == DAILY_RESET_DEPLOYMENT
@pytest.mark.parametrize(
("now", "seconds_to_reset"),
[
(RESET_AT_EIGHT + timedelta(seconds=1), 86399),
(datetime(2026, 10, 28, 23, 59, 30, tzinfo=timezone.utc), 28830),
(datetime(2026, 10, 29, 7, 30, 0, tzinfo=timezone.utc), 1800),
],
)
def test_daily_429_retry_after_is_seconds_until_daily_reset_time(self, now: datetime, seconds_to_reset: int):
deployment = {
**RPD_DEPLOYMENT,
"litellm_params": {"model": "gpt-4.1-mini", "rpd": 0, "daily_reset_time": "08:00"},
}
check = ModelRateLimitingCheck(dual_cache=DualCache(), clock=Clock(now))
with pytest.raises(litellm.RateLimitError) as exc_info:
_pre_call(check, deployment)
assert exc_info.value.response.headers["retry-after"] == str(seconds_to_reset)
@pytest.mark.asyncio
async def test_tokens_logged_before_daily_reset_time_do_not_count_after_it(self):
deployment = {
**RPD_DEPLOYMENT,
"litellm_params": {"model": "gpt-4.1-mini", "tpd": 100, "daily_reset_time": "08:00"},
}
clock = Clock(RESET_AT_EIGHT - timedelta(minutes=1))
check = ModelRateLimitingCheck(dual_cache=DualCache(), clock=clock)
await _async_log_success(
check,
{
"litellm_params": {"daily_reset_time": "08:00", "metadata": {}},
"standard_logging_object": {
"model_id": "daily-test-id",
"total_tokens": 120,
"hidden_params": {"litellm_model_name": "gpt-4.1-mini"},
},
},
)
with pytest.raises(litellm.RateLimitError, match="TPD limit=100, current usage=120"):
await _async_pre_call(check, deployment)
clock.now = RESET_AT_EIGHT
assert await _async_pre_call(check, deployment) == deployment
@pytest.mark.parametrize(
"deployment",
[
{"litellm_params": {"model": "gpt-4.1-mini"}, "model_info": {"id": "x", "rpd": 0}, "model_name": "m"},
{"litellm_params": {"model": "gpt-4.1-mini"}, "model_info": {"id": "x"}, "model_name": "m", "rpd": 0},
],
)
def test_rpd_outside_litellm_params_is_not_a_limit(self, deployment: dict[str, object]):
check = ModelRateLimitingCheck(dual_cache=DualCache(), clock=Clock(DAY_ONE))
assert _pre_call(check, deployment) == deployment
@pytest.mark.parametrize("daily_reset_time", ["8am", "08:00:30", "08:00+05:00", "25:00"])
def test_router_rejects_a_daily_reset_time_that_is_not_hh_mm(self, daily_reset_time: str):
with pytest.raises(Exception, match="daily_reset_time"):
Router(
model_list=[
{
"model_name": "daily-group",
"litellm_params": {"model": "gpt-4.1-mini", "rpd": 2, "daily_reset_time": daily_reset_time},
}
]
)
@pytest.mark.asyncio
async def test_a_daily_reset_time_sent_in_the_request_does_not_move_the_tokens_out_of_the_deployment_window(self):
router = Router(
model_list=[
{
"model_name": "daily-group",
"litellm_params": {
"model": "openai/gpt-4.1-mini",
"api_key": "k",
"mock_response": "a",
"tpd": 100,
},
"model_info": {"id": "daily-test-id"},
}
]
)
check = ModelRateLimitingCheck(dual_cache=router.cache, clock=Clock(RESET_AT_EIGHT - timedelta(minutes=1)))
router.optional_callbacks = [check]
litellm.logging_callback_manager.add_litellm_callback(check)
try:
await router.acompletion( # pyright: ignore[reportUnknownMemberType] # Router.acompletion kwargs are untyped
model="daily-group", messages=[{"role": "user", "content": "hi"}], daily_reset_time="08:00"
)
logged = await _tpd_tokens_once_logged(router.cache, "daily-test-id:openai/gpt-4.1-mini:tpd:2026-10-28")
finally:
router.discard()
assert logged > 0
day_before: Final = _LOGGED_TOKENS.validate_python(
router.cache.in_memory_cache.get_cache("daily-test-id:openai/gpt-4.1-mini:tpd:2026-10-27") # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # InMemoryCache.get_cache is untyped
)
assert day_before is None

View file

@ -10883,3 +10883,23 @@ def test_get_optional_params_forwards_a_tool_config_cache_point_to_a_downstream_
)
assert optional_params["extra_body"] == {"kept": True, "cache_control_injection_points": [point]}
@pytest.mark.parametrize(
("provider", "expected_warnings"),
(("streamlake", 1), ("veniceai", 1), ("my-own-llm", 0)),
)
def test_custom_provider_named_like_a_built_in_warns_once(
provider: str, expected_warnings: int, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None:
monkeypatch.setattr(litellm, "custom_provider_map", [{"provider": provider, "custom_handler": litellm.CustomLLM()}])
monkeypatch.setattr(litellm, "provider_list", list(litellm.provider_list))
monkeypatch.setattr(litellm, "_custom_providers", [])
with caplog.at_level(logging.WARNING, logger=verbose_logger.name):
litellm.utils.custom_llm_setup()
litellm.utils.custom_llm_setup()
warnings: Final = [r for r in caplog.records if f"provider '{provider}'" in r.getMessage()]
assert len(warnings) == expected_warnings
assert provider in litellm._custom_providers

View file

@ -156,6 +156,7 @@ OPTION_NAMES: Final = (
"tpm",
"rpd",
"tpd",
"daily_reset_time",
"itpm",
"otpm",
"default_api_key_rpm_limit",

View file

@ -1,5 +1,5 @@
import { useSyncExternalStore } from "react";
import { act, renderWithProviders, screen, waitFor, within } from "../../../../../tests/test-utils";
import { act, fireEvent, renderWithProviders, screen, waitFor, within } from "../../../../../tests/test-utils";
import type { OnUrlUpdateFunction } from "nuqs/adapters/testing";
import userEvent, { PointerEventsCheckLevel } from "@testing-library/user-event";
import { beforeEach, describe, expect, it, vi } from "vitest";
@ -80,6 +80,13 @@ vi.mock("@/app/(dashboard)/hooks/providers/useProviderFields", () => ({
default_model_placeholder: "jev-latest",
credential_fields: [{ key: "api_key", label: "API Key", field_type: "password", required: false }],
},
{
provider: "StreamLake",
provider_display_name: "StreamLake",
litellm_provider: "streamlake",
default_model_placeholder: "streamlake/<model-or-endpoint>",
credential_fields: [{ key: "api_key", label: "API Key", field_type: "password", required: false }],
},
],
isLoading: false,
error: null,
@ -278,6 +285,28 @@ describe("AddModelPanel submit payload contract", () => {
});
});
it("creates a StreamLake model from a public model selection and API key", async () => {
mockUseModelCostMap.mockReturnValue({ data: { "streamlake/GLM-5.3": { litellm_provider: "streamlake" } } });
const { user, submit } = await setup();
await user.click(screen.getByRole("combobox", { name: /provider/i }));
await user.click(await screen.findByText("StreamLake"));
await user.click(screen.getByRole("combobox", { name: "Select models" }));
await user.click(await screen.findByText("streamlake/GLM-5.3"));
fireEvent.change(screen.getByLabelText("API Key"), { target: { value: "sk-streamlake-test" } });
await submit();
expect(lastCreatedModel()).toStrictEqual({
model_name: "streamlake/GLM-5.3",
litellm_params: {
model: "streamlake/GLM-5.3",
custom_llm_provider: "streamlake",
api_key: "sk-streamlake-test",
},
model_info: { ...baseModelInfo },
});
});
it("registers four more keys as undefined once Advanced Settings opens", async () => {
const { openAdvanced, fillRequired, submit } = await setup();
await fillRequired();

View file

@ -222,6 +222,7 @@ describe("provider_info_helpers", () => {
Providers.PG_VECTOR,
Providers.PREDIBASE,
Providers.Sail,
Providers.StreamLake,
Providers.WANDB,
Providers.ZAI,
Providers.ZeroGPU,
@ -462,6 +463,19 @@ describe("provider_info_helpers", () => {
expect(result).toEqual(["gpt-3.5-turbo", "gpt-4"]);
});
it("should return StreamLake models and exclude models from other providers", () => {
const modelMap = {
"streamlake/GLM-5.3": { litellm_provider: "streamlake" },
"streamlake/DeepSeek-V3": { litellm_provider: "streamlake" },
"gpt-4o": { litellm_provider: "openai" },
};
const result = getProviderModels(Providers.StreamLake, modelMap);
expect(provider_map.StreamLake).toBe("streamlake");
expect(result).toEqual(["streamlake/GLM-5.3", "streamlake/DeepSeek-V3"]);
expect(result).not.toContain("gpt-4o");
});
it("should return models whose litellm_provider is a prefix-anchored variant of the provider", () => {
const modelMap = {
"anthropic-text-model": { litellm_provider: "anthropic_text" },

View file

@ -177,6 +177,7 @@ export enum Providers {
Snowflake = "Snowflake",
Soniox = "Soniox",
StrandsDecider = "Strands Decider",
StreamLake = "StreamLake",
TEXT_COMPLETION_CODESTRAL = "Text-Completion-Codestral",
Tencent = "Tencent",
TogetherAI = "TogetherAI",
@ -306,6 +307,7 @@ export const provider_map: Record<string, string> = {
Snowflake: "snowflake",
Soniox: "soniox",
StrandsDecider: "strands_decider",
StreamLake: "streamlake",
TEXT_COMPLETION_CODESTRAL: "text-completion-codestral",
Tencent: "tencent",
TogetherAI: "together_ai",
@ -494,6 +496,7 @@ const providerPlaceholderMap: Partial<Record<Providers, string>> = {
[Providers.Tencent]: "tencent/deepseek-v4-pro",
[Providers.TypeSafe]: "typesafe/jev-latest",
[Providers.Nace]: "nace/drex-latest",
[Providers.StreamLake]: "streamlake/<model-or-endpoint>",
[Providers.Vertex_AI]: "gemini-pro",
[Providers.VolcEngine]: "volcengine/<any-model-on-volcengine>",
[Providers.Voyage]: "voyage/",

View file

@ -35397,6 +35397,8 @@ export interface components {
cost_per_second?: number | null;
/** Custom Llm Provider */
custom_llm_provider?: string | null;
/** Daily Reset Time */
daily_reset_time?: string | null;
/** Default Api Key Rpm Limit */
default_api_key_rpm_limit?: number | null;
/** Default Api Key Tpm Limit */
@ -50793,6 +50795,8 @@ export interface components {
cost_per_second?: number | null;
/** Custom Llm Provider */
custom_llm_provider?: string | null;
/** Daily Reset Time */
daily_reset_time?: string | null;
/** Default Api Key Rpm Limit */
default_api_key_rpm_limit?: number | null;
/** Default Api Key Tpm Limit */