mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge remote-tracking branch 'origin/main' into add-vispark-litellm
# Conflicts: # litellm/llms/openai_like/json_loader.py # litellm/llms/openai_like/providers.json
This commit is contained in:
commit
d4ce13aef8
83 changed files with 4295 additions and 380 deletions
|
|
@ -15,7 +15,10 @@ import litellm
|
|||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.guardrails._content_utils import iter_message_text
|
||||
from litellm.proxy.guardrails._content_utils import (
|
||||
is_non_conversational_request,
|
||||
iter_message_text,
|
||||
)
|
||||
from litellm.types.utils import CallTypesLiteral
|
||||
|
||||
_THRESHOLD_VALUE: Final = TypeAdapter(object)
|
||||
|
|
@ -94,7 +97,7 @@ class ENTERPRISE_GoogleTextModeration(CustomLogger):
|
|||
|
||||
async def async_moderation_hook(
|
||||
self,
|
||||
data: dict,
|
||||
data: dict[str, object],
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
call_type: CallTypesLiteral,
|
||||
):
|
||||
|
|
@ -102,6 +105,9 @@ class ENTERPRISE_GoogleTextModeration(CustomLogger):
|
|||
- Calls Google's Text Moderation API
|
||||
- Rejects request if it fails safety check
|
||||
"""
|
||||
if is_non_conversational_request(call_type, data):
|
||||
verbose_proxy_logger.debug("Google text moderation: skipping non-conversational call type %s", call_type)
|
||||
return
|
||||
# Covers multimodal list content + Responses-API input.
|
||||
text = "".join(iter_message_text(data))
|
||||
if text:
|
||||
|
|
|
|||
|
|
@ -33,9 +33,8 @@ class EnterpriseCallbackControls:
|
|||
|
||||
try:
|
||||
disabled_callbacks = EnterpriseCallbackControls.get_disabled_callbacks(litellm_params, standard_callback_dynamic_params)
|
||||
verbose_logger.debug(f"Dynamically disabled callbacks from {X_LITELLM_DISABLE_CALLBACKS}: {disabled_callbacks}")
|
||||
verbose_logger.debug(f"Checking if {callback} is disabled via headers. Disable callbacks from headers: {disabled_callbacks}")
|
||||
if disabled_callbacks is not None:
|
||||
verbose_logger.debug(f"Checking if {callback} is in dynamically disabled callbacks: {disabled_callbacks}")
|
||||
#########################################################
|
||||
# premium user check
|
||||
#########################################################
|
||||
|
|
|
|||
|
|
@ -1,14 +1,31 @@
|
|||
use serde_json::Value;
|
||||
|
||||
#[macro_rules_attribute::apply(crate::wire_type)]
|
||||
pub struct AudioTranscriptionUsage {
|
||||
pub input_tokens: u64,
|
||||
pub output_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::wire_type)]
|
||||
pub struct AudioTranscriptionResponseData {
|
||||
pub text: String,
|
||||
pub usage: Option<AudioTranscriptionUsage>,
|
||||
}
|
||||
|
||||
impl AudioTranscriptionResponseData {
|
||||
pub fn into_json(self) -> Value {
|
||||
serde_json::json!({
|
||||
"text": self.text,
|
||||
})
|
||||
match self.usage {
|
||||
None => serde_json::json!({"text": self.text}),
|
||||
Some(usage) => serde_json::json!({
|
||||
"text": self.text,
|
||||
"usage": {
|
||||
"type": "tokens",
|
||||
"input_tokens": usage.input_tokens,
|
||||
"output_tokens": usage.output_tokens,
|
||||
"total_tokens": usage.total_tokens,
|
||||
},
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,8 +6,8 @@ use litellm_auth_aws::{
|
|||
};
|
||||
use litellm_core_utils::core_helpers::json_type_name;
|
||||
use litellm_llms_types::{
|
||||
formats::audio_transcription::AudioTranscriptionResponseData,
|
||||
providers::bedrock::ConverseResponse,
|
||||
formats::audio_transcription::{AudioTranscriptionResponseData, AudioTranscriptionUsage},
|
||||
providers::bedrock::{ConverseResponse, ConverseUsage},
|
||||
};
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Map, Value, json};
|
||||
|
|
@ -101,10 +101,10 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
|
|||
if let Some(prompt) = optional_string(&optional_params, "prompt") {
|
||||
instruction.push_str(&format!(" Additional context: {prompt}"));
|
||||
}
|
||||
let mut inference_config = Map::from_iter([("maxTokens".to_string(), json!(4096))]);
|
||||
if let Some(temperature) = optional_params.get("temperature") {
|
||||
inference_config.insert("temperature".to_string(), temperature.clone());
|
||||
}
|
||||
let temperature = optional_params
|
||||
.get("temperature")
|
||||
.cloned()
|
||||
.unwrap_or(json!(0));
|
||||
Ok(AudioTranscriptionRequestData {
|
||||
body: json!({
|
||||
"messages": [{
|
||||
|
|
@ -114,8 +114,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
|
|||
{"text": instruction}
|
||||
]
|
||||
}],
|
||||
"system": [{"text": "You are a transcription assistant."}],
|
||||
"inferenceConfig": inference_config,
|
||||
"inferenceConfig": {"maxTokens": 4096, "temperature": temperature},
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
|
@ -125,6 +124,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
|
|||
_model: &str,
|
||||
response_json: Value,
|
||||
) -> Result<AudioTranscriptionResponseData, Error> {
|
||||
let reports_usage = response_json.get("usage").is_some();
|
||||
let response: ConverseResponse =
|
||||
serde_json::from_value(response_json).map_err(|error| {
|
||||
Error::InvalidResponse(ErrorDetail::invalid("Converse response", error))
|
||||
|
|
@ -137,6 +137,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
|
|||
}
|
||||
Ok(AudioTranscriptionResponseData {
|
||||
text: response.content_text(),
|
||||
usage: reports_usage.then(|| transcription_usage(&response.usage)),
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -191,22 +192,41 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
|
|||
}
|
||||
}
|
||||
|
||||
fn transcription_usage(usage: &ConverseUsage) -> AudioTranscriptionUsage {
|
||||
AudioTranscriptionUsage {
|
||||
input_tokens: usage.input_tokens,
|
||||
output_tokens: usage.output_tokens,
|
||||
total_tokens: usage
|
||||
.total_tokens
|
||||
.unwrap_or(usage.input_tokens + usage.output_tokens),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rstest::rstest;
|
||||
|
||||
fn no_env(_: &str) -> Option<String> {
|
||||
None
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_matches_python_shape() {
|
||||
let params = Map::from_iter([
|
||||
("language".to_string(), json!("en")),
|
||||
("prompt".to_string(), json!("Speaker names")),
|
||||
("temperature".to_string(), json!(0)),
|
||||
("timestamp_granularities".to_string(), json!(["word"])),
|
||||
]);
|
||||
#[rstest]
|
||||
#[case::caller_temperature(Some(json!(0.4)), json!(0.4))]
|
||||
#[case::default_temperature(None, json!(0))]
|
||||
fn request_sends_audio_and_instruction_without_system_block(
|
||||
#[case] temperature: Option<Value>,
|
||||
#[case] expected_temperature: Value,
|
||||
) {
|
||||
let params = Map::from_iter(
|
||||
[
|
||||
("language".to_string(), json!("en")),
|
||||
("prompt".to_string(), json!("Speaker names")),
|
||||
("timestamp_granularities".to_string(), json!(["word"])),
|
||||
]
|
||||
.into_iter()
|
||||
.chain(temperature.map(|value| ("temperature".to_string(), value))),
|
||||
);
|
||||
let params = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG.map_transcription_params(¶ms);
|
||||
let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG
|
||||
.transform_audio_transcription_request(
|
||||
|
|
@ -225,22 +245,39 @@ mod tests {
|
|||
{"text": "Transcribe the audio. Respond with only the transcript. The audio language is en. Additional context: Speaker names"}
|
||||
]
|
||||
}],
|
||||
"system": [{"text": "You are a transcription assistant."}],
|
||||
"inferenceConfig": {"maxTokens": 4096, "temperature": 0}
|
||||
"inferenceConfig": {"maxTokens": 4096, "temperature": expected_temperature}
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_concatenates_content_blocks() {
|
||||
#[rstest]
|
||||
#[case::reported_total(
|
||||
json!({"inputTokens": 390, "outputTokens": 60, "totalTokens": 450}),
|
||||
json!({"text": "hello world", "usage": {"type": "tokens", "input_tokens": 390, "output_tokens": 60, "total_tokens": 450}})
|
||||
)]
|
||||
#[case::derived_total(
|
||||
json!({"inputTokens": 1, "outputTokens": 2}),
|
||||
json!({"text": "hello world", "usage": {"type": "tokens", "input_tokens": 1, "output_tokens": 2, "total_tokens": 3}})
|
||||
)]
|
||||
fn response_carries_text_and_converse_usage(#[case] usage: Value, #[case] expected: Value) {
|
||||
let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG
|
||||
.transform_audio_transcription_response(
|
||||
"model",
|
||||
json!({"output": {"message": {"content": [{"text": "hello "}, {"text": "world"}]}}, "usage": {"inputTokens": 1, "outputTokens": 2}}),
|
||||
json!({"output": {"message": {"content": [{"text": "hello "}, {"text": "world"}]}}, "usage": usage}),
|
||||
)
|
||||
.expect("response");
|
||||
assert_eq!(result.text, "hello world");
|
||||
assert_eq!(result.into_json(), json!({"text": "hello world"}));
|
||||
assert_eq!(result.into_json(), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn response_without_usage_reports_none() {
|
||||
let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG
|
||||
.transform_audio_transcription_response(
|
||||
"model",
|
||||
json!({"output": {"message": {"content": [{"text": "hello"}]}}}),
|
||||
)
|
||||
.expect("response");
|
||||
assert_eq!(result.into_json(), json!({"text": "hello"}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -681,6 +681,7 @@ moonshot_models: Set = set()
|
|||
publicai_models: Set = set()
|
||||
darkbloom_models: Set = set()
|
||||
openinfer_models: Final[Set[str]] = set()
|
||||
streamlake_models: Final[Set[str]] = set()
|
||||
zerogpu_models: Final[Set[str]] = set()
|
||||
v0_models: Set = set()
|
||||
vispark_models: Set[str] = set()
|
||||
|
|
@ -971,6 +972,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
|
|||
darkbloom_models.add(key)
|
||||
elif value.get("litellm_provider") == "openinfer":
|
||||
openinfer_models.add(key)
|
||||
elif value.get("litellm_provider") == "streamlake":
|
||||
streamlake_models.add(key)
|
||||
elif value.get("litellm_provider") == "zerogpu":
|
||||
zerogpu_models.add(key)
|
||||
elif value.get("litellm_provider") == "v0":
|
||||
|
|
@ -1148,6 +1151,7 @@ model_list = list(
|
|||
| publicai_models
|
||||
| darkbloom_models
|
||||
| openinfer_models
|
||||
| streamlake_models
|
||||
| zerogpu_models
|
||||
| v0_models
|
||||
| vispark_models
|
||||
|
|
@ -1269,6 +1273,7 @@ def _build_models_by_provider() -> dict:
|
|||
"publicai": publicai_models,
|
||||
"darkbloom": darkbloom_models,
|
||||
"openinfer": openinfer_models,
|
||||
"streamlake": streamlake_models,
|
||||
"zerogpu": zerogpu_models,
|
||||
"v0": v0_models,
|
||||
"vispark": vispark_models,
|
||||
|
|
|
|||
|
|
@ -1065,6 +1065,7 @@ openai_compatible_endpoints: Final[list[str]] = [
|
|||
"https://api.reka.ai/v1",
|
||||
"https://gigachat.devices.sberbank.ru/api/v1",
|
||||
"https://api.openinfer.ai/v1",
|
||||
"https://vanchin.streamlake.ai/api/gateway/v1/endpoints",
|
||||
]
|
||||
|
||||
|
||||
|
|
@ -1147,6 +1148,7 @@ openai_compatible_providers: Final[list[str]] = [
|
|||
"prism",
|
||||
"sail",
|
||||
"reka",
|
||||
"streamlake",
|
||||
]
|
||||
|
||||
OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS: Final = frozenset({"openai"} | frozenset(openai_compatible_providers))
|
||||
|
|
|
|||
|
|
@ -693,7 +693,7 @@ def cost_per_token(
|
|||
elif call_type == "atranscription" or call_type == "transcription":
|
||||
if _transcription_usage_has_token_details(usage_block):
|
||||
return generic_cost_per_token(
|
||||
model=model_without_prefix,
|
||||
model=model,
|
||||
usage=usage_block,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
service_tier=service_tier,
|
||||
|
|
|
|||
|
|
@ -77,6 +77,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
|
|||
"otpm",
|
||||
"tpd",
|
||||
"rpd",
|
||||
"daily_reset_time",
|
||||
"use_xai_oauth",
|
||||
"chatgpt_auth_file",
|
||||
"github_copilot_auth_type",
|
||||
|
|
|
|||
|
|
@ -1130,9 +1130,6 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
}
|
||||
self.litellm_request_debug = litellm_params.get("litellm_request_debug", False)
|
||||
self.logger_fn = litellm_params.get("logger_fn", None)
|
||||
if is_debugging_on() or self.litellm_request_debug:
|
||||
verbose_logger.debug("self.optional_params: %s", self.optional_params)
|
||||
|
||||
self.model_call_details.update(
|
||||
{
|
||||
"model": self.model,
|
||||
|
|
|
|||
|
|
@ -27,13 +27,18 @@ class TranscriptionUsageObjectTransformation:
|
|||
if isinstance(usage_object, TranscriptionUsageDurationObject):
|
||||
return None
|
||||
elif isinstance(usage_object, TranscriptionUsageTokensObject):
|
||||
input_token_details: Final = usage_object.input_token_details
|
||||
return Usage(
|
||||
prompt_tokens=usage_object.input_tokens,
|
||||
completion_tokens=usage_object.output_tokens,
|
||||
total_tokens=usage_object.total_tokens,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
text_tokens=usage_object.input_token_details.text_tokens,
|
||||
audio_tokens=usage_object.input_token_details.audio_tokens,
|
||||
prompt_tokens_details=(
|
||||
None
|
||||
if input_token_details is None
|
||||
else PromptTokensDetailsWrapper(
|
||||
text_tokens=input_token_details.text_tokens,
|
||||
audio_tokens=input_token_details.audio_tokens,
|
||||
)
|
||||
),
|
||||
)
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -92,6 +92,18 @@ def _next_sync_or_exhausted(it: Iterator[object]) -> object:
|
|||
return _SYNC_ITER_EXHAUSTED
|
||||
|
||||
|
||||
def _stamp_usage_cost(
|
||||
usage_chunk: ModelResponse | TextCompletionResponse | ModelResponseStream,
|
||||
complete_response: ModelResponse | TextCompletionResponse | ModelResponseStream,
|
||||
) -> None:
|
||||
bill: Final = (get_hidden_params(complete_response) or {}).get("response_cost")
|
||||
usage: Final[object] = getattr(usage_chunk, "usage", None)
|
||||
if usage is None or isinstance(bill, bool) or not isinstance(bill, (int, float)):
|
||||
return
|
||||
if bill > 0 or isinstance(getattr(usage, "cost", None), (int, float)):
|
||||
setattr(usage, "cost", float(bill)) # noqa: B010 # cost is an optional extra on the usage model
|
||||
|
||||
|
||||
def _stamp_served_service_tier(response: ModelResponseStream, complete_streaming_response: ModelResponse) -> None:
|
||||
served_tier: Final = complete_streaming_response.model_dump().get("service_tier")
|
||||
if isinstance(served_tier, str) and served_tier:
|
||||
|
|
@ -2141,6 +2153,7 @@ class CustomStreamWrapper:
|
|||
getattr(complete_streaming_response, "usage"),
|
||||
)
|
||||
_stamp_served_service_tier(response, complete_streaming_response)
|
||||
_stamp_usage_cost(response, complete_streaming_response)
|
||||
try:
|
||||
_cache_copy = complete_streaming_response.model_copy(deep=True)
|
||||
_log_copy = complete_streaming_response.model_copy(deep=True)
|
||||
|
|
@ -2361,6 +2374,7 @@ class CustomStreamWrapper:
|
|||
getattr(complete_streaming_response, "usage"),
|
||||
)
|
||||
_stamp_served_service_tier(response, complete_streaming_response)
|
||||
_stamp_usage_cost(response, complete_streaming_response)
|
||||
try:
|
||||
_copy = complete_streaming_response.model_copy(deep=True)
|
||||
except RuntimeError:
|
||||
|
|
|
|||
|
|
@ -621,7 +621,7 @@ class BaseAzureLLM(BaseOpenAILLM):
|
|||
|
||||
# For Azure v1 API, use standard OpenAI client instead of AzureOpenAI
|
||||
# See: https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#api-specs
|
||||
if self._is_azure_v1_api_version(api_version):
|
||||
if self.is_azure_v1_api_version(api_version):
|
||||
# Extract only params that OpenAI client accepts
|
||||
# Always use /openai/v1/ regardless of whether user passed "v1", "latest", or "preview"
|
||||
# The OpenAI client accepts a callable for `api_key` and re-invokes it
|
||||
|
|
@ -922,7 +922,7 @@ class BaseAzureLLM(BaseOpenAILLM):
|
|||
else:
|
||||
new_url = api_base
|
||||
|
||||
if BaseAzureLLM._is_azure_v1_api_version(api_version):
|
||||
if BaseAzureLLM.is_azure_v1_api_version(api_version):
|
||||
# ensure the request go to /openai/v1 and not just /openai
|
||||
if "/openai/v1" not in new_url:
|
||||
parsed_url: Final = httpx.URL(new_url)
|
||||
|
|
@ -952,7 +952,7 @@ class BaseAzureLLM(BaseOpenAILLM):
|
|||
|
||||
Returns None when ``api_version`` is a dated one, which still uses the deployment route.
|
||||
"""
|
||||
if not BaseAzureLLM._is_azure_v1_api_version(api_version):
|
||||
if not BaseAzureLLM.is_azure_v1_api_version(api_version):
|
||||
return None
|
||||
|
||||
base_url: Final = httpx.URL(api_base)
|
||||
|
|
@ -970,7 +970,7 @@ class BaseAzureLLM(BaseOpenAILLM):
|
|||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_azure_v1_api_version(api_version: str | None) -> bool:
|
||||
def is_azure_v1_api_version(api_version: str | None) -> bool:
|
||||
if api_version is None:
|
||||
return False
|
||||
return api_version in {"preview", "latest", "v1"}
|
||||
|
|
|
|||
|
|
@ -54,6 +54,16 @@ def _resolve_responses_api_version(api_base: str | None, litellm_params: Mapping
|
|||
return api_version
|
||||
|
||||
|
||||
def _v1_responses_url(api_base: str) -> httpx.URL:
|
||||
parsed_url: Final = httpx.URL(api_base)
|
||||
path: Final = parsed_url.path.rstrip("/")
|
||||
root: Final = next(
|
||||
(path.removesuffix(route) for route in ("/openai/v1/responses", "/openai/responses") if path.endswith(route)),
|
||||
path,
|
||||
)
|
||||
return parsed_url.copy_with(path=f"{root}/openai/v1/responses", query=None)
|
||||
|
||||
|
||||
def _with_responses_api_version(api_base: str | None, litellm_params: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return MappingProxyType({**litellm_params, "api_version": _resolve_responses_api_version(api_base, litellm_params)})
|
||||
|
||||
|
|
@ -247,20 +257,20 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
if api_base is None:
|
||||
raise ValueError("api_base is required for Azure WebSocket")
|
||||
|
||||
parsed_url: Final = httpx.URL(api_base)
|
||||
path = parsed_url.path.rstrip("/")
|
||||
# Strip existing /openai/responses path if the api_base already contains it
|
||||
for suffix in ("/openai/v1/responses", "/openai/responses"):
|
||||
if path.endswith(suffix):
|
||||
path = path[: -len(suffix)]
|
||||
break
|
||||
scheme: Final = "wss" if parsed_url.scheme == "https" else "ws"
|
||||
return str(parsed_url.copy_with(scheme=scheme, path=f"{path}/openai/v1/responses", query=None))
|
||||
v1_url: Final = _v1_responses_url(api_base)
|
||||
return str(v1_url.copy_with(scheme="wss" if v1_url.scheme == "https" else "ws"))
|
||||
|
||||
def model_in_websocket_url(self) -> bool:
|
||||
# Azure sends the model in the response.create body, not the URL
|
||||
return False
|
||||
|
||||
def compact_url(self, api_base: str) -> str:
|
||||
params: Final = httpx.URL(api_base).params
|
||||
v1_params: Final = (
|
||||
params if BaseAzureLLM.is_azure_v1_api_version(params.get("api-version")) else params.remove("api-version")
|
||||
)
|
||||
return super().compact_url(str(_v1_responses_url(api_base).copy_with(params=v1_params)))
|
||||
|
||||
#########################################################
|
||||
########## DELETE RESPONSE API TRANSFORMATION ##############
|
||||
#########################################################
|
||||
|
|
|
|||
|
|
@ -1,7 +1,11 @@
|
|||
import base64
|
||||
import math
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
from litellm.litellm_core_utils.audio_utils.utils import process_audio_file
|
||||
from litellm.rust_bridge import runtime
|
||||
|
|
@ -14,7 +18,72 @@ from litellm.rust_bridge.transcription.native import (
|
|||
RustAtranscription,
|
||||
RustTranscription,
|
||||
)
|
||||
from litellm.types.utils import FileTypes, TranscriptionResponse
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.utils import FileTypes, TranscriptionResponse, TranscriptionUsageTokensObject
|
||||
|
||||
SUPPORTED_OPENAI_TRANSCRIPTION_PARAMS: Final = ("language", "prompt", "temperature", "response_format")
|
||||
TEXT_RESPONSE_FORMATS: Final = frozenset({"json", "text"})
|
||||
|
||||
|
||||
def _as_float(value: object) -> float | None:
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float, str)):
|
||||
return None
|
||||
try:
|
||||
number: Final = float(value)
|
||||
except ValueError:
|
||||
return None
|
||||
return number if math.isfinite(number) else None
|
||||
|
||||
|
||||
def _rejection(key: str, value: object) -> str | None:
|
||||
if key not in SUPPORTED_OPENAI_TRANSCRIPTION_PARAMS:
|
||||
return f"bedrock transcription does not support {key}."
|
||||
if key == "response_format" and value not in TEXT_RESPONSE_FORMATS:
|
||||
return f"bedrock transcription returns only the transcript text, so response_format={value!r} is not supported."
|
||||
if key == "temperature" and _as_float(value) is None:
|
||||
return f"bedrock transcription needs a number for temperature, got {value!r}."
|
||||
return None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class UnsupportedTranscriptionParams:
|
||||
message: str
|
||||
|
||||
|
||||
def map_openai_transcription_params(
|
||||
non_default_params: Mapping[str, object], drop_params: bool
|
||||
) -> Mapping[str, object] | UnsupportedTranscriptionParams:
|
||||
rejections: Final = {
|
||||
key: reason for key, value in non_default_params.items() if (reason := _rejection(key, value)) is not None
|
||||
}
|
||||
if rejections and not drop_params:
|
||||
return UnsupportedTranscriptionParams(
|
||||
message=f"{' '.join(rejections.values())} To drop it from the call, set `litellm.drop_params = True`."
|
||||
)
|
||||
return {
|
||||
key: _as_float(value) if key == "temperature" else value
|
||||
for key, value in non_default_params.items()
|
||||
if key not in rejections
|
||||
}
|
||||
|
||||
|
||||
class _RustTranscriptionResult(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
text: str
|
||||
usage: TranscriptionUsageTokensObject | None = None
|
||||
|
||||
|
||||
def _transcription_response(
|
||||
result: Mapping[str, object], optional_params: Mapping[str, object]
|
||||
) -> TranscriptionResponse:
|
||||
parsed: Final = _RustTranscriptionResult.model_validate(result)
|
||||
response: Final = TranscriptionResponse(text=parsed.text).model_copy(update={"usage": parsed.usage})
|
||||
region_name: Final = (
|
||||
optional_params.get("aws_region_name") or get_secret_str("AWS_REGION_NAME") or get_secret_str("AWS_REGION")
|
||||
)
|
||||
response.hidden_params = {**response.hidden_params, "region_name": region_name}
|
||||
return response
|
||||
|
||||
|
||||
class BedrockAudioTranscriptionRustDispatch:
|
||||
|
|
@ -64,7 +133,7 @@ class BedrockAudioTranscriptionRustDispatch:
|
|||
"timeout_seconds": timeout_to_seconds(timeout),
|
||||
}
|
||||
call: Final = NativeCall(args=(), kwargs=fields, base={})
|
||||
return TranscriptionResponse(**rust(call))
|
||||
return _transcription_response(rust(call), optional_params)
|
||||
|
||||
return runtime.run(
|
||||
RouteContext(Route.TRANSCRIPTION, provider=custom_llm_provider, model=model),
|
||||
|
|
@ -97,7 +166,7 @@ class BedrockAudioTranscriptionRustDispatch:
|
|||
"timeout_seconds": timeout_to_seconds(timeout),
|
||||
}
|
||||
call: Final = NativeCall(args=(), kwargs=fields, base={})
|
||||
return TranscriptionResponse(**await rust(call))
|
||||
return _transcription_response(await rust(call), optional_params)
|
||||
|
||||
return await runtime.arun(
|
||||
RouteContext(Route.TRANSCRIPTION, provider=custom_llm_provider, model=model),
|
||||
|
|
|
|||
|
|
@ -32,11 +32,11 @@ import copy
|
|||
import json
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Mapping, MutableSequence, Sequence
|
||||
from collections.abc import Iterable, Mapping, MutableSequence, Sequence
|
||||
from dataclasses import dataclass
|
||||
from itertools import accumulate, chain, repeat
|
||||
from itertools import accumulate, chain, groupby, repeat
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, Union, cast
|
||||
from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, TypeVar, Union, cast
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
|
@ -55,7 +55,9 @@ from litellm.llms.base_llm.guardrail_translation.base_translation import (
|
|||
)
|
||||
from litellm.llms.base_llm.guardrail_translation.utils import (
|
||||
blocked_responses_stream_usage,
|
||||
effective_scan_only_tool_results_for_guardrail,
|
||||
effective_skip_system_message_for_guardrail,
|
||||
effective_skip_tool_message_for_guardrail,
|
||||
list_field,
|
||||
merge_guardrailed_scoped_messages,
|
||||
role_out_of_guardrail_scope,
|
||||
|
|
@ -65,7 +67,10 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
|
|||
stream_item_items,
|
||||
unappliable_request_rewrite,
|
||||
)
|
||||
from litellm.llms.openai.responses.guardrail_translation.tool_merge import merge_guardrailed_tools
|
||||
from litellm.llms.openai.responses.guardrail_translation.tool_merge import (
|
||||
append_returned_tools,
|
||||
merge_guardrailed_tools,
|
||||
)
|
||||
from litellm.responses.litellm_completion_transformation.compaction_items import (
|
||||
compaction_summary,
|
||||
decode_compaction_item,
|
||||
|
|
@ -74,7 +79,9 @@ from litellm.responses.litellm_completion_transformation.compaction_items import
|
|||
from litellm.responses.litellm_completion_transformation.custom_tools import without_malformed_tool_flags
|
||||
from litellm.responses.litellm_completion_transformation.tool_search import is_server_tool_search_tool
|
||||
from litellm.responses.litellm_completion_transformation.transformation import (
|
||||
TOOL_CALL_OUTPUT_ITEM_TYPES,
|
||||
LiteLLMCompletionResponsesConfig,
|
||||
tool_output_to_chat_content,
|
||||
)
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.llms.openai import (
|
||||
|
|
@ -270,6 +277,7 @@ _TOOL_CALL_PAYLOAD_EVENT_TYPES: Final = _TOOL_CALL_PAYLOAD_DELTA_EVENT_TYPES | f
|
|||
)
|
||||
_OUTPUT_ITEM_EVENT_TYPES: Final = frozenset({"response.output_item.added", "response.output_item.done"})
|
||||
_OUTPUT_TEXT_EVENT_TYPES: Final = frozenset({"response.output_text.delta", "response.output_text.done"})
|
||||
_TOOL_OUTPUT_TEXT_PART_TYPES: Final = frozenset({"input_text", "output_text", "text"})
|
||||
_PATCHABLE_ITEM_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
|
||||
{"function_call_output": "output", "custom_tool_call_output": "output", "message": "content"}
|
||||
)
|
||||
|
|
@ -519,10 +527,109 @@ def scannable_instructions(data: Mapping[str, object], *, skip_system: bool = Fa
|
|||
|
||||
|
||||
def _input_item_role(item: object) -> str:
|
||||
if stream_item_field(item, "type") in TOOL_CALL_OUTPUT_ITEM_TYPES:
|
||||
return "tool"
|
||||
role: Final = item.get("role") if isinstance(item, Mapping) else None
|
||||
return role.lower() if isinstance(role, str) else ""
|
||||
|
||||
|
||||
def _scanned_text_field(item: Mapping[str, object]) -> str:
|
||||
return "output" if item.get("type") in TOOL_CALL_OUTPUT_ITEM_TYPES else "content"
|
||||
|
||||
|
||||
def _tool_output_part_text(part: object) -> str | None:
|
||||
if not isinstance(part, dict):
|
||||
return None
|
||||
fields: Final = _item_fields_adapter.validate_python(part)
|
||||
text: Final = fields.get("text")
|
||||
return text if fields.get("type") in _TOOL_OUTPUT_TEXT_PART_TYPES and isinstance(text, str) else None
|
||||
|
||||
|
||||
def tool_output_texts(item: object) -> tuple[tuple[int | None, str], ...]:
|
||||
if not stream_item_field(item, "call_id") or stream_item_field(item, "type") == "computer_call_output":
|
||||
return ()
|
||||
parts: Final = stream_item_items(item, "output")
|
||||
part_texts: Final = tuple(
|
||||
(index, text) for index, part in enumerate(parts) if (text := _tool_output_part_text(part))
|
||||
)
|
||||
if part_texts:
|
||||
return part_texts
|
||||
tool_content: Final = tool_output_to_chat_content(stream_item_field(item, "output"))
|
||||
return ((None, tool_content),) if isinstance(tool_content, str) and tool_content else ()
|
||||
|
||||
|
||||
_Grouped: Final = TypeVar("_Grouped")
|
||||
|
||||
|
||||
def _pair_call_id(pair: tuple[str, object]) -> str:
|
||||
return pair[0]
|
||||
|
||||
|
||||
def _grouped_by_call_id(pairs: Iterable[tuple[str, _Grouped]]) -> Mapping[str, tuple[_Grouped, ...]]:
|
||||
ordered: Final = sorted(pairs, key=_pair_call_id)
|
||||
return MappingProxyType(
|
||||
{call_id: tuple(value for _, value in group) for call_id, group in groupby(ordered, key=_pair_call_id)}
|
||||
)
|
||||
|
||||
|
||||
def _texts_by_tool_row(
|
||||
messages: Sequence[Mapping[str, object]], input_data: "str | ResponseInputParam"
|
||||
) -> Mapping[int, tuple[str, ...]]:
|
||||
if isinstance(input_data, str):
|
||||
return MappingProxyType({})
|
||||
output_texts_by_call_id: Final = _grouped_by_call_id(
|
||||
(str(stream_item_field(item, "call_id")), tuple(text for _, text in tool_output_texts(item)))
|
||||
for item in input_data
|
||||
if stream_item_field(item, "type") in TOOL_CALL_OUTPUT_ITEM_TYPES
|
||||
)
|
||||
row_indices_by_call_id: Final = _grouped_by_call_id(
|
||||
(str(message.get("tool_call_id")), index)
|
||||
for index, message in enumerate(messages)
|
||||
if message.get("role") == "tool"
|
||||
)
|
||||
return MappingProxyType(
|
||||
dict(
|
||||
chain.from_iterable(
|
||||
zip(indices, output_texts_by_call_id[call_id], strict=True)
|
||||
for call_id, indices in row_indices_by_call_id.items()
|
||||
if len(indices) == len(output_texts_by_call_id.get(call_id, ()))
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _is_string_tool_row(message: Mapping[str, object]) -> bool:
|
||||
return message.get("role") == "tool" and isinstance(message.get("content"), str)
|
||||
|
||||
|
||||
def _split_into_text_parts(message: AllMessageValues, texts: tuple[str, ...]) -> AllMessageValues:
|
||||
if not _is_string_tool_row(message) or len(texts) < 2:
|
||||
return message
|
||||
return cast( # cast-ok: a tool row whose content is its text parts as chat text blocks
|
||||
"AllMessageValues", {**message, "content": [{"type": "text", "text": text} for text in texts]}
|
||||
)
|
||||
|
||||
|
||||
def _with_split_rows_restored(
|
||||
rows: Sequence[AllMessageValues], split_rows: Sequence[tuple[AllMessageValues, AllMessageValues]]
|
||||
) -> Sequence[AllMessageValues]:
|
||||
if not isinstance(rows, list):
|
||||
return rows
|
||||
return [next((original for split, original in split_rows if row == split), row) for row in rows]
|
||||
|
||||
|
||||
def _is_textless_tool_row(message: Mapping[str, object], texts: tuple[str, ...] | None) -> bool:
|
||||
return _is_string_tool_row(message) and (not message.get("content") or texts == ())
|
||||
|
||||
|
||||
def _rewritten_whole_text(item: object, field: str, guardrail_response: str) -> object | None:
|
||||
if isinstance(stream_item_field(item, field), str):
|
||||
return guardrail_response
|
||||
if tool_output_texts(item) == ((None, guardrail_response),):
|
||||
return None
|
||||
return guardrail_response
|
||||
|
||||
|
||||
def _patched_request_fields(
|
||||
raw_input: object,
|
||||
instructions: object,
|
||||
|
|
@ -636,17 +743,28 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
|
||||
Handles both string input and list of message objects.
|
||||
"""
|
||||
input_data: Final[str | ResponseInputParam | None] = data.get("input")
|
||||
if not isinstance(input_data, (str, list)):
|
||||
raw_input: Final[str | ResponseInputParam | None] = data.get("input")
|
||||
if not isinstance(raw_input, (str, list)):
|
||||
return data
|
||||
input_data: Final = cast("str | ResponseInputParam", raw_input) # cast-ok: fields read via stream_item_field
|
||||
skip_system: Final = effective_skip_system_message_for_guardrail(guardrail_to_apply)
|
||||
skip_tool: Final = effective_skip_tool_message_for_guardrail(guardrail_to_apply)
|
||||
scan_only: Final = effective_scan_only_tool_results_for_guardrail(guardrail_to_apply)
|
||||
structured_messages: Final = self.get_structured_messages(data)
|
||||
scoped_indices: Final = scoped_structured_message_indices(
|
||||
structured_messages or [], scan_only_tool_results=False, skip_system=skip_system, skip_tool=False
|
||||
all_messages: Final = structured_messages or []
|
||||
texts_by_tool_row: Final = _texts_by_tool_row(all_messages, input_data)
|
||||
scoped_indices: Final = tuple(
|
||||
index
|
||||
for index in scoped_structured_message_indices(
|
||||
all_messages, scan_only_tool_results=scan_only, skip_system=skip_system, skip_tool=skip_tool
|
||||
)
|
||||
if not _is_textless_tool_row(all_messages[index], texts_by_tool_row.get(index))
|
||||
)
|
||||
scoped_structured_messages: Final = (
|
||||
[structured_messages[index] for index in scoped_indices] if structured_messages else None
|
||||
split_rows: Final = tuple(
|
||||
(_split_into_text_parts(all_messages[index], texts_by_tool_row.get(index, ())), all_messages[index])
|
||||
for index in scoped_indices
|
||||
)
|
||||
scoped_structured_messages: Final = [split for split, _ in split_rows] if structured_messages else None
|
||||
raw_tools: Final = data.get("tools")
|
||||
original_tools: Final[tuple[Mapping[str, object], ...]] = (
|
||||
tuple(raw_tools) if isinstance(raw_tools, list) else ()
|
||||
|
|
@ -661,7 +779,12 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
)
|
||||
)
|
||||
extracted: Final = self._extract_guardrail_inputs(
|
||||
data, input_data, flattened_tool_groups, skip_system=skip_system
|
||||
data,
|
||||
input_data,
|
||||
() if scan_only else flattened_tool_groups,
|
||||
skip_system=skip_system,
|
||||
skip_tool=skip_tool,
|
||||
scan_only=scan_only,
|
||||
)
|
||||
if not extracted.inputs.get("texts"):
|
||||
return data
|
||||
|
|
@ -674,13 +797,14 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
logging_obj=litellm_logging_obj,
|
||||
)
|
||||
self._apply_guardrailed_tools_to_data(
|
||||
data, original_tools, flattened_tool_groups, guardrailed_inputs.get("tools")
|
||||
data, original_tools, flattened_tool_groups, guardrailed_inputs.get("tools"), scan_only=scan_only
|
||||
)
|
||||
written_back: Final = self._written_back_request_fields(
|
||||
data,
|
||||
structured_messages or (),
|
||||
scoped_indices,
|
||||
scoped_structured_messages,
|
||||
tuple((split, original) for split, original in split_rows if split is not original),
|
||||
guardrail_to_apply,
|
||||
guardrailed_inputs,
|
||||
)
|
||||
|
|
@ -730,8 +854,10 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
flattened_tool_groups: Sequence[Sequence[Mapping[str, object]]],
|
||||
*,
|
||||
skip_system: bool = False,
|
||||
skip_tool: bool = False,
|
||||
scan_only: bool = False,
|
||||
) -> _ExtractedInputs:
|
||||
instructions: Final = scannable_instructions(data, skip_system=skip_system)
|
||||
instructions: Final = scannable_instructions(data, skip_system=skip_system or scan_only)
|
||||
texts_to_check: Final[list[str]] = [] if instructions is None else [instructions]
|
||||
images_to_check: Final[list[str]] = []
|
||||
task_mappings: Final[list[tuple[int, int | None]]] = []
|
||||
|
|
@ -745,11 +871,15 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
)
|
||||
)
|
||||
if isinstance(input_data, str):
|
||||
texts_to_check.append(input_data)
|
||||
if not scan_only:
|
||||
texts_to_check.append(input_data)
|
||||
else:
|
||||
for msg_idx, message in enumerate(input_data):
|
||||
if role_out_of_guardrail_scope(
|
||||
_input_item_role(message), skip_system_message=skip_system, skip_tool_message=False
|
||||
_input_item_role(message),
|
||||
skip_system_message=skip_system,
|
||||
skip_tool_message=skip_tool,
|
||||
scan_only_tool_results=scan_only,
|
||||
):
|
||||
continue
|
||||
self._extract_input_text_and_images(
|
||||
|
|
@ -775,12 +905,14 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
structured_messages: Sequence[AllMessageValues],
|
||||
scoped_indices: Sequence[int],
|
||||
scoped_structured_messages: Sequence[AllMessageValues] | None,
|
||||
split_rows: Sequence[tuple[AllMessageValues, AllMessageValues]],
|
||||
guardrail_to_apply: "CustomGuardrail",
|
||||
guardrailed_inputs: GenericGuardrailAPIInputs,
|
||||
) -> _RequestFields | None:
|
||||
guardrailed: Final = guardrailed_inputs.get("structured_messages")
|
||||
if guardrailed is None or guardrailed is scoped_structured_messages:
|
||||
returned: Final = guardrailed_inputs.get("structured_messages")
|
||||
if returned is None or returned is scoped_structured_messages:
|
||||
return None
|
||||
guardrailed: Final = _with_split_rows_restored(returned, split_rows)
|
||||
covers_full_request: Final = len(scoped_indices) == len(structured_messages) or (
|
||||
guardrail_to_apply.structured_messages_cover_full_request() and len(guardrailed) == len(structured_messages)
|
||||
)
|
||||
|
|
@ -814,11 +946,14 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
original_tools: Sequence[Mapping[str, object]],
|
||||
flattened_tool_groups: Sequence[Sequence[Mapping[str, object]]],
|
||||
guardrailed_tools: Sequence[ChatCompletionToolParam] | None,
|
||||
*,
|
||||
scan_only: bool,
|
||||
) -> None:
|
||||
if guardrailed_tools is None:
|
||||
return
|
||||
merge: Final = append_returned_tools if scan_only else merge_guardrailed_tools
|
||||
data["tools"] = list( # rebind-ok: in-place request rewrite
|
||||
merge_guardrailed_tools(original_tools, flattened_tool_groups, guardrailed_tools)
|
||||
merge(original_tools, flattened_tool_groups, guardrailed_tools)
|
||||
)
|
||||
|
||||
def _extract_input_text_and_images(
|
||||
|
|
@ -839,6 +974,11 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
texts_to_check.append(summary)
|
||||
task_mappings.append((msg_idx, None))
|
||||
return
|
||||
if message.get("type") in TOOL_CALL_OUTPUT_ITEM_TYPES:
|
||||
for content_idx, text in tool_output_texts(message):
|
||||
texts_to_check.append(text)
|
||||
task_mappings.append((msg_idx, content_idx))
|
||||
return
|
||||
content: Final = message.get("content", None)
|
||||
if content is None:
|
||||
return
|
||||
|
|
@ -890,18 +1030,20 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
)
|
||||
continue
|
||||
|
||||
content = messages[msg_idx].get("content", None)
|
||||
field = _scanned_text_field(messages[msg_idx])
|
||||
content = messages[msg_idx].get(field, None)
|
||||
if content is None:
|
||||
continue
|
||||
|
||||
if isinstance(content, str) and content_idx_optional is None:
|
||||
# Replace string content with guardrail response
|
||||
messages[msg_idx]["content"] = guardrail_response
|
||||
if content_idx_optional is None:
|
||||
rewritten = _rewritten_whole_text(messages[msg_idx], field, guardrail_response)
|
||||
if rewritten is not None:
|
||||
messages[msg_idx][field] = rewritten
|
||||
|
||||
elif isinstance(content, list) and content_idx_optional is not None:
|
||||
elif isinstance(content, list):
|
||||
# Replace specific text item in list content
|
||||
if isinstance(messages[msg_idx]["content"][content_idx_optional], dict):
|
||||
messages[msg_idx]["content"][content_idx_optional]["text"] = guardrail_response
|
||||
if isinstance(messages[msg_idx][field][content_idx_optional], dict):
|
||||
messages[msg_idx][field][content_idx_optional]["text"] = guardrail_response
|
||||
|
||||
async def process_output_response(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ from typing import Final, TypeAlias
|
|||
from pydantic import BaseModel, TypeAdapter, ValidationError
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.llms.base_llm.guardrail_translation.utils import merge_returned_tools_into_request_tools, openai_tool_name
|
||||
from litellm.responses.litellm_completion_transformation.custom_tools import custom_tool_grammar_suffix
|
||||
from litellm.responses.litellm_completion_transformation.transformation import (
|
||||
NAMESPACE_DESCRIPTION_SEPARATOR,
|
||||
|
|
@ -183,3 +184,18 @@ def merge_guardrailed_tools(
|
|||
tuple(tool for key, tool in zip(guardrailed_keys, guardrailed) if key not in owned_keys)
|
||||
)
|
||||
return tuple(chain(merged_originals, appended))
|
||||
|
||||
|
||||
def append_returned_tools(
|
||||
original_tools: Sequence[Tool],
|
||||
flattened_groups: Sequence[Sequence[Tool]],
|
||||
returned_tools: Iterable[object],
|
||||
) -> tuple[Tool, ...]:
|
||||
request_chat_tools: Final = tuple(chain.from_iterable(flattened_groups))
|
||||
additions: Final = merge_returned_tools_into_request_tools(
|
||||
request_tools=request_chat_tools, returned_tools=_validated_tools(returned_tools), tool_name=openai_tool_name
|
||||
)[len(request_chat_tools) :]
|
||||
return (
|
||||
*original_tools,
|
||||
*LiteLLMCompletionResponsesConfig.transform_chat_completion_tool_params_to_responses_api_tools(additions),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -66,6 +66,29 @@ _PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS: Final = frozenset({LlmProviders.AZURE,
|
|||
_PROVIDERS_REPLAYING_ONLY_THEIR_OWN_REASONING: Final = _PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS
|
||||
_ROLES_READING_PROMPT_CACHE_BREAKPOINTS: Final = frozenset({"system", "developer", "user"})
|
||||
_PART_TYPES_READING_PROMPT_CACHE_BREAKPOINTS: Final = frozenset({"input_text", "input_image", "input_file"})
|
||||
_COMPACT_TRACKING_PARAMS: Final = frozenset({"metadata", "user"})
|
||||
_COMPACT_REJECTED_PARAMS: Final = frozenset(
|
||||
{
|
||||
"background",
|
||||
"context_management",
|
||||
"max_output_tokens",
|
||||
"max_tool_calls",
|
||||
"moderation",
|
||||
"partial_images",
|
||||
"prompt",
|
||||
"safety_identifier",
|
||||
"store",
|
||||
"stream_options",
|
||||
"temperature",
|
||||
"tool_choice",
|
||||
"top_logprobs",
|
||||
"top_p",
|
||||
"truncation",
|
||||
}
|
||||
)
|
||||
_PROVIDERS_WITH_STRICT_COMPACT_PARAMS: Final = frozenset(
|
||||
{LlmProviders.OPENAI, LlmProviders.AZURE, LlmProviders.AZURE_AI}
|
||||
)
|
||||
_INPUT_ITEM: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
|
||||
_CONTENT_PARTS: Final[TypeAdapter[Sequence[Mapping[str, object]]]] = TypeAdapter(Sequence[Mapping[str, object]])
|
||||
|
||||
|
|
@ -1182,6 +1205,17 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
|||
#########################################################
|
||||
########## COMPACT RESPONSE API TRANSFORMATION ##########
|
||||
#########################################################
|
||||
def compact_url(self, api_base: str) -> str:
|
||||
parsed_url: Final = httpx.URL(api_base)
|
||||
return str(parsed_url.copy_with(path=parsed_url.path.rstrip("/") + "/compact"))
|
||||
|
||||
def compact_dropped_params(self, litellm_params: GenericLiteLLMParams) -> frozenset[str]:
|
||||
if self.custom_llm_provider not in _PROVIDERS_WITH_STRICT_COMPACT_PARAMS:
|
||||
return frozenset()
|
||||
if _drops_params(litellm_params):
|
||||
return _COMPACT_TRACKING_PARAMS | _COMPACT_REJECTED_PARAMS
|
||||
return _COMPACT_TRACKING_PARAMS
|
||||
|
||||
def transform_compact_response_api_request(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -1197,10 +1231,8 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
|||
OpenAI API expects the following request
|
||||
- POST /v1/responses/compact
|
||||
"""
|
||||
# Preserve query params (e.g., api-version) while appending /compact.
|
||||
parsed_url: Final = httpx.URL(api_base)
|
||||
compact_path: Final = parsed_url.path.rstrip("/") + "/compact"
|
||||
url: Final = str(parsed_url.copy_with(path=compact_path))
|
||||
url: Final = self.compact_url(api_base)
|
||||
dropped_params: Final = self.compact_dropped_params(litellm_params)
|
||||
|
||||
request_params: Final = cast( # cast-ok: the base signature leaves the params bag untyped
|
||||
"Mapping[str, object]", response_api_optional_request_params
|
||||
|
|
@ -1214,14 +1246,11 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
|||
)
|
||||
if sanitized_tools is not None:
|
||||
response_api_optional_request_params["tools"] = sanitized_tools
|
||||
data: Final = cast( # cast-ok: typed dict spread is inferred as Unknown by basedpyright
|
||||
dict[str, object],
|
||||
{
|
||||
"model": model,
|
||||
"input": replay_safe_input,
|
||||
**response_api_optional_request_params,
|
||||
},
|
||||
)
|
||||
data: Final = {
|
||||
"model": model,
|
||||
"input": replay_safe_input,
|
||||
**{key: value for key, value in request_params.items() if key not in dropped_params},
|
||||
}
|
||||
|
||||
return url, data
|
||||
|
||||
|
|
|
|||
|
|
@ -110,6 +110,15 @@ def create_config_class(provider: SimpleProviderConfig):
|
|||
return super()._transform_messages(messages=transformed, model=model, is_async=True)
|
||||
return super()._transform_messages(messages=transformed, model=model, is_async=False)
|
||||
|
||||
def _should_preserve_cache_control_for_endpoint(
|
||||
self,
|
||||
custom_llm_provider: str | None,
|
||||
api_base: str | None,
|
||||
) -> bool:
|
||||
return provider.special_handling.get(
|
||||
"preserve_cache_control", False
|
||||
) or super()._should_preserve_cache_control_for_endpoint(custom_llm_provider, api_base)
|
||||
|
||||
def _get_openai_compatible_provider_info(
|
||||
self, api_base: str | None, api_key: str | None
|
||||
) -> tuple[str | None, str | None]:
|
||||
|
|
@ -243,7 +252,7 @@ def create_config_class(provider: SimpleProviderConfig):
|
|||
return JSONProviderConfig
|
||||
|
||||
|
||||
_responses_config_cache: Final[dict] = {}
|
||||
_responses_config_cache: Final[dict[str, "type[OpenAILikeResponsesConfig]"]] = {}
|
||||
|
||||
|
||||
def create_responses_config_class(provider: SimpleProviderConfig) -> "type[OpenAILikeResponsesConfig]":
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ JSON-based provider configuration loader for OpenAI-compatible providers.
|
|||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
|
|
@ -12,7 +13,7 @@ from litellm._logging import verbose_logger
|
|||
from litellm.types.utils import ProviderSpecificModelInfo
|
||||
|
||||
_PROVIDER_INFO_ADAPTER: Final = TypeAdapter(ProviderSpecificModelInfo)
|
||||
_SPECIAL_HANDLING_ADAPTER: Final = TypeAdapter(dict[str, bool])
|
||||
_SPECIAL_HANDLING_ADAPTER: Final = TypeAdapter(Mapping[str, bool])
|
||||
|
||||
|
||||
class SimpleProviderConfig:
|
||||
|
|
@ -21,12 +22,14 @@ class SimpleProviderConfig:
|
|||
def __init__(self, slug: str, data: dict):
|
||||
self.slug = slug
|
||||
self.base_url = data["base_url"]
|
||||
self.api_key_env = data["api_key_env"]
|
||||
self.api_base_env = data.get("api_base_env")
|
||||
self.api_key_env: str = data["api_key_env"]
|
||||
self.api_base_env: str | None = data.get("api_base_env")
|
||||
self.base_class = data.get("base_class", "openai_gpt")
|
||||
self.param_mappings = data.get("param_mappings", {})
|
||||
self.constraints = data.get("constraints", {})
|
||||
self.special_handling: Final = _SPECIAL_HANDLING_ADAPTER.validate_python(data.get("special_handling", {}))
|
||||
self.special_handling: Mapping[str, bool] = _SPECIAL_HANDLING_ADAPTER.validate_python(
|
||||
data.get("special_handling", {})
|
||||
)
|
||||
self.supported_endpoints = data.get("supported_endpoints", [])
|
||||
self.provider_info: ProviderSpecificModelInfo = (
|
||||
_PROVIDER_INFO_ADAPTER.validate_python(data["provider_info"]) if "provider_info" in data else {}
|
||||
|
|
|
|||
|
|
@ -270,6 +270,22 @@
|
|||
"/v1/responses"
|
||||
]
|
||||
},
|
||||
"streamlake": {
|
||||
"base_url": "https://vanchin.streamlake.ai/api/gateway/v1/endpoints",
|
||||
"api_key_env": "STREAMLAKE_API_KEY",
|
||||
"api_base_env": "STREAMLAKE_API_BASE",
|
||||
"param_mappings": {
|
||||
"max_tokens": "max_completion_tokens"
|
||||
},
|
||||
"provider_info": {
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true
|
||||
},
|
||||
"special_handling": {
|
||||
"preserve_cache_control": true
|
||||
},
|
||||
"supported_endpoints": ["/v1/chat/completions"]
|
||||
},
|
||||
"vispark": {
|
||||
"base_url": "https://api.lab.vispark.in/v1",
|
||||
"api_key_env": "VISPARK_LAB_API_KEY",
|
||||
|
|
|
|||
|
|
@ -41939,6 +41939,120 @@
|
|||
"supports_system_messages": true,
|
||||
"supports_native_structured_output": true
|
||||
},
|
||||
"bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
"input_cost_per_token": 4.12e-08,
|
||||
"input_cost_per_token_batches": 2.06e-08,
|
||||
"input_cost_per_token_flex": 2.06e-08,
|
||||
"input_cost_per_token_priority": 7.21e-08,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 8192,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4.12e-08,
|
||||
"output_cost_per_token_batches": 2.06e-08,
|
||||
"output_cost_per_token_flex": 2.06e-08,
|
||||
"output_cost_per_token_priority": 7.21e-08,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json",
|
||||
"supports_audio_input": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_native_structured_output": true
|
||||
},
|
||||
"bedrock/ap-southeast-2/mistral.voxtral-small-24b-2507": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
"input_cost_per_token": 1.03e-07,
|
||||
"input_cost_per_token_batches": 5.15e-08,
|
||||
"input_cost_per_token_flex": 5.15e-08,
|
||||
"input_cost_per_token_priority": 1.8025e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 8192,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.09e-07,
|
||||
"output_cost_per_token_batches": 1.545e-07,
|
||||
"output_cost_per_token_flex": 1.545e-07,
|
||||
"output_cost_per_token_priority": 5.4075e-07,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json",
|
||||
"supports_audio_input": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_native_structured_output": true
|
||||
},
|
||||
"bedrock/ap-northeast-1/mistral.voxtral-mini-3b-2507": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
"input_cost_per_token": 5e-08,
|
||||
"input_cost_per_token_batches": 3e-08,
|
||||
"input_cost_per_token_flex": 3e-08,
|
||||
"input_cost_per_token_priority": 9e-08,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 8192,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5e-08,
|
||||
"output_cost_per_token_batches": 3e-08,
|
||||
"output_cost_per_token_flex": 3e-08,
|
||||
"output_cost_per_token_priority": 9e-08,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-northeast-1/index.json",
|
||||
"supports_audio_input": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_native_structured_output": true
|
||||
},
|
||||
"bedrock/ap-northeast-1/mistral.voxtral-small-24b-2507": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"input_cost_per_token_batches": 6e-08,
|
||||
"input_cost_per_token_flex": 6e-08,
|
||||
"input_cost_per_token_priority": 2.1e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 8192,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.6e-07,
|
||||
"output_cost_per_token_batches": 1.8e-07,
|
||||
"output_cost_per_token_flex": 1.8e-07,
|
||||
"output_cost_per_token_priority": 6.3e-07,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-northeast-1/index.json",
|
||||
"supports_audio_input": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_native_structured_output": true
|
||||
},
|
||||
"bedrock/eu-west-2/mistral.voxtral-mini-3b-2507": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
"input_cost_per_token": 6e-08,
|
||||
"input_cost_per_token_batches": 3e-08,
|
||||
"input_cost_per_token_flex": 3e-08,
|
||||
"input_cost_per_token_priority": 1.1e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 8192,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 6e-08,
|
||||
"output_cost_per_token_batches": 3e-08,
|
||||
"output_cost_per_token_flex": 3e-08,
|
||||
"output_cost_per_token_priority": 1.1e-07,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/eu-west-2/index.json",
|
||||
"supports_audio_input": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_native_structured_output": true
|
||||
},
|
||||
"bedrock/eu-west-2/mistral.voxtral-small-24b-2507": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
"input_cost_per_token": 1.6e-07,
|
||||
"input_cost_per_token_batches": 8e-08,
|
||||
"input_cost_per_token_flex": 8e-08,
|
||||
"input_cost_per_token_priority": 2.8e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 8192,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4.7e-07,
|
||||
"output_cost_per_token_batches": 2.3e-07,
|
||||
"output_cost_per_token_flex": 2.3e-07,
|
||||
"output_cost_per_token_priority": 8.2e-07,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/eu-west-2/index.json",
|
||||
"supports_audio_input": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_native_structured_output": true
|
||||
},
|
||||
"mistral/codestral-2508": {
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"input_cost_per_token": 3e-07,
|
||||
|
|
@ -61374,6 +61488,359 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"streamlake/deepseek-v4-flash-0731": {
|
||||
"cache_read_input_token_cost": 1.4e-08,
|
||||
"input_cost_per_token": 4.4e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 384000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.32e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/DeepSeek-V4-Pro-0813": {
|
||||
"cache_read_input_token_cost": 4.4e-08,
|
||||
"input_cost_per_token": 1.32e-06,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 1048566,
|
||||
"max_output_tokens": 384000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.96e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/DeepSeek-V3": {
|
||||
"cache_read_input_token_cost": 1.14e-07,
|
||||
"input_cost_per_token": 2.86e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 56000,
|
||||
"max_output_tokens": 8000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.143e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": false,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/GLM-5.3": {
|
||||
"cache_read_input_token_cost": 2.6e-07,
|
||||
"input_cost_per_token": 1.4e-06,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4.4e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/GLM-5.3-Flash": {
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5e-07,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": true,
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/Kimi-K2-Instruct-0905": {
|
||||
"cache_read_input_token_cost": 1.43e-07,
|
||||
"input_cost_per_token": 5.71e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 224000,
|
||||
"max_output_tokens": 32000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.286e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": false,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/deepseek-r1": {
|
||||
"cache_read_input_token_cost": 2.29e-07,
|
||||
"input_cost_per_token": 5.71e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 98304,
|
||||
"max_output_tokens": 16000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.286e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/deepseek-v3.1": {
|
||||
"cache_read_input_token_cost": 2.29e-07,
|
||||
"input_cost_per_token": 5.71e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 98304,
|
||||
"max_output_tokens": 32000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.714e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/deepseek-v4-flash": {
|
||||
"cache_read_input_token_cost": 2.8e-08,
|
||||
"input_cost_per_token": 1.4e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 384000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.8e-07,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/deepseek-v4-pro": {
|
||||
"cache_read_input_token_cost": 1.45e-07,
|
||||
"input_cost_per_token": 1.74e-06,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 384000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.48e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/deepseek-v4.1-flash-1017": {
|
||||
"cache_read_input_token_cost": 6e-09,
|
||||
"input_cost_per_token": 3e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 384000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.2e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_vision": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/deepseek-v4.1-flash-260910": {
|
||||
"cache_read_input_token_cost": 6e-09,
|
||||
"input_cost_per_token": 3e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 1048568,
|
||||
"max_output_tokens": 384000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.2e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_vision": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/glm-5": {
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"input_cost_per_token": 1e-06,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 202752,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.2e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/glm-5.1": {
|
||||
"cache_read_input_token_cost": 2.6e-07,
|
||||
"input_cost_per_token": 1.4e-06,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 202752,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4.4e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/glm-5.2": {
|
||||
"cache_read_input_token_cost": 2.6e-07,
|
||||
"input_cost_per_token": 1.4e-06,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 1048566,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4.4e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/kimi-k2.6": {
|
||||
"cache_read_input_token_cost": 1.6e-07,
|
||||
"input_cost_per_token": 9.5e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 256000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_vision": true,
|
||||
"supports_reasoning": false,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/kimi-k2.7-code": {
|
||||
"cache_read_input_token_cost": 1.9e-07,
|
||||
"input_cost_per_token": 9.5e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 32000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_vision": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/mimo-v2.6-flash": {
|
||||
"cache_read_input_token_cost": 2.8e-09,
|
||||
"input_cost_per_token": 1.4e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.8e-07,
|
||||
"supports_vision": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/mimo-v2.6-pro": {
|
||||
"cache_read_input_token_cost": 3.6e-09,
|
||||
"input_cost_per_token": 4.35e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 8.7e-07,
|
||||
"supports_vision": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/minimax-m2.5": {
|
||||
"cache_creation_input_token_cost": 3.75e-07,
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"input_cost_per_token": 3e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 196608,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.2e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/minimax-m3": {
|
||||
"cache_read_input_token_cost": 1.2e-07,
|
||||
"cache_read_input_token_cost_above_512k_tokens": 2.4e-07,
|
||||
"input_cost_per_token": 6e-07,
|
||||
"input_cost_per_token_above_512k_tokens": 1.2e-06,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 512000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.4e-06,
|
||||
"output_cost_per_token_above_512k_tokens": 4.8e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_vision": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/qwen3.5-397b-a17b": {
|
||||
"cache_read_input_token_cost": 1.2e-07,
|
||||
"input_cost_per_token": 6e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 258048,
|
||||
"max_output_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.6e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_vision": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/qwen3.6-27b": {
|
||||
"input_cost_per_token": 6e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 229376,
|
||||
"max_output_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.6e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_vision": true,
|
||||
"supports_reasoning": false,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/qwen3.6-35b-a3b": {
|
||||
"input_cost_per_token": 3.75e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 229376,
|
||||
"max_output_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.25e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_vision": true,
|
||||
"supports_reasoning": false,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/qwen3.8-2.4t-a95b": {
|
||||
"cache_read_input_token_cost": 2.5e-07,
|
||||
"input_cost_per_token": 2e-06,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 983616,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 6e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_vision": true,
|
||||
"cache_creation_input_token_cost": 2.5e-06,
|
||||
"explicit_cache_read_input_token_cost": 1.7e-07,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/qwen3.8-27b": {
|
||||
"cache_read_input_token_cost": 1e-07,
|
||||
"input_cost_per_token": 5e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 983616,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_vision": true,
|
||||
"cache_creation_input_token_cost": 6.25e-07,
|
||||
"explicit_cache_read_input_token_cost": 5e-08,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"deepseek-v4-flash": {
|
||||
"cache_creation_input_token_cost": 0.0,
|
||||
"cache_read_input_token_cost": 6e-09,
|
||||
|
|
@ -62689,6 +63156,14 @@
|
|||
"litellm_provider": "anthropic"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "typesafe-jev-ids",
|
||||
"pattern": "^jev-(?:latest|preview|1\\.13\\.0|1\\.14\\.0)$",
|
||||
"description": "A bare Jev id that has a typesafe/ price entry (jev-latest, jev-preview, jev-1.13.0, jev-1.14.0), the name the /typesafe pass-through forwards to TypeSafe. Listed by name, matched case-insensitively like every rule here, so foo/jev-latest or a model group named like a Jev id TypeSafe does not serve, such as jev-1.0 or jev-internal-router, never resolves to typesafe. A new Jev version goes here together with its price entry. Provider inference feeds the proxy's provider/* wildcard access checks, so this rule is what lets typesafe/* on a key, team or project allowlist cover a jev-latest request.",
|
||||
"model_info": {
|
||||
"litellm_provider": "typesafe"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "claude-family-baseline",
|
||||
"pattern": "claude-[a-z]+-\\d+(?:[-.]\\d+)?(?:-\\d{8})?",
|
||||
|
|
|
|||
|
|
@ -3785,6 +3785,32 @@
|
|||
"generateContent": true
|
||||
}
|
||||
},
|
||||
"streamlake": {
|
||||
"display_name": "StreamLake (`streamlake`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/openai_compatible",
|
||||
"endpoints": {
|
||||
"chat_completions": true,
|
||||
"responses": true,
|
||||
"text_completion": true,
|
||||
"embeddings": false,
|
||||
"image_generations": false,
|
||||
"image_edits": false,
|
||||
"audio_transcriptions": false,
|
||||
"audio_speech": false,
|
||||
"rerank": false,
|
||||
"ocr": false,
|
||||
"search": false,
|
||||
"moderations": false,
|
||||
"realtime": false,
|
||||
"video_generations": false,
|
||||
"batches": false,
|
||||
"messages": true,
|
||||
"interactions": true,
|
||||
"systemone": false,
|
||||
"a2a": true,
|
||||
"generateContent": true
|
||||
}
|
||||
},
|
||||
"tencent": {
|
||||
"display_name": "Tencent TokenHub (`tencent`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/tencent",
|
||||
|
|
|
|||
|
|
@ -8075,7 +8075,6 @@ async def vector_store_access_check(
|
|||
rag_ids: Final = (rag_vector_store_id,) if rag_vector_store_id is not None else ()
|
||||
vector_store_ids_to_run: Final = tuple(dict.fromkeys((*registry_ids, *rag_ids)))
|
||||
if not vector_store_ids_to_run:
|
||||
verbose_proxy_logger.debug("Vector store to run not found, skipping vector store access check")
|
||||
return True
|
||||
|
||||
#########################################################
|
||||
|
|
|
|||
|
|
@ -2581,7 +2581,6 @@ class ProxyBaseLLMRequestProcessing:
|
|||
# have mutated `self.data` in place, and the audit-trail snapshot taken in
|
||||
# add_litellm_data_to_request predates that mutation.
|
||||
refresh_proxy_server_request_body_snapshot(self.data, guardrails_applied=True)
|
||||
verbose_proxy_logger.debug("receiving data: %s", self.data)
|
||||
|
||||
if "messages" in self.data and self.data["messages"]:
|
||||
logging_obj.update_messages(self.data["messages"])
|
||||
|
|
|
|||
|
|
@ -845,5 +845,3 @@ def _add_vector_store_id_from_path(request_data: dict, request: Request) -> None
|
|||
"populate_request_with_path_params: Updated request_data with vector_store_ids=%s",
|
||||
request_data.get("vector_store_ids"),
|
||||
)
|
||||
else:
|
||||
verbose_proxy_logger.debug("populate_request_with_path_params: No vector_store_id present in path=%s", path)
|
||||
|
|
|
|||
|
|
@ -62,6 +62,20 @@ def is_non_conversational_call_type(call_type: str) -> bool:
|
|||
return call_type in NON_CONVERSATIONAL_CALL_TYPES
|
||||
|
||||
|
||||
_PASSTHROUGH_EMBEDDINGS_ENDPOINT: Final = re.compile(r"/?(?:[\w-]+/)*embeddings/?")
|
||||
|
||||
|
||||
def is_non_conversational_request(call_type: str, data: Mapping[str, object]) -> bool:
|
||||
if is_non_conversational_call_type(call_type):
|
||||
return True
|
||||
endpoint: Final = data.get("endpoint")
|
||||
return (
|
||||
call_type == "allm_passthrough_route"
|
||||
and isinstance(endpoint, str)
|
||||
and _PASSTHROUGH_EMBEDDINGS_ENDPOINT.fullmatch(endpoint) is not None
|
||||
)
|
||||
|
||||
|
||||
TEXT_PART_TYPES: Final[frozenset[str]] = frozenset(
|
||||
{"text", "input_text", "output_text", "summary_text", "reasoning_text"}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
|
|||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, bedrock_bearer_token, run_aws_signing
|
||||
from litellm.llms.bedrock.guardrail_attachments import find_request_attachments
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
AsyncHTTPHandler,
|
||||
get_async_httpx_client,
|
||||
httpxSpecialProvider,
|
||||
)
|
||||
|
|
@ -286,9 +287,12 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
|
|||
streaming_sampling_rate: int | None = None,
|
||||
streaming_end_of_stream_only: bool | None = None,
|
||||
streaming_buffer_release_on_scan: bool | None = None,
|
||||
async_handler: AsyncHTTPHandler | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
self.async_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.GuardrailCallback)
|
||||
self.async_handler = async_handler or get_async_httpx_client(
|
||||
llm_provider=httpxSpecialProvider.GuardrailCallback
|
||||
)
|
||||
self._set_streaming_params(
|
||||
BedrockGuardrailStreamingParams.from_extras(
|
||||
MappingProxyType(
|
||||
|
|
|
|||
|
|
@ -6,20 +6,19 @@ model_list:
|
|||
model: gpt-4
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
|
||||
litellm_settings:
|
||||
guardrails:
|
||||
- guardrail_name: "my-generic-guardrail"
|
||||
litellm_params:
|
||||
guardrail: generic_guardrail_api
|
||||
mode: pre_call # Options: pre_call, post_call, during_call, [pre_call, post_call]
|
||||
api_key: os.environ/GENERIC_GUARDRAIL_API_KEY # Optional if using Bearer auth
|
||||
api_base: http://localhost:8080 # Required. Endpoint /beta/litellm_basic_guardrail_api is automatically appended
|
||||
unreachable_fallback: fail_closed # Options: fail_closed (default, raise), fail_open (proceed if endpoint unreachable or upstream returns 502/503/504)
|
||||
default_on: false # Set to true to apply to all requests by default
|
||||
additional_provider_specific_params:
|
||||
# Any additional parameters your guardrail API needs
|
||||
api_version: "v1"
|
||||
custom_param: "value"
|
||||
guardrails:
|
||||
- guardrail_name: "my-generic-guardrail"
|
||||
litellm_params:
|
||||
guardrail: generic_guardrail_api
|
||||
mode: pre_call # Options: pre_call, post_call, during_call, [pre_call, post_call]
|
||||
api_key: os.environ/GENERIC_GUARDRAIL_API_KEY # Optional if using Bearer auth
|
||||
api_base: http://localhost:8080 # Required. Endpoint /beta/litellm_basic_guardrail_api is automatically appended
|
||||
unreachable_fallback: fail_closed # Options: fail_closed (default, raise), fail_open (proceed if endpoint unreachable or upstream returns 502/503/504)
|
||||
default_on: false # Set to true to apply to all requests by default
|
||||
additional_provider_specific_params:
|
||||
# Any additional parameters your guardrail API needs
|
||||
api_version: "v1"
|
||||
custom_param: "value"
|
||||
|
||||
# Usage examples:
|
||||
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ from litellm.integrations.custom_guardrail import (
|
|||
from litellm.llms.base_llm.guardrail_translation.utils import (
|
||||
effective_scan_only_tool_results_for_guardrail,
|
||||
effective_skip_system_message_for_guardrail,
|
||||
effective_skip_tool_message_for_guardrail,
|
||||
role_out_of_guardrail_scope,
|
||||
)
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
|
|
@ -37,12 +38,16 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
get_async_httpx_client,
|
||||
httpxSpecialProvider,
|
||||
)
|
||||
from litellm.llms.openai.responses.guardrail_translation.handler import scannable_instructions
|
||||
from litellm.llms.openai.responses.guardrail_translation.handler import (
|
||||
scannable_instructions,
|
||||
tool_output_texts,
|
||||
)
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.common_utils.callback_utils import (
|
||||
add_guardrail_scan_id,
|
||||
add_guardrail_to_applied_guardrails_header,
|
||||
)
|
||||
from litellm.responses.litellm_completion_transformation.transformation import TOOL_CALL_OUTPUT_ITEM_TYPES
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
|
@ -112,12 +117,16 @@ class _ResponsesInputItem(LiteLLMBaseModel):
|
|||
type: str | None = None
|
||||
role: str | None = None
|
||||
content: str | tuple[_ResponsesContentPart, ...] | None = None
|
||||
output: object = None
|
||||
call_id: object = None
|
||||
|
||||
def text_count(self, *, skip_system: bool) -> int:
|
||||
if role_out_of_guardrail_scope(
|
||||
(self.role or "").lower(), skip_system_message=skip_system, skip_tool_message=False
|
||||
):
|
||||
def text_count(self, *, skip_system: bool, skip_tool: bool) -> int:
|
||||
is_tool_output: Final = self.type in TOOL_CALL_OUTPUT_ITEM_TYPES
|
||||
role: Final = "tool" if is_tool_output else (self.role or "").lower()
|
||||
if role_out_of_guardrail_scope(role, skip_system_message=skip_system, skip_tool_message=skip_tool):
|
||||
return 0
|
||||
if is_tool_output:
|
||||
return len(tool_output_texts(self))
|
||||
if isinstance(self.content, str):
|
||||
return 1
|
||||
if self.content is None:
|
||||
|
|
@ -1692,13 +1701,14 @@ class PanwPrismaAirsHandler(CustomGuardrail):
|
|||
request_data: Mapping[str, object],
|
||||
*,
|
||||
skip_system: bool,
|
||||
skip_tool: bool,
|
||||
) -> frozenset[int] | None:
|
||||
"""Return the ``texts`` indices flattened from Responses ``reasoning`` input items.
|
||||
|
||||
The Responses translation handler gives those model-authored items the default
|
||||
``user`` role, so the latest-turn selection must not mistake one for a human turn.
|
||||
Empty for requests without a Responses ``input`` item list; None when the raw items
|
||||
(after the leading ``instructions`` text, both minus whatever ``skip_system`` drops)
|
||||
(after the leading ``instructions`` text, both minus whatever ``skip_system`` and ``skip_tool`` drop)
|
||||
do not account for every entry of ``texts``.
|
||||
"""
|
||||
try:
|
||||
|
|
@ -1708,7 +1718,7 @@ class PanwPrismaAirsHandler(CustomGuardrail):
|
|||
if not isinstance(raw_input, tuple):
|
||||
return frozenset()
|
||||
offset: Final = 0 if scannable_instructions(request_data, skip_system=skip_system) is None else 1
|
||||
counts: Final = tuple(item.text_count(skip_system=skip_system) for item in raw_input)
|
||||
counts: Final = tuple(item.text_count(skip_system=skip_system, skip_tool=skip_tool) for item in raw_input)
|
||||
if offset + sum(counts) != len(texts):
|
||||
return None
|
||||
starts: Final = itertools.accumulate(counts, initial=offset)
|
||||
|
|
@ -1738,7 +1748,10 @@ class PanwPrismaAirsHandler(CustomGuardrail):
|
|||
if sources is None:
|
||||
return None
|
||||
reasoning: Final = self._reasoning_item_text_indices(
|
||||
texts, request_data, skip_system=effective_skip_system_message_for_guardrail(self)
|
||||
texts,
|
||||
request_data,
|
||||
skip_system=effective_skip_system_message_for_guardrail(self),
|
||||
skip_tool=effective_skip_tool_message_for_guardrail(self),
|
||||
)
|
||||
if reasoning is None:
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -3024,13 +3024,7 @@ async def add_litellm_data_to_request(
|
|||
getattr(request.state, "litellm_roi_estimator", False) is True
|
||||
)
|
||||
|
||||
verbose_proxy_logger.debug(
|
||||
"[PROXY] returned data from litellm_pre_call_utils: %s",
|
||||
data,
|
||||
)
|
||||
|
||||
# Team/Project credential overrides from model_config
|
||||
# Placed after the debug log to avoid leaking credential secrets in logs
|
||||
_apply_credential_overrides_from_model_config(
|
||||
data=data,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
|
|
|
|||
|
|
@ -50,6 +50,7 @@ from litellm.constants import (
|
|||
from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
|
||||
from litellm.integrations.custom_guardrail import guardrail_request_data_with_streaming, is_guardrail_intervention
|
||||
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
|
||||
from litellm.litellm_core_utils.fallback_generalizations import match_routing_generalization
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, merge_anthropic_beta_headers
|
||||
from litellm.llms.azure.passthrough.transformation import (
|
||||
foreign_azure_deployment,
|
||||
|
|
@ -698,6 +699,20 @@ async def typesafe_proxy_route(
|
|||
return await endpoint_func(request, fastapi_response, user_api_key_dict)
|
||||
|
||||
|
||||
_COST_MAP_ENTRIES: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def typesafe_allowlist_model_names() -> tuple[str, ...]:
|
||||
if not passthrough_endpoint_router.has_credentials(custom_llm_provider="typesafe"):
|
||||
return ()
|
||||
cost_map: Final = _COST_MAP_ENTRIES.validate_python(litellm.model_cost) # pyright: ignore[reportUnknownMemberType] # legacy registry is validated at this boundary
|
||||
priced_names: Final = (name.removeprefix("typesafe/") for name in cost_map if name.startswith("typesafe/"))
|
||||
jev_models_typesafe_serves: Final = sorted(
|
||||
name for name in priced_names if match_routing_generalization(name) == "typesafe"
|
||||
)
|
||||
return ("typesafe/*", *jev_models_typesafe_serves)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/laya/v1/systemone",
|
||||
tags=["Laya Pass-through", "pass-through"],
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import json
|
||||
import os
|
||||
from collections.abc import Callable
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
|
|
@ -88,6 +89,14 @@ class PassthroughEndpointRouter:
|
|||
)
|
||||
return _resolve_oidc_reference(get_secret_str(_env_variable_name))
|
||||
|
||||
def has_credentials(self, custom_llm_provider: str) -> bool:
|
||||
if self._get_deployment_api_key(custom_llm_provider=custom_llm_provider, region_name=None) is not None:
|
||||
return True
|
||||
env_variable_name: Final = self._get_default_env_variable_name_passthrough_endpoint(
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
return bool(os.environ.get(env_variable_name))
|
||||
|
||||
def _get_deployment_api_key(
|
||||
self,
|
||||
custom_llm_provider: str,
|
||||
|
|
|
|||
|
|
@ -12390,6 +12390,12 @@ async def model_list(
|
|||
|
||||
hidden_names: Final = blocked_names | unhealthy_names
|
||||
|
||||
from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import typesafe_allowlist_model_names
|
||||
|
||||
asks_for_allowlist_names: Final = (
|
||||
include_model_access_groups is True and return_wildcard_routes is True and only_model_access_groups is not True
|
||||
)
|
||||
|
||||
# If scope=expand and user has admin privileges, return all proxy models
|
||||
if should_expand_scope:
|
||||
# Get all proxy models as if user is a proxy admin
|
||||
|
|
@ -12410,7 +12416,10 @@ async def model_list(
|
|||
all_models = get_complete_model_list(
|
||||
key_models=[],
|
||||
team_models=[],
|
||||
proxy_model_list=proxy_model_list,
|
||||
proxy_model_list=[
|
||||
*proxy_model_list,
|
||||
*(typesafe_allowlist_model_names() if asks_for_allowlist_names else ()),
|
||||
],
|
||||
user_model=None,
|
||||
infer_model_from_keys=False,
|
||||
return_wildcard_routes=include_wildcard_routes,
|
||||
|
|
@ -12471,7 +12480,7 @@ async def model_list(
|
|||
|
||||
# Otherwise, use the normal behavior (current implementation)
|
||||
# Get available models for the user
|
||||
all_models = await get_available_models_for_user(
|
||||
available_models: Final = await get_available_models_for_user(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
llm_router=llm_router,
|
||||
general_settings=general_settings,
|
||||
|
|
@ -12484,6 +12493,11 @@ async def model_list(
|
|||
return_wildcard_routes=include_wildcard_routes,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
)
|
||||
offers_typesafe_allowlist_names: Final = (
|
||||
asks_for_allowlist_names and team_id is None and user_api_key_has_admin_view(user_api_key_dict)
|
||||
)
|
||||
typesafe_names_to_offer: Final = typesafe_allowlist_model_names() if offers_typesafe_allowlist_names else ()
|
||||
all_models = [*available_models, *(name for name in typesafe_names_to_offer if name not in available_models)]
|
||||
|
||||
undiscoverable_names: Final = undiscoverable_model_names(
|
||||
all_models, llm_router, user_api_key_dict, team_id or user_api_key_dict.team_id
|
||||
|
|
@ -18437,7 +18451,6 @@ async def async_queue_request(
|
|||
"body": copy.copy(data), # use copy instead of deepcopy
|
||||
}
|
||||
|
||||
verbose_proxy_logger.debug("receiving data: %s", data)
|
||||
data["model"] = (
|
||||
general_settings.get("completion_model", None) # server default
|
||||
or user_model # model name passed via cli args
|
||||
|
|
|
|||
|
|
@ -3519,6 +3519,34 @@
|
|||
],
|
||||
"default_model_placeholder": "inferenceapis/deepseek-ai/DeepSeek-V4-Flash"
|
||||
},
|
||||
{
|
||||
"provider": "StreamLake",
|
||||
"provider_display_name": "StreamLake",
|
||||
"litellm_provider": "streamlake",
|
||||
"credential_fields": [
|
||||
{
|
||||
"key": "api_base",
|
||||
"label": "API Base",
|
||||
"placeholder": "https://vanchin.streamlake.ai/api/gateway/v1/endpoints",
|
||||
"tooltip": null,
|
||||
"required": false,
|
||||
"field_type": "text",
|
||||
"options": null,
|
||||
"default_value": null
|
||||
},
|
||||
{
|
||||
"key": "api_key",
|
||||
"label": "API Key",
|
||||
"placeholder": null,
|
||||
"tooltip": null,
|
||||
"required": true,
|
||||
"field_type": "password",
|
||||
"options": null,
|
||||
"default_value": null
|
||||
}
|
||||
],
|
||||
"default_model_placeholder": "streamlake/<model-or-endpoint>"
|
||||
},
|
||||
{
|
||||
"provider": "Snowflake",
|
||||
"provider_display_name": "Snowflake",
|
||||
|
|
|
|||
|
|
@ -321,7 +321,7 @@ def _responses_file_part(item: Mapping[str, object]) -> _ResponsesFilePart:
|
|||
)
|
||||
|
||||
|
||||
def _decode_responses_media(item: Mapping[str, object]) -> _ResponsesMediaPart | None:
|
||||
def decode_responses_media(item: Mapping[str, object]) -> _ResponsesMediaPart | None:
|
||||
kind: Final = item.get("type")
|
||||
if kind in ("input_image", "image_url", "computer_screenshot") and any(key in item for key in ("image_url", "url")):
|
||||
image: Final = _responses_image_part(item)
|
||||
|
|
@ -383,6 +383,52 @@ def _attribute_fields(value: object) -> dict[str, object]:
|
|||
return dict(cast("Iterable[tuple[str, object]]", value)) # cast-ok: dict() raises on non-pair values, as before
|
||||
|
||||
|
||||
def tool_output_to_chat_content(
|
||||
output: object,
|
||||
) -> str | list[ToolMessageContentPart]:
|
||||
if output is None:
|
||||
return ""
|
||||
if isinstance(output, str):
|
||||
return output
|
||||
|
||||
single_media: Final = decode_responses_media(output) if isinstance(output, Mapping) else None
|
||||
if single_media is not None:
|
||||
return [single_media]
|
||||
if isinstance(output, list):
|
||||
normalized_blocks: Final[list[ToolMessageContentPart]] = []
|
||||
text_acc: Final[list[str]] = []
|
||||
for part in output:
|
||||
if not isinstance(part, dict):
|
||||
continue
|
||||
part_type = part.get("type")
|
||||
if part_type in ("input_text", "output_text", "text"):
|
||||
txt = part.get("text")
|
||||
if isinstance(txt, str) and txt:
|
||||
text_acc.append(txt)
|
||||
normalized_blocks.append({"type": "text", "text": txt})
|
||||
elif (media := decode_responses_media(part)) is not None:
|
||||
normalized_blocks.append(media)
|
||||
|
||||
if any(b.get("type") != "text" for b in normalized_blocks):
|
||||
return normalized_blocks
|
||||
if text_acc:
|
||||
return "".join(text_acc)
|
||||
try:
|
||||
return json.dumps(output)
|
||||
except Exception:
|
||||
return str(output)
|
||||
|
||||
try:
|
||||
return json.dumps(output)
|
||||
except Exception:
|
||||
return str(output)
|
||||
|
||||
|
||||
TOOL_CALL_OUTPUT_ITEM_TYPES: Final = frozenset(
|
||||
{"function_call_output", "custom_tool_call_output", "computer_call_output", "tool_result"}
|
||||
)
|
||||
|
||||
|
||||
def _input_item_role(input_item: Mapping[str, object]) -> str:
|
||||
return cast(str, input_item.get("role") or "user") # cast-ok: client-supplied role forwarded verbatim, unvalidated
|
||||
|
||||
|
|
@ -2371,12 +2417,7 @@ class LiteLLMCompletionResponsesConfig:
|
|||
"""
|
||||
Check if the input item is a tool call output
|
||||
"""
|
||||
return input_item.get("type") in [
|
||||
"function_call_output",
|
||||
"custom_tool_call_output",
|
||||
"computer_call_output",
|
||||
"tool_result", # Anthropic/MCP format
|
||||
]
|
||||
return input_item.get("type") in TOOL_CALL_OUTPUT_ITEM_TYPES
|
||||
|
||||
@staticmethod
|
||||
def _is_input_item_function_call(input_item: Mapping[str, object]) -> bool:
|
||||
|
|
@ -2400,58 +2441,7 @@ class LiteLLMCompletionResponsesConfig:
|
|||
if not call_id:
|
||||
return []
|
||||
|
||||
def _normalize_function_call_output_to_tool_content(
|
||||
output: object,
|
||||
) -> str | list[ToolMessageContentPart]:
|
||||
"""
|
||||
Preserve structured media for downstream chat adapters and token estimation.
|
||||
Text-only outputs retain their existing string representation.
|
||||
"""
|
||||
if output is None:
|
||||
return ""
|
||||
if isinstance(output, str):
|
||||
return output
|
||||
|
||||
single_media: Final = _decode_responses_media(output) if isinstance(output, Mapping) else None
|
||||
if single_media is not None:
|
||||
return [single_media]
|
||||
# Some adapters represent tool output as a list of "input_*" parts
|
||||
if isinstance(output, list):
|
||||
normalized_blocks: Final[list[ToolMessageContentPart]] = []
|
||||
text_acc: Final[list[str]] = []
|
||||
for part in output:
|
||||
if not isinstance(part, dict):
|
||||
continue
|
||||
part_type = part.get("type")
|
||||
if part_type in ("input_text", "output_text", "text"):
|
||||
txt = part.get("text")
|
||||
if isinstance(txt, str) and txt:
|
||||
text_acc.append(txt)
|
||||
normalized_blocks.append({"type": "text", "text": txt})
|
||||
elif (media := _decode_responses_media(part)) is not None:
|
||||
normalized_blocks.append(media)
|
||||
|
||||
if any(b.get("type") != "text" for b in normalized_blocks):
|
||||
return normalized_blocks
|
||||
if text_acc:
|
||||
return "".join(text_acc)
|
||||
try:
|
||||
# last resort: keep something meaningful for providers that require a string
|
||||
import json as _json
|
||||
|
||||
return _json.dumps(output)
|
||||
except Exception:
|
||||
return str(output)
|
||||
|
||||
# Fallback for dict/number/etc.
|
||||
try:
|
||||
import json as _json
|
||||
|
||||
return _json.dumps(output)
|
||||
except Exception:
|
||||
return str(output)
|
||||
|
||||
tool_content: Final = _normalize_function_call_output_to_tool_content(tool_call_output.get("output"))
|
||||
tool_content: Final = tool_output_to_chat_content(tool_call_output.get("output"))
|
||||
output_cache_control: Final = _input_item_cache_control(tool_call_output)
|
||||
tool_output_message: Final = (
|
||||
ChatCompletionToolMessage(role="tool", content=tool_content, tool_call_id=str(call_id))
|
||||
|
|
@ -2606,7 +2596,7 @@ class LiteLLMCompletionResponsesConfig:
|
|||
content_list.append(item)
|
||||
elif isinstance(item, dict):
|
||||
if (
|
||||
media := _decode_responses_media(
|
||||
media := decode_responses_media(
|
||||
fields := _validate_responses_media(_STR_KEY_DICT_ADAPTER, item)
|
||||
)
|
||||
) is not None:
|
||||
|
|
|
|||
|
|
@ -39,7 +39,11 @@ from litellm.constants import (
|
|||
)
|
||||
from litellm.exceptions import MidStreamFallbackError, RateLimitError
|
||||
from litellm.litellm_core_utils.asyncify import run_async_function
|
||||
from litellm.litellm_core_utils.core_helpers import process_response_headers
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
process_response_headers,
|
||||
provider_reported_usage_cost,
|
||||
set_response_cost_in_hidden_params,
|
||||
)
|
||||
from litellm.litellm_core_utils.exception_mapping_utils import with_deployment_api_base
|
||||
from litellm.litellm_core_utils.internal_call_metadata import (
|
||||
is_background_response,
|
||||
|
|
@ -1825,20 +1829,32 @@ def _estimate_usage_safely(
|
|||
def stamp_responses_usage_cost(
|
||||
response_obj: ResponsesAPIResponse | None, logging_obj: LiteLLMLoggingObj | None
|
||||
) -> None:
|
||||
if response_obj is None or logging_obj is None:
|
||||
"""The terminal response's ``usage.cost`` is the bill litellm writes to the spend row. It is priced on a copy
|
||||
shaped the way the success handler shapes the spend row's result (the reported ``usage.cost`` in the
|
||||
response-cost header, the usage in chat shape), so the client and the row read the same number."""
|
||||
if response_obj is None:
|
||||
return
|
||||
usage_obj: Final[ResponseAPIUsage | None] = _usage_as_model(getattr(response_obj, "usage", None))
|
||||
if usage_obj is None:
|
||||
return
|
||||
response_obj.usage = usage_obj # rebind-ok: the stamped cost has to ride on the response the client receives
|
||||
if isinstance(getattr(usage_obj, "cost", None), (int, float)):
|
||||
response_obj.usage = usage_obj # rebind-ok: the typed usage has to ride on the response the client receives
|
||||
if logging_obj is None:
|
||||
return
|
||||
priced: Final = response_obj.model_copy(
|
||||
update={"usage": ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(usage_obj)}
|
||||
)
|
||||
priced.hidden_params = {}
|
||||
set_response_cost_in_hidden_params(
|
||||
priced, provider_reported_usage_cost(usage_obj, logging_obj.model_call_details.get("custom_llm_provider"))
|
||||
)
|
||||
try:
|
||||
cost: Final[float | None] = logging_obj.response_cost_calculator(result=response_obj)
|
||||
except Exception:
|
||||
bill: Final = logging_obj.response_cost_calculator(result=priced)
|
||||
except Exception: # noqa: BLE001 # a pricing failure leaves the client usage as the upstream sent it
|
||||
return
|
||||
if isinstance(cost, (int, float)) and cost > 0:
|
||||
setattr(usage_obj, "cost", cost)
|
||||
if isinstance(bill, bool) or not isinstance(bill, (int, float)):
|
||||
return
|
||||
if bill > 0 or isinstance(getattr(usage_obj, "cost", None), (int, float)):
|
||||
setattr(usage_obj, "cost", float(bill))
|
||||
|
||||
|
||||
def opening_item_payload(payload: Mapping[str, object]) -> Mapping[str, object]:
|
||||
|
|
|
|||
|
|
@ -766,6 +766,12 @@ class ResponsesAPIRequestUtils:
|
|||
model_id=model_id,
|
||||
response_id=decoded_response_id,
|
||||
)
|
||||
except ValueError:
|
||||
return DecodedResponseId(
|
||||
custom_llm_provider=None,
|
||||
model_id=None,
|
||||
response_id=response_id,
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.debug("Error decoding response_id '%s': %s", response_id, e)
|
||||
return DecodedResponseId(
|
||||
|
|
|
|||
|
|
@ -4921,6 +4921,7 @@ class Router:
|
|||
deployment=deployment, forwarded_keys=kwargs.pop(_ALIAS_MARKER_FORWARDED_PARAMS_KWARG, ())
|
||||
):
|
||||
kwargs.pop(key, None)
|
||||
kwargs.pop("daily_reset_time", None) # pyright: ignore[reportUnknownMemberType] # kwargs is an untyped dict
|
||||
defaults: Final = _LITELLM_PARAMS_ADAPTER.validate_python(self.default_litellm_params)
|
||||
self._merge_tools_from_deployment(deployment=deployment, kwargs=kwargs, defaults=defaults)
|
||||
|
||||
|
|
@ -5668,7 +5669,7 @@ class Router:
|
|||
def _image_generation(self, prompt: str, model: str, **kwargs):
|
||||
model_name: Final = ""
|
||||
try:
|
||||
verbose_router_logger.debug("Inside _image_generation()- model: %s; kwargs: %s", model, kwargs)
|
||||
verbose_router_logger.debug("Inside _image_generation()- model: %s", model)
|
||||
deployment: Final = self.get_available_deployment(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "prompt"}],
|
||||
|
|
@ -5730,7 +5731,7 @@ class Router:
|
|||
async def _aimage_generation(self, prompt: str, model: str, **kwargs):
|
||||
model_name = model
|
||||
try:
|
||||
verbose_router_logger.debug("Inside _image_generation()- model: %s; kwargs: %s", model, kwargs)
|
||||
verbose_router_logger.debug("Inside _image_generation()- model: %s", model)
|
||||
parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs)
|
||||
deployment: Final = await self.async_get_available_deployment(
|
||||
model=model,
|
||||
|
|
@ -5815,7 +5816,7 @@ class Router:
|
|||
async def _atranscription(self, file: FileTypes, model: str, **kwargs):
|
||||
model_name: Final = model
|
||||
try:
|
||||
verbose_router_logger.debug("Inside _atranscription()- model: %s; kwargs: %s", model, kwargs)
|
||||
verbose_router_logger.debug("Inside _atranscription()- model: %s", model)
|
||||
parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs)
|
||||
deployment: Final = await self.async_get_available_deployment(
|
||||
model=model,
|
||||
|
|
@ -5908,7 +5909,7 @@ class Router:
|
|||
async def _aspeech(self, model: str, input: str, voice: str | None = None, **kwargs):
|
||||
model_name: Final = model
|
||||
try:
|
||||
verbose_router_logger.debug("Inside _aspeech()- model: %s; kwargs: %s", model, kwargs)
|
||||
verbose_router_logger.debug("Inside _aspeech()- model: %s", model)
|
||||
parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs)
|
||||
deployment: Final = await self.async_get_available_deployment(
|
||||
model=model,
|
||||
|
|
@ -5970,7 +5971,7 @@ class Router:
|
|||
async def _arerank(self, model: str, **kwargs):
|
||||
model_name = None
|
||||
try:
|
||||
verbose_router_logger.debug("Inside _rerank()- model: %s; kwargs: %s", model, kwargs)
|
||||
verbose_router_logger.debug("Inside _rerank()- model: %s", model)
|
||||
deployment: Final = await self.async_get_available_deployment(
|
||||
model=model,
|
||||
specific_deployment=kwargs.pop("specific_deployment", None),
|
||||
|
|
@ -6079,7 +6080,7 @@ class Router:
|
|||
|
||||
async def _atext_completion(self, model: str, prompt: str, **kwargs):
|
||||
try:
|
||||
verbose_router_logger.debug("Inside _atext_completion()- model: %s; kwargs: %s", model, kwargs)
|
||||
verbose_router_logger.debug("Inside _atext_completion()- model: %s", model)
|
||||
parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs)
|
||||
deployment: Final = await self.async_get_available_deployment(
|
||||
model=model,
|
||||
|
|
@ -6149,7 +6150,7 @@ class Router:
|
|||
|
||||
async def _aadapter_completion(self, adapter_id: str, model: str, **kwargs):
|
||||
try:
|
||||
verbose_router_logger.debug("Inside _aadapter_completion()- model: %s; kwargs: %s", model, kwargs)
|
||||
verbose_router_logger.debug("Inside _aadapter_completion()- model: %s", model)
|
||||
parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs)
|
||||
deployment: Final = await self.async_get_available_deployment(
|
||||
model=model,
|
||||
|
|
@ -6243,7 +6244,7 @@ class Router:
|
|||
kwargs=kwargs,
|
||||
metadata_variable_name="litellm_metadata",
|
||||
)
|
||||
verbose_router_logger.debug("Inside aguardrail() - guardrail_name: %s; kwargs: %s", guardrail_name, kwargs)
|
||||
verbose_router_logger.debug("Inside aguardrail() - guardrail_name: %s", guardrail_name)
|
||||
response: Final = await self.async_function_with_fallbacks(**kwargs)
|
||||
return response
|
||||
|
||||
|
|
@ -6324,9 +6325,7 @@ class Router:
|
|||
controls: Final = per_request_fallback_controls(kwargs)
|
||||
kwargs[MID_STREAM_FALLBACK_CONTROLS_KEY] = controls # rebind-ok: forwarded to every hop
|
||||
self._update_kwargs_before_fallbacks(model=model, kwargs=kwargs, metadata_variable_name="litellm_metadata")
|
||||
verbose_router_logger.debug(
|
||||
"Inside ageneric_api_call_with_fallbacks() - model: %s; kwargs: %s", model, kwargs
|
||||
)
|
||||
verbose_router_logger.debug("Inside ageneric_api_call_with_fallbacks() - model: %s", model)
|
||||
response: Final = await self.async_function_with_fallbacks(**kwargs)
|
||||
return response
|
||||
except Exception as e:
|
||||
|
|
@ -7146,9 +7145,7 @@ class Router:
|
|||
handler_name: Final = original_function.__name__
|
||||
metadata_variable_name: Final = get_router_metadata_variable_name(function_name="generic_api_call")
|
||||
try:
|
||||
verbose_router_logger.debug(
|
||||
"Inside _generic_api_call() - handler: %s, model: %s; kwargs: %s", handler_name, model, kwargs
|
||||
)
|
||||
verbose_router_logger.debug("Inside _generic_api_call() - handler: %s, model: %s", handler_name, model)
|
||||
self._update_kwargs_before_fallbacks(
|
||||
model=model,
|
||||
kwargs=kwargs,
|
||||
|
|
@ -7223,7 +7220,7 @@ class Router:
|
|||
def _embedding(self, input: str | list[object], model: str, **kwargs):
|
||||
model_name = None
|
||||
try:
|
||||
verbose_router_logger.debug("Inside embedding()- model: %s; kwargs: %s", model, kwargs)
|
||||
verbose_router_logger.debug("Inside embedding()- model: %s", model)
|
||||
with embedding_deployment_selection():
|
||||
deployment: Final = self.get_available_deployment(
|
||||
model=model,
|
||||
|
|
@ -7312,7 +7309,7 @@ class Router:
|
|||
async def _aembedding(self, input: str | list[object], model: str, **kwargs):
|
||||
model_name = None
|
||||
try:
|
||||
verbose_router_logger.debug("Inside _aembedding()- model: %s; kwargs: %s", model, kwargs)
|
||||
verbose_router_logger.debug("Inside _aembedding()- model: %s", model)
|
||||
parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs)
|
||||
with embedding_deployment_selection():
|
||||
deployment: Final = await self.async_get_available_deployment(
|
||||
|
|
@ -7396,7 +7393,7 @@ class Router:
|
|||
try:
|
||||
from litellm.router_utils.common_utils import add_model_file_id_mappings
|
||||
|
||||
verbose_router_logger.debug("Inside _atext_completion()- model: %s; kwargs: %s", model, kwargs)
|
||||
verbose_router_logger.debug("Inside _atext_completion()- model: %s", model)
|
||||
parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs)
|
||||
healthy_deployments: Final = await self.async_get_healthy_deployments(
|
||||
model=model,
|
||||
|
|
@ -7634,7 +7631,7 @@ class Router:
|
|||
**kwargs,
|
||||
) -> LiteLLMBatch:
|
||||
try:
|
||||
verbose_router_logger.debug("Inside _acreate_batch()- model: %s; kwargs: %s", model, kwargs)
|
||||
verbose_router_logger.debug("Inside _acreate_batch()- model: %s", model)
|
||||
parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs)
|
||||
with batch_create_routing():
|
||||
deployment: Final = await self.async_get_available_deployment(
|
||||
|
|
@ -7833,7 +7830,7 @@ class Router:
|
|||
**kwargs,
|
||||
) -> LiteLLMBatch:
|
||||
try:
|
||||
verbose_router_logger.debug("Inside _acancel_batch()- model: %s; kwargs: %s", model, kwargs)
|
||||
verbose_router_logger.debug("Inside _acancel_batch()- model: %s", model)
|
||||
parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs)
|
||||
deployment: Final = await self.async_get_available_deployment(
|
||||
model=model,
|
||||
|
|
@ -10470,7 +10467,7 @@ class Router:
|
|||
|
||||
@staticmethod
|
||||
def _inherit_builtin_base_rates_for_scoped_pricing(
|
||||
model_info: dict, # mutable-ok: cost-map entry filled in place
|
||||
model_info: dict[str, object], # mutable-ok: cost-map entry filled in place
|
||||
backend_model: str,
|
||||
custom_llm_provider: str | None,
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ is logged the first time such a deployment is seen.
|
|||
"""
|
||||
|
||||
import contextlib
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Callable, Mapping
|
||||
from datetime import datetime, timedelta
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal
|
||||
|
||||
|
|
@ -38,7 +38,7 @@ from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import (
|
|||
)
|
||||
from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET
|
||||
from litellm.types.caching import RedisPipelineIncrementOperation
|
||||
from litellm.types.router import RouterErrors
|
||||
from litellm.types.router import RouterErrors, parse_daily_reset_time
|
||||
from litellm.types.utils import StandardLoggingPayload
|
||||
from litellm.utils import get_utc_datetime
|
||||
|
||||
|
|
@ -58,42 +58,49 @@ class RoutingArgs:
|
|||
DAY_FORMAT: Final = "%Y-%m-%d"
|
||||
|
||||
|
||||
def seconds_until_utc_midnight(now: datetime) -> int:
|
||||
next_midnight: Final = (now + timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
return max(1, int((next_midnight - now).total_seconds()))
|
||||
def daily_window_day(now: datetime, reset_offset: timedelta) -> str:
|
||||
return (now - reset_offset).strftime(DAY_FORMAT)
|
||||
|
||||
|
||||
def _first_set(*values: int | None) -> int | None:
|
||||
return next((value for value in values if value is not None), None)
|
||||
def seconds_until_daily_reset(now: datetime, reset_offset: timedelta = timedelta(0)) -> int:
|
||||
window_start: Final = (now - reset_offset).replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
next_reset: Final = window_start + timedelta(days=1) + reset_offset
|
||||
return max(1, int((next_reset - now).total_seconds()))
|
||||
|
||||
|
||||
class _DailyLimitFields(BaseModel):
|
||||
class _DailyLimitParams(BaseModel):
|
||||
"""rpd, tpd and daily_reset_time live in litellm_params only."""
|
||||
|
||||
model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True)
|
||||
model: str | None = None
|
||||
tpd: int | None = None
|
||||
rpd: int | None = None
|
||||
daily_reset_time: str | None = None
|
||||
|
||||
def reset_offset(self) -> timedelta:
|
||||
return parse_daily_reset_time(self.daily_reset_time)
|
||||
|
||||
|
||||
class _DailyLimitParams(_DailyLimitFields):
|
||||
model: str | None = None
|
||||
|
||||
|
||||
class _DailyLimitModelInfo(_DailyLimitFields):
|
||||
class _DailyLimitModelInfo(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True)
|
||||
id: str | None = None
|
||||
|
||||
|
||||
class _DailyLimitDeployment(_DailyLimitFields):
|
||||
class _DailyLimitDeployment(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True)
|
||||
model_name: str = ""
|
||||
litellm_params: _DailyLimitParams = _DailyLimitParams()
|
||||
model_info: _DailyLimitModelInfo = _DailyLimitModelInfo()
|
||||
|
||||
def limits(self) -> tuple[int | None, int | None]:
|
||||
return (
|
||||
_first_set(self.tpd, self.litellm_params.tpd, self.model_info.tpd),
|
||||
_first_set(self.rpd, self.litellm_params.rpd, self.model_info.rpd),
|
||||
)
|
||||
return self.litellm_params.tpd, self.litellm_params.rpd
|
||||
|
||||
def cache_keys(self, current_day: str) -> tuple[str, str]:
|
||||
def reset_offset(self) -> timedelta:
|
||||
return self.litellm_params.reset_offset()
|
||||
|
||||
def cache_keys(self, now: datetime) -> tuple[str, str]:
|
||||
prefix: Final = f"{self.model_info.id}:{self.litellm_params.model}"
|
||||
current_day: Final = daily_window_day(now, self.reset_offset())
|
||||
return f"{prefix}:tpd:{current_day}", f"{prefix}:rpd:{current_day}"
|
||||
|
||||
def limit_error(self, kind: Literal["tpd", "rpd"], limit: int, usage: int, now: datetime) -> litellm.RateLimitError:
|
||||
|
|
@ -104,12 +111,19 @@ class _DailyLimitDeployment(_DailyLimitFields):
|
|||
response=httpx.Response(
|
||||
status_code=429,
|
||||
content=f"{RouterErrors.user_defined_ratelimit_error.value} {kind} limit={limit}. current usage={usage}. id={self.model_info.id}, model_group={self.model_name}",
|
||||
headers={"retry-after": str(seconds_until_utc_midnight(now))},
|
||||
headers={"retry-after": str(seconds_until_daily_reset(now, self.reset_offset()))},
|
||||
request=httpx.Request(method="model_rate_limit_check", url="https://github.com/BerriAI/litellm"),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _reset_offset_from_kwargs(kwargs: Mapping[str, object]) -> timedelta:
|
||||
litellm_params: Final = kwargs.get("litellm_params")
|
||||
if litellm_params is None:
|
||||
return timedelta(0)
|
||||
return _DailyLimitParams.model_validate(litellm_params).reset_offset()
|
||||
|
||||
|
||||
class ModelRateLimitingCheck(CustomLogger):
|
||||
"""
|
||||
Pre-call check that enforces TPM/RPM or ITPM/OTPM limits on model deployments.
|
||||
|
|
@ -242,7 +256,7 @@ class ModelRateLimitingCheck(CustomLogger):
|
|||
tpd_limit, rpd_limit = deployment.limits()
|
||||
if tpd_limit is None and rpd_limit is None:
|
||||
return
|
||||
tpd_key, rpd_key = deployment.cache_keys(now.strftime(DAY_FORMAT))
|
||||
tpd_key, rpd_key = deployment.cache_keys(now)
|
||||
if tpd_limit is not None:
|
||||
current_tpd: Final = self._get_current_tpm(tpd_key, tpd_limit)
|
||||
if current_tpd is not None and current_tpd >= tpd_limit:
|
||||
|
|
@ -394,7 +408,7 @@ class ModelRateLimitingCheck(CustomLogger):
|
|||
|
||||
current_minute: Final = dt.strftime("%H-%M")
|
||||
tpm_key, rpm_key = self._get_cache_keys(deployment, current_minute)
|
||||
tpd_key, rpd_key = daily.cache_keys(dt.strftime(DAY_FORMAT))
|
||||
tpd_key, rpd_key = daily.cache_keys(dt)
|
||||
|
||||
model_id: Final = deployment.get("model_info", {}).get("id")
|
||||
model_name: Final = deployment.get("litellm_params", {}).get("model")
|
||||
|
|
@ -510,7 +524,8 @@ class ModelRateLimitingCheck(CustomLogger):
|
|||
dt: Final = self.clock()
|
||||
current_minute: Final = dt.strftime("%H-%M")
|
||||
tpm_key: Final = f"{model_id}:{model}:tpm:{current_minute}"
|
||||
tpd_key: Final = f"{model_id}:{model}:tpd:{dt.strftime(DAY_FORMAT)}"
|
||||
reset_offset: Final = _reset_offset_from_kwargs(kwargs) # pyright: ignore[reportUnknownArgumentType] # CustomLogger passes an untyped kwargs dict
|
||||
tpd_key: Final = f"{model_id}:{model}:tpd:{daily_window_day(dt, reset_offset)}"
|
||||
|
||||
verbose_router_logger.debug("[TPM TRACKING] Incrementing %s by %s", tpm_key, total_tokens)
|
||||
|
||||
|
|
@ -570,7 +585,8 @@ class ModelRateLimitingCheck(CustomLogger):
|
|||
dt: Final = self.clock()
|
||||
current_minute: Final = dt.strftime("%H-%M")
|
||||
tpm_key: Final = f"{model_id}:{model}:tpm:{current_minute}"
|
||||
tpd_key: Final = f"{model_id}:{model}:tpd:{dt.strftime(DAY_FORMAT)}"
|
||||
reset_offset: Final = _reset_offset_from_kwargs(kwargs) # pyright: ignore[reportUnknownArgumentType] # CustomLogger passes an untyped kwargs dict
|
||||
tpd_key: Final = f"{model_id}:{model}:tpd:{daily_window_day(dt, reset_offset)}"
|
||||
|
||||
tokens: Final = int(total_tokens)
|
||||
self._increment(tpm_key, tokens, RoutingArgs.ttl)
|
||||
|
|
|
|||
|
|
@ -155,10 +155,9 @@ class SearchAPIRouter:
|
|||
|
||||
available_search_tool_names: Final = [tool.get("search_tool_name") for tool in router_instance.search_tools]
|
||||
verbose_router_logger.debug(
|
||||
"Inside SearchAPIRouter.async_search_with_fallbacks() - search_tool_name: %s, Available Search Tools: %s, kwargs: %s",
|
||||
"Inside SearchAPIRouter.async_search_with_fallbacks() - search_tool_name: %s, Available Search Tools: %s",
|
||||
search_tool_name,
|
||||
available_search_tool_names,
|
||||
kwargs,
|
||||
)
|
||||
|
||||
# Use the existing retry/fallback infrastructure
|
||||
|
|
|
|||
|
|
@ -191,6 +191,7 @@ class DeploymentOptions:
|
|||
otpm: int | None = None
|
||||
rpd: int | None = None
|
||||
tpd: int | None = None
|
||||
daily_reset_time: str | None = None
|
||||
kubernetes_pod_discovery: bool | None = None
|
||||
default_api_key_rpm_limit: int | None = None
|
||||
default_api_key_tpm_limit: int | None = None
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ litellm.Router Types - includes RouterConfig, UpdateRouterConfig, ModelInfo etc
|
|||
|
||||
import datetime
|
||||
import enum
|
||||
import re
|
||||
from collections.abc import Container, Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import (
|
||||
|
|
@ -518,6 +519,19 @@ def holds_secret_pointer(param_name: str) -> bool:
|
|||
_RESERVED_INIT_KEYS: Final = frozenset({"self", "params", "__class__"})
|
||||
|
||||
|
||||
_DAILY_RESET_TIME_PATTERN: Final = re.compile(r"\d{2}:\d{2}")
|
||||
|
||||
|
||||
def parse_daily_reset_time(value: str | None) -> datetime.timedelta:
|
||||
"""`daily_reset_time` is "HH:MM" in UTC, when the rpd/tpd window starts over; unset means midnight UTC."""
|
||||
if value is None:
|
||||
return datetime.timedelta(0)
|
||||
if _DAILY_RESET_TIME_PATTERN.fullmatch(value) is None:
|
||||
raise ValueError(f"daily_reset_time must be HH:MM in UTC, got {value!r}")
|
||||
parsed: Final = datetime.time.fromisoformat(value)
|
||||
return datetime.timedelta(hours=parsed.hour, minutes=parsed.minute)
|
||||
|
||||
|
||||
class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
|
||||
"""
|
||||
LiteLLM Params without 'model' arg (used across completion / assistants api)
|
||||
|
|
@ -535,6 +549,7 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
|
|||
otpm: int | None = None
|
||||
tpd: int | None = None
|
||||
rpd: int | None = None
|
||||
daily_reset_time: str | None = None
|
||||
timeout: float | str | httpx.Timeout | SDKTimeout | None = None # if str, pass in as os.environ/
|
||||
# read timeout of a live stream, a faked stream takes the longer of this and timeout. If str, pass in as os.environ/
|
||||
stream_timeout: float | str | None = None
|
||||
|
|
@ -626,6 +641,13 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
|
|||
valkey_text_field: str | None = None
|
||||
valkey_embedding_field: str | None = None
|
||||
|
||||
@field_validator("daily_reset_time")
|
||||
@classmethod
|
||||
def _daily_reset_time_is_utc_hh_mm(cls, value: str | None) -> str | None:
|
||||
if value is not None:
|
||||
parse_daily_reset_time(value)
|
||||
return value
|
||||
|
||||
@field_validator("timeout")
|
||||
@classmethod
|
||||
def normalize_timeout(
|
||||
|
|
@ -742,6 +764,7 @@ class LiteLLMParamsTypedDict(TypedDict, total=False):
|
|||
otpm: int | None
|
||||
tpd: ReadOnly[int | None]
|
||||
rpd: ReadOnly[int | None]
|
||||
daily_reset_time: ReadOnly[str | None]
|
||||
order: int | None
|
||||
weight: int | None
|
||||
max_parallel_requests: int | None
|
||||
|
|
|
|||
|
|
@ -2879,7 +2879,7 @@ class TranscriptionUsageTokensObject(LiteLLMBaseModel):
|
|||
input_tokens: int
|
||||
output_tokens: int
|
||||
total_tokens: int
|
||||
input_token_details: TranscriptionUsageInputTokenDetailsObject
|
||||
input_token_details: TranscriptionUsageInputTokenDetailsObject | None = None
|
||||
|
||||
|
||||
class TranscriptionResponse(OpenAIObject):
|
||||
|
|
@ -4595,6 +4595,7 @@ class LlmProviders(str, Enum):
|
|||
META = "meta"
|
||||
SAIL = "sail"
|
||||
REKA = "reka"
|
||||
STREAMLAKE = "streamlake"
|
||||
LITELLM_AGENT = "litellm_agent"
|
||||
CURSOR = "cursor"
|
||||
BEDROCK_MANTLE = "bedrock_mantle"
|
||||
|
|
|
|||
|
|
@ -304,7 +304,7 @@ except (ImportError, AttributeError, TypeError):
|
|||
# Convert to str (if necessary)
|
||||
claude_json_str = json.dumps(json_data)
|
||||
import importlib.metadata
|
||||
from collections.abc import AsyncIterator, Callable, Collection, Iterable, Iterator, Mapping, Sequence
|
||||
from collections.abc import AsyncIterator, Callable, Collection, Iterable, Iterator, Mapping, Sequence, Sized
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypeVar, cast, runtime_checkable
|
||||
|
||||
from typing_extensions import assert_never
|
||||
|
|
@ -606,9 +606,23 @@ def custom_llm_setup():
|
|||
litellm.provider_list.append(custom_llm["provider"])
|
||||
|
||||
if custom_llm["provider"] not in litellm._custom_providers:
|
||||
_warn_if_built_in_provider_shadows_custom_handler(custom_llm["provider"])
|
||||
litellm._custom_providers.append(custom_llm["provider"])
|
||||
|
||||
|
||||
def _warn_if_built_in_provider_shadows_custom_handler(provider: str) -> None:
|
||||
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
|
||||
|
||||
if provider not in LlmProvidersSet and not JSONProviderRegistry.exists(provider):
|
||||
return
|
||||
verbose_logger.warning(
|
||||
"custom_provider_map provider '%s' has the same name as a built-in LiteLLM provider, which handles "
|
||||
"'%s/' models instead of your custom_handler. Rename the provider and its models to use your handler",
|
||||
provider,
|
||||
provider,
|
||||
)
|
||||
|
||||
|
||||
def add_custom_logger_callback_to_specific_event(callback: str, logging_event: Literal["success", "failure"]) -> None:
|
||||
"""
|
||||
Add a custom logger callback to the specific event
|
||||
|
|
@ -4067,7 +4081,9 @@ def get_optional_params_transcription(
|
|||
"timestamp_granularities": None,
|
||||
}
|
||||
|
||||
non_default_params = {k: v for k, v in passed_params.items() if (k in default_params and v != default_params[k])}
|
||||
non_default_params: Final = {
|
||||
k: v for k, v in passed_params.items() if (k in default_params and v != default_params[k])
|
||||
}
|
||||
optional_params = {}
|
||||
|
||||
## raise exception if non-default value passed for non-openai/azure embedding calls
|
||||
|
|
@ -4104,6 +4120,19 @@ def get_optional_params_transcription(
|
|||
model=model,
|
||||
drop_params=drop_params if drop_params is not None else False,
|
||||
)
|
||||
elif custom_llm_provider == "bedrock":
|
||||
from litellm.llms.bedrock.audio_transcription import (
|
||||
UnsupportedTranscriptionParams,
|
||||
map_openai_transcription_params,
|
||||
)
|
||||
|
||||
match map_openai_transcription_params(
|
||||
non_default_params, drop_params=drop_params is True or litellm.drop_params is True
|
||||
):
|
||||
case UnsupportedTranscriptionParams(message=message):
|
||||
raise UnsupportedParamsError(message=message, llm_provider="bedrock")
|
||||
case mapped_params:
|
||||
optional_params = dict(mapped_params)
|
||||
elif provider_config is not None: # custom audio transcription config
|
||||
supported_params = provider_config.get_supported_openai_params(model=model)
|
||||
_check_valid_arg(supported_params=supported_params)
|
||||
|
|
@ -4118,7 +4147,7 @@ def get_optional_params_transcription(
|
|||
optional_params=optional_params,
|
||||
passed_params=passed_params,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
openai_params=OPENAI_TRANSCRIPTION_PARAMS,
|
||||
openai_params=list(default_params) if custom_llm_provider == "bedrock" else OPENAI_TRANSCRIPTION_PARAMS,
|
||||
additional_drop_params=kwargs.get("additional_drop_params", None),
|
||||
)
|
||||
|
||||
|
|
@ -5059,8 +5088,10 @@ def get_optional_params(
|
|||
supported_params: List[str] - supported params from the litellm config
|
||||
"""
|
||||
verbose_logger.info("\nLiteLLM completion() model= %s; provider = %s", model, custom_llm_provider)
|
||||
verbose_logger.debug("\nLiteLLM: Params passed to completion() %s", passed_params)
|
||||
verbose_logger.debug("\nLiteLLM: Non-Default params passed to completion() %s", non_default_params)
|
||||
if verbose_logger.isEnabledFor(logging.DEBUG):
|
||||
verbose_logger.debug(
|
||||
"\nLiteLLM: Non-Default params passed to completion() %s", _debug_params_repr(non_default_params)
|
||||
)
|
||||
unsupported_params: Final = {}
|
||||
for k in non_default_params:
|
||||
if k not in supported_params:
|
||||
|
|
@ -5601,7 +5632,9 @@ def get_optional_params(
|
|||
additional_drop_params=additional_drop_params,
|
||||
)
|
||||
if _print_verbose_is_active():
|
||||
print_verbose(f"Final returned optional params: {redact_credentials_in_payload(optional_params)}")
|
||||
print_verbose(
|
||||
f"Final returned optional params: {_debug_params_repr(redact_credentials_in_payload(optional_params))}"
|
||||
)
|
||||
optional_params = apply_openai_param_overrides(
|
||||
optional_params=optional_params,
|
||||
non_default_params=non_default_params,
|
||||
|
|
@ -6610,8 +6643,6 @@ def get_model_info_helper(
|
|||
model=model, custom_llm_provider=custom_llm_provider or declared_authenticating_provider(model)
|
||||
)
|
||||
|
||||
verbose_logger.debug("checking potential_model_names in litellm.model_cost: %s", potential_model_names)
|
||||
|
||||
combined_model_name: Final = potential_model_names["combined_model_name"]
|
||||
stripped_model_name: Final = potential_model_names["stripped_model_name"]
|
||||
combined_stripped_model_name: Final = potential_model_names["combined_stripped_model_name"]
|
||||
|
|
@ -6714,6 +6745,7 @@ def get_model_info_helper(
|
|||
key, _model_info = generalization
|
||||
|
||||
if _model_info is None or key is None:
|
||||
verbose_logger.debug("model not found in litellm.model_cost, tried: %s", potential_model_names)
|
||||
raise ModelNotMappedError(_model_not_mapped_message(model, custom_llm_provider))
|
||||
_input_cost_per_token: float | None = _model_info.get("input_cost_per_token")
|
||||
if _input_cost_per_token is None:
|
||||
|
|
@ -8534,6 +8566,23 @@ def get_valid_models(
|
|||
return [] # NON-Blocking
|
||||
|
||||
|
||||
_MAX_INLINE_DEBUG_ARG_CHARS: Final = 500
|
||||
|
||||
|
||||
def _debug_arg_repr(value: object) -> str:
|
||||
"""Short form of a call argument, since the outgoing provider request log already prints the request body."""
|
||||
rendered: Final = repr(value)
|
||||
if len(rendered) <= _MAX_INLINE_DEBUG_ARG_CHARS:
|
||||
return rendered
|
||||
if isinstance(value, Sized):
|
||||
return f"<{type(value).__name__} len={len(value)}>"
|
||||
return f"<{type(value).__name__}>"
|
||||
|
||||
|
||||
def _debug_params_repr(params: Mapping[str, object]) -> str:
|
||||
return "{" + ", ".join(f"{key!r}: {_debug_arg_repr(value)}" for key, value in params.items()) + "}"
|
||||
|
||||
|
||||
def print_args_passed_to_litellm(original_function, args, kwargs):
|
||||
if not is_debugging_on():
|
||||
return
|
||||
|
|
@ -8554,9 +8603,9 @@ def print_args_passed_to_litellm(original_function, args, kwargs):
|
|||
):
|
||||
return
|
||||
|
||||
args_str: Final = ", ".join(map(repr, args))
|
||||
args_str: Final = ", ".join(map(_debug_arg_repr, args))
|
||||
redacted_kwargs: Final = redact_credentials_in_payload(kwargs)
|
||||
kwargs_str: Final = ", ".join(f"{key}={value!r}" for key, value in redacted_kwargs.items())
|
||||
kwargs_str: Final = ", ".join(f"{key}={_debug_arg_repr(value)}" for key, value in redacted_kwargs.items())
|
||||
print_verbose(
|
||||
"\n",
|
||||
) # new line before
|
||||
|
|
|
|||
|
|
@ -41939,6 +41939,120 @@
|
|||
"supports_system_messages": true,
|
||||
"supports_native_structured_output": true
|
||||
},
|
||||
"bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
"input_cost_per_token": 4.12e-08,
|
||||
"input_cost_per_token_batches": 2.06e-08,
|
||||
"input_cost_per_token_flex": 2.06e-08,
|
||||
"input_cost_per_token_priority": 7.21e-08,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 8192,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4.12e-08,
|
||||
"output_cost_per_token_batches": 2.06e-08,
|
||||
"output_cost_per_token_flex": 2.06e-08,
|
||||
"output_cost_per_token_priority": 7.21e-08,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json",
|
||||
"supports_audio_input": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_native_structured_output": true
|
||||
},
|
||||
"bedrock/ap-southeast-2/mistral.voxtral-small-24b-2507": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
"input_cost_per_token": 1.03e-07,
|
||||
"input_cost_per_token_batches": 5.15e-08,
|
||||
"input_cost_per_token_flex": 5.15e-08,
|
||||
"input_cost_per_token_priority": 1.8025e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 8192,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.09e-07,
|
||||
"output_cost_per_token_batches": 1.545e-07,
|
||||
"output_cost_per_token_flex": 1.545e-07,
|
||||
"output_cost_per_token_priority": 5.4075e-07,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json",
|
||||
"supports_audio_input": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_native_structured_output": true
|
||||
},
|
||||
"bedrock/ap-northeast-1/mistral.voxtral-mini-3b-2507": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
"input_cost_per_token": 5e-08,
|
||||
"input_cost_per_token_batches": 3e-08,
|
||||
"input_cost_per_token_flex": 3e-08,
|
||||
"input_cost_per_token_priority": 9e-08,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 8192,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5e-08,
|
||||
"output_cost_per_token_batches": 3e-08,
|
||||
"output_cost_per_token_flex": 3e-08,
|
||||
"output_cost_per_token_priority": 9e-08,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-northeast-1/index.json",
|
||||
"supports_audio_input": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_native_structured_output": true
|
||||
},
|
||||
"bedrock/ap-northeast-1/mistral.voxtral-small-24b-2507": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"input_cost_per_token_batches": 6e-08,
|
||||
"input_cost_per_token_flex": 6e-08,
|
||||
"input_cost_per_token_priority": 2.1e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 8192,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.6e-07,
|
||||
"output_cost_per_token_batches": 1.8e-07,
|
||||
"output_cost_per_token_flex": 1.8e-07,
|
||||
"output_cost_per_token_priority": 6.3e-07,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-northeast-1/index.json",
|
||||
"supports_audio_input": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_native_structured_output": true
|
||||
},
|
||||
"bedrock/eu-west-2/mistral.voxtral-mini-3b-2507": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
"input_cost_per_token": 6e-08,
|
||||
"input_cost_per_token_batches": 3e-08,
|
||||
"input_cost_per_token_flex": 3e-08,
|
||||
"input_cost_per_token_priority": 1.1e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 8192,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 6e-08,
|
||||
"output_cost_per_token_batches": 3e-08,
|
||||
"output_cost_per_token_flex": 3e-08,
|
||||
"output_cost_per_token_priority": 1.1e-07,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/eu-west-2/index.json",
|
||||
"supports_audio_input": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_native_structured_output": true
|
||||
},
|
||||
"bedrock/eu-west-2/mistral.voxtral-small-24b-2507": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
"input_cost_per_token": 1.6e-07,
|
||||
"input_cost_per_token_batches": 8e-08,
|
||||
"input_cost_per_token_flex": 8e-08,
|
||||
"input_cost_per_token_priority": 2.8e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 8192,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4.7e-07,
|
||||
"output_cost_per_token_batches": 2.3e-07,
|
||||
"output_cost_per_token_flex": 2.3e-07,
|
||||
"output_cost_per_token_priority": 8.2e-07,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/eu-west-2/index.json",
|
||||
"supports_audio_input": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_native_structured_output": true
|
||||
},
|
||||
"mistral/codestral-2508": {
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"input_cost_per_token": 3e-07,
|
||||
|
|
@ -61374,6 +61488,359 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"streamlake/deepseek-v4-flash-0731": {
|
||||
"cache_read_input_token_cost": 1.4e-08,
|
||||
"input_cost_per_token": 4.4e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 384000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.32e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/DeepSeek-V4-Pro-0813": {
|
||||
"cache_read_input_token_cost": 4.4e-08,
|
||||
"input_cost_per_token": 1.32e-06,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 1048566,
|
||||
"max_output_tokens": 384000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.96e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/DeepSeek-V3": {
|
||||
"cache_read_input_token_cost": 1.14e-07,
|
||||
"input_cost_per_token": 2.86e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 56000,
|
||||
"max_output_tokens": 8000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.143e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": false,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/GLM-5.3": {
|
||||
"cache_read_input_token_cost": 2.6e-07,
|
||||
"input_cost_per_token": 1.4e-06,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4.4e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/GLM-5.3-Flash": {
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5e-07,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": true,
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/Kimi-K2-Instruct-0905": {
|
||||
"cache_read_input_token_cost": 1.43e-07,
|
||||
"input_cost_per_token": 5.71e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 224000,
|
||||
"max_output_tokens": 32000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.286e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": false,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/deepseek-r1": {
|
||||
"cache_read_input_token_cost": 2.29e-07,
|
||||
"input_cost_per_token": 5.71e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 98304,
|
||||
"max_output_tokens": 16000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.286e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/deepseek-v3.1": {
|
||||
"cache_read_input_token_cost": 2.29e-07,
|
||||
"input_cost_per_token": 5.71e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 98304,
|
||||
"max_output_tokens": 32000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.714e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/deepseek-v4-flash": {
|
||||
"cache_read_input_token_cost": 2.8e-08,
|
||||
"input_cost_per_token": 1.4e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 384000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.8e-07,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/deepseek-v4-pro": {
|
||||
"cache_read_input_token_cost": 1.45e-07,
|
||||
"input_cost_per_token": 1.74e-06,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 384000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.48e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/deepseek-v4.1-flash-1017": {
|
||||
"cache_read_input_token_cost": 6e-09,
|
||||
"input_cost_per_token": 3e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 384000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.2e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_vision": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/deepseek-v4.1-flash-260910": {
|
||||
"cache_read_input_token_cost": 6e-09,
|
||||
"input_cost_per_token": 3e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 1048568,
|
||||
"max_output_tokens": 384000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.2e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_vision": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/glm-5": {
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"input_cost_per_token": 1e-06,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 202752,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.2e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/glm-5.1": {
|
||||
"cache_read_input_token_cost": 2.6e-07,
|
||||
"input_cost_per_token": 1.4e-06,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 202752,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4.4e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/glm-5.2": {
|
||||
"cache_read_input_token_cost": 2.6e-07,
|
||||
"input_cost_per_token": 1.4e-06,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 1048566,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4.4e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/kimi-k2.6": {
|
||||
"cache_read_input_token_cost": 1.6e-07,
|
||||
"input_cost_per_token": 9.5e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 256000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_vision": true,
|
||||
"supports_reasoning": false,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/kimi-k2.7-code": {
|
||||
"cache_read_input_token_cost": 1.9e-07,
|
||||
"input_cost_per_token": 9.5e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 32000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_vision": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/mimo-v2.6-flash": {
|
||||
"cache_read_input_token_cost": 2.8e-09,
|
||||
"input_cost_per_token": 1.4e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.8e-07,
|
||||
"supports_vision": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/mimo-v2.6-pro": {
|
||||
"cache_read_input_token_cost": 3.6e-09,
|
||||
"input_cost_per_token": 4.35e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 8.7e-07,
|
||||
"supports_vision": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/minimax-m2.5": {
|
||||
"cache_creation_input_token_cost": 3.75e-07,
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"input_cost_per_token": 3e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 196608,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.2e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/minimax-m3": {
|
||||
"cache_read_input_token_cost": 1.2e-07,
|
||||
"cache_read_input_token_cost_above_512k_tokens": 2.4e-07,
|
||||
"input_cost_per_token": 6e-07,
|
||||
"input_cost_per_token_above_512k_tokens": 1.2e-06,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 512000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.4e-06,
|
||||
"output_cost_per_token_above_512k_tokens": 4.8e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_vision": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/qwen3.5-397b-a17b": {
|
||||
"cache_read_input_token_cost": 1.2e-07,
|
||||
"input_cost_per_token": 6e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 258048,
|
||||
"max_output_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.6e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_vision": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/qwen3.6-27b": {
|
||||
"input_cost_per_token": 6e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 229376,
|
||||
"max_output_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.6e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_vision": true,
|
||||
"supports_reasoning": false,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/qwen3.6-35b-a3b": {
|
||||
"input_cost_per_token": 3.75e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 229376,
|
||||
"max_output_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.25e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_vision": true,
|
||||
"supports_reasoning": false,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/qwen3.8-2.4t-a95b": {
|
||||
"cache_read_input_token_cost": 2.5e-07,
|
||||
"input_cost_per_token": 2e-06,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 983616,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 6e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_vision": true,
|
||||
"cache_creation_input_token_cost": 2.5e-06,
|
||||
"explicit_cache_read_input_token_cost": 1.7e-07,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"streamlake/qwen3.8-27b": {
|
||||
"cache_read_input_token_cost": 1e-07,
|
||||
"input_cost_per_token": 5e-07,
|
||||
"litellm_provider": "streamlake",
|
||||
"max_input_tokens": 983616,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_vision": true,
|
||||
"cache_creation_input_token_cost": 6.25e-07,
|
||||
"explicit_cache_read_input_token_cost": 5e-08,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
|
||||
},
|
||||
"deepseek-v4-flash": {
|
||||
"cache_creation_input_token_cost": 0.0,
|
||||
"cache_read_input_token_cost": 6e-09,
|
||||
|
|
@ -62689,6 +63156,14 @@
|
|||
"litellm_provider": "anthropic"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "typesafe-jev-ids",
|
||||
"pattern": "^jev-(?:latest|preview|1\\.13\\.0|1\\.14\\.0)$",
|
||||
"description": "A bare Jev id that has a typesafe/ price entry (jev-latest, jev-preview, jev-1.13.0, jev-1.14.0), the name the /typesafe pass-through forwards to TypeSafe. Listed by name, matched case-insensitively like every rule here, so foo/jev-latest or a model group named like a Jev id TypeSafe does not serve, such as jev-1.0 or jev-internal-router, never resolves to typesafe. A new Jev version goes here together with its price entry. Provider inference feeds the proxy's provider/* wildcard access checks, so this rule is what lets typesafe/* on a key, team or project allowlist cover a jev-latest request.",
|
||||
"model_info": {
|
||||
"litellm_provider": "typesafe"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "claude-family-baseline",
|
||||
"pattern": "claude-[a-z]+-\\d+(?:[-.]\\d+)?(?:-\\d{8})?",
|
||||
|
|
|
|||
|
|
@ -3785,6 +3785,32 @@
|
|||
"generateContent": true
|
||||
}
|
||||
},
|
||||
"streamlake": {
|
||||
"display_name": "StreamLake (`streamlake`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/openai_compatible",
|
||||
"endpoints": {
|
||||
"chat_completions": true,
|
||||
"responses": true,
|
||||
"text_completion": true,
|
||||
"embeddings": false,
|
||||
"image_generations": false,
|
||||
"image_edits": false,
|
||||
"audio_transcriptions": false,
|
||||
"audio_speech": false,
|
||||
"rerank": false,
|
||||
"ocr": false,
|
||||
"search": false,
|
||||
"moderations": false,
|
||||
"realtime": false,
|
||||
"video_generations": false,
|
||||
"batches": false,
|
||||
"messages": true,
|
||||
"interactions": true,
|
||||
"systemone": false,
|
||||
"a2a": true,
|
||||
"generateContent": true
|
||||
}
|
||||
},
|
||||
"tencent": {
|
||||
"display_name": "Tencent TokenHub (`tencent`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/tencent",
|
||||
|
|
|
|||
|
|
@ -8,6 +8,10 @@ the same tests against Google's live spec
|
|||
Unit tests must not depend on the current time or execution speed. Use fixed timestamps for fixtures
|
||||
and controlled clocks for time-dependent behavior. Incidental clock reads inside production code are allowed
|
||||
|
||||
A unix socket goes under the `socket_dir` or `socket_path` fixture, never `tmp_path`, whose paths run past macOS's
|
||||
104-byte AF_UNIX limit and fail only on a Mac. Code that opens a socket in its own `tempfile` directory runs under
|
||||
`socket_tempdir`
|
||||
|
||||
## What good looks like
|
||||
|
||||
```python
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@ import asyncio
|
|||
import base64
|
||||
import importlib
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
from collections.abc import Callable, Coroutine, Iterator
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
|
@ -217,6 +219,38 @@ def _flush_client_caches() -> None:
|
|||
_reset_aws_auth_caches()
|
||||
|
||||
|
||||
_MACOS_LONGEST_SOCKET_PATH: Final = 103
|
||||
_LONGEST_SOCKET_SUFFIX: Final = len("/sock12345678/d/litellm-pgbouncer-12345678/.s.PGSQL.65535")
|
||||
|
||||
|
||||
def _socket_root() -> str:
|
||||
default: Final = tempfile.gettempdir()
|
||||
fits: Final = len(os.fsencode(default)) + _LONGEST_SOCKET_SUFFIX <= _MACOS_LONGEST_SOCKET_PATH
|
||||
return default if fits else "/tmp"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def socket_dir(tmp_path: Path) -> Iterator[Path]:
|
||||
"""``tmp_path`` behind a short link, so unix socket paths fit the sun_path limit that it exceeds on macOS."""
|
||||
link_dir: Final = Path(tempfile.mkdtemp(prefix="sock", dir=_socket_root()))
|
||||
link: Final = link_dir / "d"
|
||||
link.symlink_to(tmp_path, target_is_directory=True)
|
||||
yield link
|
||||
shutil.rmtree(link_dir, ignore_errors=True)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def socket_path(socket_dir: Path) -> Path:
|
||||
return socket_dir / "spend.sock"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def socket_tempdir(socket_dir: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
|
||||
"""Point ``tempfile``'s default directory at ``socket_dir``, for code that picks its own socket directory."""
|
||||
monkeypatch.setattr(tempfile, "tempdir", str(socket_dir))
|
||||
return socket_dir
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True, scope="session")
|
||||
def bundled_tiktoken_cache() -> None:
|
||||
importlib.import_module("litellm.litellm_core_utils.default_encoding")
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import unittest.mock as mock
|
||||
from typing import cast
|
||||
from typing import Final, cast
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -197,6 +197,17 @@ class TestEnterpriseCallbackControls:
|
|||
)
|
||||
assert result is False
|
||||
|
||||
def test_no_disable_request_logs_nothing(self, caplog: pytest.LogCaptureFixture) -> None:
|
||||
litellm_params: Final = {"proxy_server_request": {"url": "test", "headers": {"user-agent": "codex"}}}
|
||||
|
||||
with caplog.at_level("DEBUG", logger="LiteLLM"):
|
||||
result: Final = EnterpriseCallbackControls.is_callback_disabled_dynamically(
|
||||
DataDogLogger.__new__(DataDogLogger), litellm_params, StandardCallbackDynamicParams()
|
||||
)
|
||||
|
||||
assert result is False
|
||||
assert caplog.records == []
|
||||
|
||||
def test_callback_not_disabled_when_header_none(
|
||||
self, mock_premium_user, mock_request_headers
|
||||
):
|
||||
|
|
|
|||
|
|
@ -140,7 +140,7 @@ class TestPoolDatabaseUrl:
|
|||
assert "DATABASE_URL" in outcome.reason
|
||||
|
||||
def test_token_auth_hands_the_workers_the_pool_user_not_the_token(
|
||||
self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, socket_tempdir: Path
|
||||
):
|
||||
monkeypatch.setenv("IAM_TOKEN_DB_AUTH", "true")
|
||||
monkeypatch.setenv("AWS_REGION_NAME", "us-east-1")
|
||||
|
|
@ -149,7 +149,7 @@ class TestPoolDatabaseUrl:
|
|||
with _minted_iam_token("MINTED_TOKEN"):
|
||||
outcome: Final = pool_database_url(
|
||||
DatabaseURLSettings.from_env(),
|
||||
PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pooler(tmp_path))),
|
||||
PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pooler(socket_tempdir))),
|
||||
environ,
|
||||
)
|
||||
assert isinstance(outcome, str), outcome
|
||||
|
|
@ -160,12 +160,12 @@ class TestPoolDatabaseUrl:
|
|||
|
||||
class TestMain:
|
||||
def test_workers_inherit_the_loopback_url_the_supervisor_installed(
|
||||
self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, socket_tempdir: Path
|
||||
):
|
||||
port: Final = _free_port()
|
||||
monkeypatch.setenv("LITELLM_PGBOUNCER_ENABLED", "true")
|
||||
monkeypatch.setenv("LITELLM_PGBOUNCER_PORT", str(port))
|
||||
monkeypatch.setenv("LITELLM_PGBOUNCER_BINARY", str(_fake_pooler(tmp_path)))
|
||||
monkeypatch.setenv("LITELLM_PGBOUNCER_BINARY", str(_fake_pooler(socket_tempdir)))
|
||||
monkeypatch.setenv("KEEPALIVE_TIMEOUT", "75")
|
||||
served: Final[list[tuple[str, ...]]] = []
|
||||
main(("--workers", "4"), serve=lambda argv: served.append(tuple(argv)))
|
||||
|
|
@ -183,7 +183,7 @@ class TestMain:
|
|||
assert _query(os.environ["DATABASE_URL"])["pgbouncer"] == "true"
|
||||
|
||||
def test_iam_workers_keep_the_loopback_url_instead_of_minting_their_own(
|
||||
self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, socket_tempdir: Path
|
||||
):
|
||||
port: Final = _free_port()
|
||||
monkeypatch.delenv("DATABASE_PASSWORD")
|
||||
|
|
@ -191,7 +191,7 @@ class TestMain:
|
|||
monkeypatch.setenv("AWS_REGION_NAME", "us-east-1")
|
||||
monkeypatch.setenv("LITELLM_PGBOUNCER_ENABLED", "true")
|
||||
monkeypatch.setenv("LITELLM_PGBOUNCER_PORT", str(port))
|
||||
monkeypatch.setenv("LITELLM_PGBOUNCER_BINARY", str(_fake_pooler(tmp_path)))
|
||||
monkeypatch.setenv("LITELLM_PGBOUNCER_BINARY", str(_fake_pooler(socket_tempdir)))
|
||||
served: Final[list[tuple[str, ...]]] = []
|
||||
with _minted_iam_token("SUPERVISOR_TOKEN"):
|
||||
main(("--workers", "4"), serve=lambda argv: served.append(tuple(argv)))
|
||||
|
|
|
|||
|
|
@ -905,10 +905,10 @@ async def test_a_failed_request_is_delivered_as_an_error_log_that_keeps_the_erro
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_failing_redis_cache_is_delivered_to_datadog_as_redis_warnings(
|
||||
delivery: tuple[DataDogLogger, respx.Route], monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
delivery: tuple[DataDogLogger, respx.Route], monkeypatch: pytest.MonkeyPatch, socket_dir: Path
|
||||
) -> None:
|
||||
datadog_logger, intake = delivery
|
||||
absent_socket: Final = str(tmp_path / "absent.sock")
|
||||
absent_socket: Final = str(socket_dir / "absent.sock")
|
||||
redis_cache: Final = Cache(type="redis", url=f"unix://{absent_socket}")
|
||||
monkeypatch.setattr(redis_cache.cache.service_logger_obj, "dd_logger", datadog_logger, raising=False)
|
||||
monkeypatch.setattr(litellm, "service_callback", ["datadog"])
|
||||
|
|
|
|||
|
|
@ -714,6 +714,48 @@ class TestClaudeModelPatternMatching:
|
|||
litellm.get_llm_provider(model="claude-opus-4-9")
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
|
||||
class TestTypesafeJevPatternMatching:
|
||||
def test_every_priced_jev_id_routes_to_typesafe(
|
||||
self, shipped_generalizations: None
|
||||
) -> None:
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
|
||||
priced_ids: Final = tuple(
|
||||
key.removeprefix("typesafe/")
|
||||
for key in GetModelCostMap.load_local_model_cost_map()
|
||||
if key.startswith("typesafe/")
|
||||
)
|
||||
assert "jev-latest" in priced_ids
|
||||
assert {
|
||||
model: litellm.get_llm_provider(model=model)[1] for model in priced_ids
|
||||
} == dict.fromkeys(priced_ids, "typesafe")
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"foo/jev-latest",
|
||||
"openjev-latest",
|
||||
"jev-",
|
||||
"jev-latest model",
|
||||
"jev-internal-router",
|
||||
"jev-2.0-internal",
|
||||
"jev-1.0-acme.router",
|
||||
"jev-1.0",
|
||||
"jev-2.0",
|
||||
"jev-1.13",
|
||||
],
|
||||
)
|
||||
def test_namespaced_or_partial_jev_name_does_not_match_rule(
|
||||
self, model: str, shipped_generalizations: None
|
||||
) -> None:
|
||||
from litellm.litellm_core_utils.fallback_generalizations import (
|
||||
match_routing_generalization,
|
||||
)
|
||||
|
||||
assert match_routing_generalization(model) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("prefix", ["openai", "anthropic", "custom_openai", "openai_like"])
|
||||
def test_opencode_affinity_resolution_preserves_existing_transport(prefix: str) -> None:
|
||||
api_base: Final = "https://opencode.ai/zen/go/v1"
|
||||
|
|
|
|||
|
|
@ -6316,6 +6316,22 @@ def test_chunk_creator_preserves_hidden_provider_specific_fields_from_parsed_chu
|
|||
assert assembled._hidden_params["provider_specific_fields"] == {"traffic_type": "ON_DEMAND_FLEX"}
|
||||
|
||||
|
||||
def test_the_usage_chunk_carries_the_bill_the_builder_priced_for_the_spend_row() -> None:
|
||||
from litellm.litellm_core_utils.streaming_handler import (
|
||||
_stamp_usage_cost, # pyright: ignore[reportPrivateUsage] # the helper under test
|
||||
)
|
||||
|
||||
complete: Final = ModelResponse(model="omni-model", usage=Usage(prompt_tokens=100, completion_tokens=20))
|
||||
complete.hidden_params = {"response_cost": 0.00032}
|
||||
chunk_usage: Final = Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120)
|
||||
setattr(chunk_usage, "cost", 0.037)
|
||||
usage_chunk: Final = ModelResponseStream(id="chunk-1", model="omni-model", choices=[], usage=chunk_usage)
|
||||
|
||||
_stamp_usage_cost(usage_chunk, complete)
|
||||
|
||||
assert usage_chunk.usage is not None and usage_chunk.usage.cost == 0.00032, "the client sees what the row bills"
|
||||
|
||||
|
||||
def test_chunk_creator_keeps_provider_model_private_across_stream():
|
||||
from litellm.router_utils.add_retry_fallback_headers import (
|
||||
get_hidden_params_dict,
|
||||
|
|
|
|||
|
|
@ -165,6 +165,44 @@ def test_explicit_v1_api_base_keeps_responses_api_version_selection(
|
|||
|
||||
|
||||
@pytest.mark.serial
|
||||
@pytest.mark.parametrize(
|
||||
("api_base", "api_version", "expected_url"),
|
||||
[
|
||||
(
|
||||
"https://x.openai.azure.com",
|
||||
"2025-04-01-preview",
|
||||
"https://x.openai.azure.com/openai/v1/responses/compact",
|
||||
),
|
||||
(
|
||||
"https://x.openai.azure.com",
|
||||
"preview",
|
||||
"https://x.openai.azure.com/openai/v1/responses/compact?api-version=preview",
|
||||
),
|
||||
(
|
||||
"https://gw.example.com/foundry/openai/openai/v1?tenant=example",
|
||||
"v1",
|
||||
"https://gw.example.com/foundry/openai/openai/v1/responses/compact?tenant=example&api-version=v1",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_compact_reaches_the_v1_route_and_drops_only_a_dated_api_version(
|
||||
api_base: str, api_version: str, expected_url: str
|
||||
) -> None:
|
||||
config: Final = AzureOpenAIResponsesAPIConfig()
|
||||
responses_url: Final = config.get_complete_url(api_base=api_base, litellm_params={"api_version": api_version})
|
||||
|
||||
url, _ = config.transform_compact_response_api_request(
|
||||
model="gpt-5.4-mini",
|
||||
input="hi",
|
||||
response_api_optional_request_params={},
|
||||
api_base=responses_url,
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert url == expected_url
|
||||
|
||||
|
||||
def test_response_id_path_requests_encode_response_id():
|
||||
config = AzureOpenAIResponsesAPIConfig()
|
||||
api_base = (
|
||||
|
|
|
|||
|
|
@ -1658,9 +1658,9 @@ def test_get_azure_ad_token_fallback_to_default_azure_credential(
|
|||
)
|
||||
def test_is_azure_v1_api_version(api_version, expected):
|
||||
"""
|
||||
Test that _is_azure_v1_api_version correctly identifies v1 API versions.
|
||||
Test that is_azure_v1_api_version correctly identifies v1 API versions.
|
||||
"""
|
||||
result = BaseAzureLLM._is_azure_v1_api_version(api_version=api_version)
|
||||
result = BaseAzureLLM.is_azure_v1_api_version(api_version=api_version)
|
||||
assert result == expected
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import respx
|
|||
|
||||
import litellm
|
||||
from litellm.llms.litellm_proxy.responses.transformation import LiteLLMProxyResponsesAPIConfig
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import LlmProviders
|
||||
from litellm.utils import ProviderConfigManager
|
||||
from tests.unit.proxy.conftest import httpx_transport
|
||||
|
|
@ -73,3 +74,23 @@ def test_get_complete_url_requires_api_base(monkeypatch: pytest.MonkeyPatch) ->
|
|||
monkeypatch.delenv("LITELLM_PROXY_API_BASE", raising=False)
|
||||
with pytest.raises(ValueError, match="api_base not set"):
|
||||
LiteLLMProxyResponsesAPIConfig().get_complete_url(api_base=None, litellm_params={})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("drop_params", [False, True])
|
||||
def test_compact_request_forwards_metadata_and_user_to_the_downstream_proxy(drop_params: bool) -> None:
|
||||
url, data = LiteLLMProxyResponsesAPIConfig().transform_compact_response_api_request(
|
||||
model="gpt-5.5",
|
||||
input="hi",
|
||||
response_api_optional_request_params={"metadata": {"ticket": "qa-1"}, "user": "end-user-1", "temperature": 0.2},
|
||||
api_base="https://my-proxy.example.com/responses",
|
||||
litellm_params=GenericLiteLLMParams(drop_params=drop_params),
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert url == "https://my-proxy.example.com/responses/compact"
|
||||
assert {key: data[key] for key in data if key != "input"} == {
|
||||
"model": "gpt-5.5",
|
||||
"metadata": {"ticket": "qa-1"},
|
||||
"user": "end-user-1",
|
||||
"temperature": 0.2,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,12 +7,14 @@ with guardrail transformations.
|
|||
|
||||
import copy
|
||||
import json
|
||||
import uuid
|
||||
from collections.abc import Callable
|
||||
from typing import Any, Final, List, Literal, Optional, Tuple
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
|
||||
|
|
@ -33,6 +35,10 @@ from litellm.llms.openai.responses.guardrail_translation.handler import (
|
|||
)
|
||||
from litellm.llms.openai.responses.guardrail_translation.tool_merge import merge_guardrailed_tools
|
||||
from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api import GenericGuardrailAPI
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import BedrockGuardrail
|
||||
from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ContentFilterGuardrail
|
||||
from litellm.types.guardrails import ContentFilterAction, ContentFilterPattern
|
||||
from litellm.types.llms.openai import ChatCompletionToolCallChunk
|
||||
from litellm.responses.litellm_completion_transformation.compaction_items import compaction_summary
|
||||
from litellm.responses.litellm_completion_transformation.transformation import (
|
||||
|
|
@ -83,6 +89,7 @@ class RecordingMaskingGuardrail(MockGuardrail):
|
|||
super().__init__(guardrail_name=guardrail_name)
|
||||
self.seen_texts: list[list[str]] = []
|
||||
self.seen_message_contents: list[list[object]] = []
|
||||
self.seen_tools: list[object] = []
|
||||
|
||||
async def apply_guardrail(
|
||||
self,
|
||||
|
|
@ -93,6 +100,7 @@ class RecordingMaskingGuardrail(MockGuardrail):
|
|||
) -> GenericGuardrailAPIInputs:
|
||||
self.seen_texts.append(list(inputs.get("texts", [])))
|
||||
self.seen_message_contents.append([m["content"] for m in inputs.get("structured_messages") or []])
|
||||
self.seen_tools.append(inputs.get("tools"))
|
||||
return await super().apply_guardrail(inputs, request_data, input_type, logging_obj)
|
||||
|
||||
|
||||
|
|
@ -588,6 +596,638 @@ class TestSkipSystemMessageScopesInstructions:
|
|||
]
|
||||
|
||||
|
||||
_TOOL_TURNS: Final = (
|
||||
(
|
||||
{"type": "function_call", "call_id": "call_1", "name": "get_weather", "arguments": '{"city": "Paris"}'},
|
||||
"function_call_output",
|
||||
),
|
||||
(
|
||||
{"type": "custom_tool_call", "call_id": "call_1", "name": "run_sql", "input": "select 1"},
|
||||
"custom_tool_call_output",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _tool_result_request(call_item: dict[str, str], output_type: str, output: object) -> dict[str, object]:
|
||||
return {
|
||||
"model": "gpt-5.4-mini",
|
||||
"input": [
|
||||
{"role": "user", "content": "Weather?"},
|
||||
call_item,
|
||||
{"type": output_type, "call_id": "call_1", "output": output},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _bedrock_guardrail(
|
||||
name: str, bedrock_api: Callable[[httpx.Request], httpx.Response], **settings: bool
|
||||
) -> BedrockGuardrail:
|
||||
return BedrockGuardrail(
|
||||
guardrail_name=name,
|
||||
guardrailIdentifier="test-guardrail",
|
||||
guardrailVersion="DRAFT",
|
||||
aws_access_key_id="test-access-key",
|
||||
aws_secret_access_key="test-secret-key",
|
||||
aws_region_name="us-east-1",
|
||||
async_handler=AsyncHTTPHandler(transport=httpx.MockTransport(bedrock_api)),
|
||||
**settings,
|
||||
)
|
||||
|
||||
|
||||
def _bedrock_block_texts(request: httpx.Request) -> list[str]:
|
||||
return [block["text"]["text"] for block in json.loads(request.content)["content"]]
|
||||
|
||||
|
||||
class TestToolResultItemsReachTextsGuardrails:
|
||||
"""A Responses tool result is scanned and rewritten in its `output` field the way a chat `tool`
|
||||
message's content is, and skip_tool_message_in_guardrail keeps it out of the scan the same way."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(("call_item", "output_type"), _TOOL_TURNS, ids=["function", "custom"])
|
||||
async def test_string_output_is_scanned_and_rewritten_in_place(
|
||||
self, call_item: dict[str, str], output_type: str
|
||||
) -> None:
|
||||
guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test")
|
||||
|
||||
result: Final = await OpenAIResponsesHandler().process_input_messages(
|
||||
_tool_result_request(call_item, output_type, "SECRET forecast"), guardrail
|
||||
)
|
||||
|
||||
assert guardrail.seen_texts == [["Weather?", "SECRET forecast"]]
|
||||
assert result["input"] == [
|
||||
{"role": "user", "content": "Weather? [GUARDRAILED]"},
|
||||
call_item,
|
||||
{"type": output_type, "call_id": "call_1", "output": "SECRET forecast [GUARDRAILED]"},
|
||||
]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_content_part_output_is_scanned_and_rewritten_in_its_part(self) -> None:
|
||||
guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test")
|
||||
call_item, output_type = _TOOL_TURNS[0]
|
||||
image_part: Final = {"type": "input_image", "image_url": "https://example.com/map.png"}
|
||||
|
||||
result: Final = await OpenAIResponsesHandler().process_input_messages(
|
||||
_tool_result_request(
|
||||
call_item, output_type, [{"type": "input_text", "text": "SECRET forecast"}, image_part]
|
||||
),
|
||||
guardrail,
|
||||
)
|
||||
|
||||
assert guardrail.seen_texts == [["Weather?", "SECRET forecast"]]
|
||||
assert result["input"][2]["output"] == [
|
||||
{"type": "input_text", "text": "SECRET forecast [GUARDRAILED]"},
|
||||
image_part,
|
||||
]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("part_type", ["input_text", "output_text"])
|
||||
async def test_text_only_parts_are_scanned_and_rewritten_part_by_part(self, part_type: str) -> None:
|
||||
guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test")
|
||||
call_item, output_type = _TOOL_TURNS[0]
|
||||
parts: Final = [
|
||||
{"type": part_type, "text": "SECRET forecast: ", "annotations": []},
|
||||
{"type": part_type, "text": "sunny"},
|
||||
]
|
||||
|
||||
result: Final = await OpenAIResponsesHandler().process_input_messages(
|
||||
_tool_result_request(call_item, output_type, parts), guardrail
|
||||
)
|
||||
|
||||
assert guardrail.seen_texts == [["Weather?", "SECRET forecast: ", "sunny"]]
|
||||
assert result["input"][2]["output"] == [
|
||||
{"type": part_type, "text": "SECRET forecast: [GUARDRAILED]", "annotations": []},
|
||||
{"type": part_type, "text": "sunny [GUARDRAILED]"},
|
||||
]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_pattern_ending_at_a_part_boundary_blocks_like_it_does_on_chat(self) -> None:
|
||||
access_key: Final = "AKIA" + "SYNTHETIC0000000"
|
||||
content_filter: Final = ContentFilterGuardrail(
|
||||
guardrail_name="secrets",
|
||||
patterns=[
|
||||
ContentFilterPattern(
|
||||
pattern_type="prebuilt", pattern_name="aws_access_key", action=ContentFilterAction.BLOCK
|
||||
)
|
||||
],
|
||||
)
|
||||
call_item, output_type = _TOOL_TURNS[0]
|
||||
parts: Final = [
|
||||
{"type": "input_text", "text": f"key={access_key}"},
|
||||
{"type": "input_text", "text": "region=us-east-1"},
|
||||
]
|
||||
|
||||
with pytest.raises(HTTPException) as blocked:
|
||||
await OpenAIResponsesHandler().process_input_messages(
|
||||
_tool_result_request(call_item, output_type, parts), content_filter
|
||||
)
|
||||
|
||||
assert blocked.value.status_code == 400
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("empty_output", ["", None], ids=["empty_string", "null"])
|
||||
async def test_empty_tool_output_sends_no_text_and_stays_as_sent(self, empty_output: str | None) -> None:
|
||||
guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test")
|
||||
call_item, output_type = _TOOL_TURNS[0]
|
||||
|
||||
result: Final = await OpenAIResponsesHandler().process_input_messages(
|
||||
_tool_result_request(call_item, output_type, empty_output), guardrail
|
||||
)
|
||||
|
||||
assert guardrail.seen_texts == [["Weather?"]]
|
||||
assert result["input"][2] == {"type": output_type, "call_id": "call_1", "output": empty_output}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("latest_user_only", [False, True], ids=["all_texts", "latest_user_only"])
|
||||
@pytest.mark.parametrize(
|
||||
"tool_output",
|
||||
[
|
||||
{"type": "function_call_output", "call_id": "call_1", "output": ""},
|
||||
{"type": "function_call_output", "call_id": "call_1", "output": None},
|
||||
{
|
||||
"type": "computer_call_output",
|
||||
"call_id": "call_1",
|
||||
"output": {"type": "computer_screenshot", "file_id": "file-screen"},
|
||||
},
|
||||
],
|
||||
ids=["empty_string", "null", "file_id_screenshot"],
|
||||
)
|
||||
async def test_bedrock_masking_applies_to_a_request_with_a_textless_tool_output(
|
||||
self, tool_output: dict[str, object], latest_user_only: bool
|
||||
) -> None:
|
||||
sent_texts: Final[list[list[str]]] = []
|
||||
|
||||
def bedrock_masking_emails(request: httpx.Request) -> httpx.Response:
|
||||
texts: Final = _bedrock_block_texts(request)
|
||||
sent_texts.append(texts)
|
||||
outputs: Final = [{"text": text.replace("jane@example.com", "{EMAIL}")} for text in texts]
|
||||
return httpx.Response(200, json={"action": "GUARDRAIL_INTERVENED", "outputs": outputs})
|
||||
|
||||
guardrail: Final = _bedrock_guardrail(
|
||||
"bedrock-pre", bedrock_masking_emails, experimental_use_latest_role_message_only=latest_user_only
|
||||
)
|
||||
|
||||
request: Final = {
|
||||
"model": "gpt-5.4-mini",
|
||||
"input": [
|
||||
{"role": "user", "content": "Look up the order."},
|
||||
{"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"},
|
||||
copy.deepcopy(tool_output),
|
||||
{"role": "user", "content": "Email jane@example.com the result."},
|
||||
],
|
||||
}
|
||||
|
||||
result: Final = await OpenAIResponsesHandler().process_input_messages(request, guardrail)
|
||||
|
||||
expected_texts: Final = (
|
||||
[["Email jane@example.com the result."]]
|
||||
if latest_user_only
|
||||
else [["Look up the order.", "Email jane@example.com the result."]]
|
||||
)
|
||||
assert sent_texts == expected_texts
|
||||
assert result["input"][3]["content"] == "Email {EMAIL} the result."
|
||||
assert result["input"][2] == tool_output
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bedrock_latest_user_masking_applies_with_a_multi_part_tool_output(self) -> None:
|
||||
sent_texts: Final[list[list[str]]] = []
|
||||
|
||||
def bedrock_masking_emails(request: httpx.Request) -> httpx.Response:
|
||||
texts: Final = _bedrock_block_texts(request)
|
||||
sent_texts.append(texts)
|
||||
outputs: Final = [{"text": text.replace("jane@example.com", "{EMAIL}")} for text in texts]
|
||||
return httpx.Response(200, json={"action": "GUARDRAIL_INTERVENED", "outputs": outputs})
|
||||
|
||||
guardrail: Final = _bedrock_guardrail(
|
||||
"bedrock-pre", bedrock_masking_emails, experimental_use_latest_role_message_only=True
|
||||
)
|
||||
tool_output: Final = {
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_1",
|
||||
"output": [{"type": "input_text", "text": "order 42"}, {"type": "input_text", "text": "shipped"}],
|
||||
}
|
||||
request: Final = {
|
||||
"model": "gpt-5.4-mini",
|
||||
"input": [
|
||||
{"role": "user", "content": "Look up the order."},
|
||||
{"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"},
|
||||
copy.deepcopy(tool_output),
|
||||
{"role": "user", "content": "Email jane@example.com the result."},
|
||||
],
|
||||
}
|
||||
|
||||
result: Final = await OpenAIResponsesHandler().process_input_messages(request, guardrail)
|
||||
|
||||
assert sent_texts == [["Email jane@example.com the result."]]
|
||||
assert result["input"][3]["content"] == "Email {EMAIL} the result."
|
||||
assert result["input"][2] == tool_output
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bedrock_only_scan_new_messages_scans_only_the_new_tool_result(self) -> None:
|
||||
sent_texts: Final[list[list[str]]] = []
|
||||
|
||||
def bedrock_allowing(request: httpx.Request) -> httpx.Response:
|
||||
sent_texts.append(_bedrock_block_texts(request))
|
||||
return httpx.Response(200, json={"action": "NONE", "outputs": []})
|
||||
|
||||
guardrail: Final = _bedrock_guardrail("bedrock-incremental", bedrock_allowing, only_scan_new_messages=True)
|
||||
|
||||
first_turn: Final = [
|
||||
{"role": "user", "content": "Weather in Paris and Rome?"},
|
||||
{"type": "function_call", "call_id": "call_1", "name": "get_weather", "arguments": '{"city": "Paris"}'},
|
||||
{"type": "function_call_output", "call_id": "call_1", "output": "Paris: sunny"},
|
||||
]
|
||||
second_turn: Final = [
|
||||
*first_turn,
|
||||
{"type": "function_call", "call_id": "call_2", "name": "get_weather", "arguments": '{"city": "Rome"}'},
|
||||
{"type": "function_call_output", "call_id": "call_2", "output": "Rome: rain"},
|
||||
]
|
||||
session: Final = {"litellm_session_id": f"session-{uuid.uuid4()}"}
|
||||
|
||||
for turn in (first_turn, second_turn):
|
||||
await OpenAIResponsesHandler().process_input_messages(
|
||||
{"model": "gpt-5.4-mini", "input": copy.deepcopy(turn), **session}, guardrail
|
||||
)
|
||||
|
||||
assert sent_texts == [["Weather in Paris and Rome?", "Paris: sunny"], ["Rome: rain"]]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_id_screenshot_is_not_scanned_as_text_or_rewritten(self) -> None:
|
||||
guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test")
|
||||
screenshot_output: Final = {
|
||||
"type": "computer_call_output",
|
||||
"call_id": "cu_1",
|
||||
"output": {"type": "computer_screenshot", "file_id": "file-SECRET-screen"},
|
||||
}
|
||||
request: Final = {
|
||||
"model": "gpt-5.4-mini",
|
||||
"input": [
|
||||
{"role": "user", "content": "Open the page"},
|
||||
{"type": "computer_call", "call_id": "cu_1", "id": "cu_1", "action": {"type": "screenshot"}},
|
||||
copy.deepcopy(screenshot_output),
|
||||
],
|
||||
}
|
||||
|
||||
result: Final = await OpenAIResponsesHandler().process_input_messages(request, guardrail)
|
||||
|
||||
assert guardrail.seen_texts == [["Open the page"]]
|
||||
assert result["input"][2] == screenshot_output
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_text_parts_between_file_images_are_rewritten_in_place(self) -> None:
|
||||
guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test")
|
||||
call_item, output_type = _TOOL_TURNS[0]
|
||||
logo_part: Final = {"type": "input_image", "file_id": "file-logo"}
|
||||
image_part: Final = {"type": "input_image", "file_id": "file-map"}
|
||||
parts: Final = [
|
||||
logo_part,
|
||||
{"type": "input_text", "text": "SECRET forecast: "},
|
||||
image_part,
|
||||
{"type": "input_text", "text": "sunny"},
|
||||
]
|
||||
|
||||
result: Final = await OpenAIResponsesHandler().process_input_messages(
|
||||
_tool_result_request(call_item, output_type, parts), guardrail
|
||||
)
|
||||
|
||||
assert guardrail.seen_texts == [["Weather?", "SECRET forecast: ", "sunny"]]
|
||||
assert result["input"][2]["output"] == [
|
||||
logo_part,
|
||||
{"type": "input_text", "text": "SECRET forecast: [GUARDRAILED]"},
|
||||
image_part,
|
||||
{"type": "input_text", "text": "sunny [GUARDRAILED]"},
|
||||
]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dict_output_is_rewritten_as_the_text_the_guardrail_returns(self) -> None:
|
||||
guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test")
|
||||
call_item, output_type = _TOOL_TURNS[0]
|
||||
|
||||
result: Final = await OpenAIResponsesHandler().process_input_messages(
|
||||
_tool_result_request(call_item, output_type, {"result": "SECRET forecast"}), guardrail
|
||||
)
|
||||
|
||||
assert result["input"][2]["output"] == '{"result": "SECRET forecast"} [GUARDRAILED]'
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dict_output_the_guardrail_leaves_unchanged_stays_a_dict(self) -> None:
|
||||
guardrail: Final = TextsReplacingGuardrail(guardrail_name="test", texts=("[MASKED]", '{"result": "sunny"}'))
|
||||
call_item, output_type = _TOOL_TURNS[0]
|
||||
|
||||
result: Final = await OpenAIResponsesHandler().process_input_messages(
|
||||
_tool_result_request(call_item, output_type, {"result": "sunny"}), guardrail
|
||||
)
|
||||
|
||||
assert result["input"][0]["content"] == "[MASKED]"
|
||||
assert result["input"][2]["output"] == {"result": "sunny"}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tool_result_without_call_id_is_left_out_like_its_structured_row(self) -> None:
|
||||
guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test")
|
||||
call_item, output_type = _TOOL_TURNS[0]
|
||||
orphan: Final = {"type": output_type, "output": "SECRET orphan"}
|
||||
request: Final = {
|
||||
"model": "gpt-5.4-mini",
|
||||
"input": [
|
||||
{"role": "user", "content": "Weather?"},
|
||||
call_item,
|
||||
dict(orphan),
|
||||
{"type": output_type, "call_id": "call_1", "output": "SECRET forecast"},
|
||||
],
|
||||
}
|
||||
|
||||
result: Final = await OpenAIResponsesHandler().process_input_messages(request, guardrail)
|
||||
|
||||
assert guardrail.seen_texts == [["Weather?", "SECRET forecast"]]
|
||||
assert "SECRET orphan" not in guardrail.seen_message_contents[0]
|
||||
assert result["input"][2] == orphan
|
||||
assert result["input"][3]["output"] == "SECRET forecast [GUARDRAILED]"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_textless_output_sharing_a_call_id_keeps_the_text_output_in_structured_messages(self) -> None:
|
||||
guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test")
|
||||
call_item, output_type = _TOOL_TURNS[0]
|
||||
request: Final = _tool_result_request(call_item, output_type, "")
|
||||
request["input"].append({"type": output_type, "call_id": "call_1", "output": "SECRET forecast"})
|
||||
|
||||
result: Final = await OpenAIResponsesHandler().process_input_messages(request, guardrail)
|
||||
|
||||
assert guardrail.seen_texts == [["Weather?", "SECRET forecast"]]
|
||||
assert "SECRET forecast" in guardrail.seen_message_contents[0]
|
||||
assert "" not in guardrail.seen_message_contents[0]
|
||||
assert result["input"][3]["output"] == "SECRET forecast [GUARDRAILED]"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multi_part_outputs_sharing_a_call_id_each_keep_their_own_text_in_structured_messages(
|
||||
self,
|
||||
) -> None:
|
||||
guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test")
|
||||
call_item, output_type = _TOOL_TURNS[0]
|
||||
request: Final = _tool_result_request(
|
||||
call_item,
|
||||
output_type,
|
||||
[{"type": "input_text", "text": "first "}, {"type": "input_text", "text": "output"}],
|
||||
)
|
||||
request["input"].append(
|
||||
{
|
||||
"type": output_type,
|
||||
"call_id": "call_1",
|
||||
"output": [{"type": "input_text", "text": "second "}, {"type": "input_text", "text": "output"}],
|
||||
}
|
||||
)
|
||||
|
||||
result: Final = await OpenAIResponsesHandler().process_input_messages(request, guardrail)
|
||||
|
||||
assert guardrail.seen_texts == [["Weather?", "first ", "output", "second ", "output"]]
|
||||
assert guardrail.seen_message_contents[0][2:] == [
|
||||
[{"type": "text", "text": "first "}, {"type": "text", "text": "output"}],
|
||||
[{"type": "text", "text": "second "}, {"type": "text", "text": "output"}],
|
||||
]
|
||||
assert [part["text"] for part in result["input"][3]["output"]] == [
|
||||
"second [GUARDRAILED]",
|
||||
"output [GUARDRAILED]",
|
||||
]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"textless_output",
|
||||
[
|
||||
{"type": "function_call_output", "call_id": "call_1", "output": ""},
|
||||
{"type": "function_call_output", "call_id": "call_1", "output": None},
|
||||
{
|
||||
"type": "computer_call_output",
|
||||
"call_id": "call_1",
|
||||
"output": {"type": "computer_screenshot", "file_id": "file-screen"},
|
||||
},
|
||||
],
|
||||
ids=["empty_string", "null", "file_id_screenshot"],
|
||||
)
|
||||
async def test_multi_part_output_sharing_a_call_id_with_a_textless_output_lists_its_parts_in_structured_messages(
|
||||
self, textless_output: dict[str, object]
|
||||
) -> None:
|
||||
guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test")
|
||||
call_item, output_type = _TOOL_TURNS[0]
|
||||
request: Final = _tool_result_request(
|
||||
call_item,
|
||||
output_type,
|
||||
[{"type": "input_text", "text": "first "}, {"type": "input_text", "text": "output"}],
|
||||
)
|
||||
request["input"].append(copy.deepcopy(textless_output))
|
||||
|
||||
await OpenAIResponsesHandler().process_input_messages(request, guardrail)
|
||||
|
||||
assert guardrail.seen_texts == [["Weather?", "first ", "output"]]
|
||||
assert guardrail.seen_message_contents[0][2:] == [
|
||||
[{"type": "text", "text": "first "}, {"type": "text", "text": "output"}],
|
||||
]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tool_result_item_is_scanned_like_function_call_output(self) -> None:
|
||||
guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test")
|
||||
call_item, _ = _TOOL_TURNS[0]
|
||||
|
||||
result: Final = await OpenAIResponsesHandler().process_input_messages(
|
||||
_tool_result_request(call_item, "tool_result", "SECRET forecast"), guardrail
|
||||
)
|
||||
|
||||
assert guardrail.seen_texts == [["Weather?", "SECRET forecast"]]
|
||||
assert guardrail.seen_message_contents[0][2] == "SECRET forecast"
|
||||
assert result["input"][2]["output"] == "SECRET forecast [GUARDRAILED]"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_text_only_parts_the_guardrail_leaves_unchanged_keep_their_parts(self) -> None:
|
||||
call_item, output_type = _TOOL_TURNS[0]
|
||||
parts: Final = [{"type": "input_text", "text": "SECRET forecast: "}, {"type": "input_text", "text": "sunny"}]
|
||||
|
||||
result: Final = await OpenAIResponsesHandler().process_input_messages(
|
||||
_tool_result_request(call_item, output_type, copy.deepcopy(parts)), CustomGuardrail(guardrail_name="test")
|
||||
)
|
||||
|
||||
assert result["input"][2]["output"] == parts
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_text_parts_a_structured_guardrail_returns_unchanged_keep_their_parts(self) -> None:
|
||||
class StructuredEchoGuardrail(CustomGuardrail):
|
||||
async def apply_guardrail(
|
||||
self,
|
||||
inputs: GenericGuardrailAPIInputs,
|
||||
request_data: dict[str, object],
|
||||
input_type: Literal["request", "response"],
|
||||
logging_obj: LiteLLMLoggingObj | None = None,
|
||||
) -> GenericGuardrailAPIInputs:
|
||||
return {**inputs, "structured_messages": copy.deepcopy(inputs.get("structured_messages") or [])}
|
||||
|
||||
call_item, output_type = _TOOL_TURNS[0]
|
||||
parts: Final = [
|
||||
{"type": "output_text", "text": "SECRET forecast: ", "annotations": []},
|
||||
{"type": "output_text", "text": "sunny"},
|
||||
]
|
||||
|
||||
result: Final = await OpenAIResponsesHandler().process_input_messages(
|
||||
_tool_result_request(call_item, output_type, copy.deepcopy(parts)),
|
||||
StructuredEchoGuardrail(guardrail_name="t"),
|
||||
)
|
||||
|
||||
assert result["input"][2]["output"] == parts
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("output", "expected_texts"),
|
||||
[
|
||||
pytest.param("SECRET forecast", ["SECRET forecast"], id="string"),
|
||||
pytest.param([{"type": "input_text", "text": "SECRET forecast"}], ["SECRET forecast"], id="one_part"),
|
||||
pytest.param(
|
||||
[{"type": "input_text", "text": "SECRET "}, {"type": "output_text", "text": "forecast"}],
|
||||
["SECRET ", "forecast"],
|
||||
id="parts",
|
||||
),
|
||||
pytest.param(
|
||||
[
|
||||
{"type": "input_text", "text": "SECRET "},
|
||||
{"type": "input_image", "image_url": "https://example.com/map.png"},
|
||||
{"type": "input_text", "text": ""},
|
||||
{"type": "input_text", "text": "forecast"},
|
||||
],
|
||||
["SECRET ", "forecast"],
|
||||
id="parts_with_image",
|
||||
),
|
||||
pytest.param(
|
||||
[
|
||||
{"type": "input_text", "text": "SECRET forecast"},
|
||||
{"type": "summary_text", "text": "not tool output"},
|
||||
],
|
||||
["SECRET forecast"],
|
||||
id="non_text_part_with_text_field",
|
||||
),
|
||||
pytest.param(
|
||||
["SECRET forecast", {"type": "input_text", "text": "visible"}],
|
||||
["visible"],
|
||||
id="non_dict_part",
|
||||
),
|
||||
pytest.param([], ["[]"], id="empty"),
|
||||
pytest.param({"result": "SECRET forecast"}, ['{"result": "SECRET forecast"}'], id="dict"),
|
||||
pytest.param(
|
||||
[{"type": "input_image", "file_id": "file-map"}],
|
||||
['[{"type": "input_image", "file_id": "file-map"}]'],
|
||||
id="no_text_parts",
|
||||
),
|
||||
pytest.param({"type": "input_image", "image_url": "https://example.com/map.png"}, [], id="single_image"),
|
||||
],
|
||||
)
|
||||
async def test_tool_result_texts_match_the_structured_tool_message(
|
||||
self, output: object, expected_texts: list[str]
|
||||
) -> None:
|
||||
guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test")
|
||||
call_item, output_type = _TOOL_TURNS[0]
|
||||
|
||||
await OpenAIResponsesHandler().process_input_messages(
|
||||
_tool_result_request(call_item, output_type, output), guardrail
|
||||
)
|
||||
|
||||
tool_content: Final = guardrail.seen_message_contents[0][2]
|
||||
structured_texts: Final = (
|
||||
[tool_content]
|
||||
if isinstance(tool_content, str)
|
||||
else [part["text"] for part in tool_content if part.get("type") == "text"]
|
||||
)
|
||||
tool_texts: Final = guardrail.seen_texts[0][1:]
|
||||
assert tool_texts == expected_texts
|
||||
assert tool_texts == structured_texts
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(("call_item", "output_type"), _TOOL_TURNS, ids=["function", "custom"])
|
||||
async def test_skip_tool_message_keeps_the_tool_result_out_of_texts_and_rows(
|
||||
self, call_item: dict[str, str], output_type: str
|
||||
) -> None:
|
||||
guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test")
|
||||
guardrail.skip_tool_message_in_guardrail = True
|
||||
tool_result: Final = {"type": output_type, "call_id": "call_1", "output": "SECRET forecast"}
|
||||
|
||||
result: Final = await OpenAIResponsesHandler().process_input_messages(
|
||||
_tool_result_request(call_item, output_type, "SECRET forecast"), guardrail
|
||||
)
|
||||
|
||||
assert guardrail.seen_texts == [["Weather?"]]
|
||||
assert "SECRET forecast" not in guardrail.seen_message_contents[0]
|
||||
assert result["input"][0] == {"role": "user", "content": "Weather? [GUARDRAILED]"}
|
||||
assert result["input"][2] == tool_result
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(("call_item", "output_type"), _TOOL_TURNS, ids=["function", "custom"])
|
||||
async def test_scan_only_tool_results_sends_only_the_tool_result(
|
||||
self, call_item: dict[str, str], output_type: str
|
||||
) -> None:
|
||||
guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test")
|
||||
guardrail.scan_only_tool_results = True
|
||||
tool: Final = {"type": "function", "name": "get_weather", "parameters": {"type": "object", "properties": {}}}
|
||||
request: Final = {
|
||||
**_tool_result_request(call_item, output_type, "SECRET forecast"),
|
||||
"instructions": "Answer briefly",
|
||||
"tools": [tool],
|
||||
}
|
||||
|
||||
result: Final = await OpenAIResponsesHandler().process_input_messages(request, guardrail)
|
||||
|
||||
assert guardrail.seen_texts == [["SECRET forecast"]]
|
||||
assert guardrail.seen_message_contents == [["SECRET forecast"]]
|
||||
assert guardrail.seen_tools == [None]
|
||||
assert result["instructions"] == "Answer briefly"
|
||||
assert result["tools"] == [tool]
|
||||
assert result["input"] == [
|
||||
{"role": "user", "content": "Weather?"},
|
||||
call_item,
|
||||
{"type": output_type, "call_id": "call_1", "output": "SECRET forecast [GUARDRAILED]"},
|
||||
]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scan_only_tool_results_appends_tools_the_guardrail_adds(self) -> None:
|
||||
added_tool: Final = {
|
||||
"type": "function",
|
||||
"function": {"name": "retrieve", "description": "Fetch the full text", "parameters": {"type": "object"}},
|
||||
}
|
||||
renamed_original: Final = {
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "description": "rewritten", "parameters": {"type": "object"}},
|
||||
}
|
||||
|
||||
class ToolAddingGuardrail(RecordingMaskingGuardrail):
|
||||
async def apply_guardrail(
|
||||
self,
|
||||
inputs: GenericGuardrailAPIInputs,
|
||||
request_data: dict[str, object],
|
||||
input_type: Literal["request", "response"],
|
||||
logging_obj: LiteLLMLoggingObj | None = None,
|
||||
) -> GenericGuardrailAPIInputs:
|
||||
masked: Final = await super().apply_guardrail(inputs, request_data, input_type, logging_obj)
|
||||
return {**masked, "tools": [renamed_original, added_tool]}
|
||||
|
||||
guardrail: Final = ToolAddingGuardrail(guardrail_name="test")
|
||||
guardrail.scan_only_tool_results = True
|
||||
call_item, output_type = _TOOL_TURNS[0]
|
||||
tool: Final = {"type": "function", "name": "get_weather", "parameters": {"type": "object", "properties": {}}}
|
||||
request: Final = {**_tool_result_request(call_item, output_type, "SECRET forecast"), "tools": [tool]}
|
||||
|
||||
result: Final = await OpenAIResponsesHandler().process_input_messages(request, guardrail)
|
||||
|
||||
assert guardrail.seen_tools == [None]
|
||||
assert result["tools"][0] == tool
|
||||
assert [(t["type"], t["name"]) for t in result["tools"]] == [
|
||||
("function", "get_weather"),
|
||||
("function", "retrieve"),
|
||||
]
|
||||
assert result["tools"][1]["description"] == "Fetch the full text"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scan_only_tool_results_leaves_a_request_without_tool_results_unscanned(self) -> None:
|
||||
guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test")
|
||||
guardrail.scan_only_tool_results = True
|
||||
request: Final = {"model": "gpt-5.4-mini", "instructions": "Answer briefly", "input": "Weather?"}
|
||||
|
||||
result: Final = await OpenAIResponsesHandler().process_input_messages(dict(request), guardrail)
|
||||
|
||||
assert guardrail.seen_texts == []
|
||||
assert result == request
|
||||
|
||||
|
||||
class TestOpenAIResponsesHandlerOutputProcessing:
|
||||
"""Test output processing functionality"""
|
||||
|
||||
|
|
|
|||
|
|
@ -12,8 +12,11 @@ from pydantic import JsonValue, TypeAdapter
|
|||
import litellm
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig
|
||||
from litellm.llms.azure_ai.responses.transformation import AzureAIResponsesAPIConfig
|
||||
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
|
||||
from litellm.llms.hosted_vllm.responses.transformation import HostedVLLMResponsesAPIConfig
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
||||
from litellm.llms.xai.responses.transformation import XAIResponsesAPIConfig
|
||||
from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig
|
||||
from litellm.types.llms.openai import (
|
||||
ImageGenerationPartialImageEvent,
|
||||
|
|
@ -4277,3 +4280,116 @@ def test_dict_responses_input_filters_unset_reasoning_fields() -> None:
|
|||
function_call_item: Final = validated_input[2]
|
||||
assert function_call_item["type"] == "function_call"
|
||||
assert function_call_item["status"] == "completed"
|
||||
|
||||
|
||||
_COMPACT_CONFIGS: Final = (
|
||||
pytest.param(OpenAIResponsesAPIConfig(), "https://api.openai.com/v1/responses", id="openai"),
|
||||
pytest.param(
|
||||
AzureOpenAIResponsesAPIConfig(),
|
||||
"https://x.openai.azure.com/openai/responses?api-version=2025-04-01-preview",
|
||||
id="azure",
|
||||
),
|
||||
pytest.param(AzureAIResponsesAPIConfig(), "https://x.services.ai.azure.com/openai/v1/responses", id="azure_ai"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("config", "api_base"), _COMPACT_CONFIGS)
|
||||
def test_compact_request_omits_metadata_and_user_and_keeps_the_rest_without_drop_params(
|
||||
config: OpenAIResponsesAPIConfig, api_base: str
|
||||
) -> None:
|
||||
_, data = config.transform_compact_response_api_request(
|
||||
model="gpt-5.4-mini",
|
||||
input="hi",
|
||||
response_api_optional_request_params={
|
||||
"metadata": {"ticket": "qa-1"},
|
||||
"user": "end-user-1",
|
||||
"instructions": "be brief",
|
||||
"previous_response_id": "resp_1",
|
||||
"reasoning": {"effort": "low"},
|
||||
"temperature": 0.2,
|
||||
},
|
||||
api_base=api_base,
|
||||
litellm_params=GenericLiteLLMParams(drop_params=False),
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert {key: data[key] for key in data if key != "input"} == {
|
||||
"model": "gpt-5.4-mini",
|
||||
"instructions": "be brief",
|
||||
"previous_response_id": "resp_1",
|
||||
"reasoning": {"effort": "low"},
|
||||
"temperature": 0.2,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("config", "api_base"), _COMPACT_CONFIGS)
|
||||
def test_compact_request_drops_params_compact_rejects_under_drop_params(
|
||||
config: OpenAIResponsesAPIConfig, api_base: str
|
||||
) -> None:
|
||||
_, data = config.transform_compact_response_api_request(
|
||||
model="gpt-5.4-mini",
|
||||
input="hi",
|
||||
response_api_optional_request_params={
|
||||
"metadata": {"ticket": "qa-1"},
|
||||
"user": "end-user-1",
|
||||
"temperature": 0.2,
|
||||
"store": False,
|
||||
"truncation": "auto",
|
||||
"max_output_tokens": 100,
|
||||
"tool_choice": "auto",
|
||||
"reasoning": {"effort": "low"},
|
||||
"include": ["reasoning.encrypted_content"],
|
||||
"text": {"format": {"type": "text"}},
|
||||
},
|
||||
api_base=api_base,
|
||||
litellm_params=GenericLiteLLMParams(drop_params=True),
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert {key: data[key] for key in data if key != "input"} == {
|
||||
"model": "gpt-5.4-mini",
|
||||
"reasoning": {"effort": "low"},
|
||||
"include": ["reasoning.encrypted_content"],
|
||||
"text": {"format": {"type": "text"}},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("config", "api_base"),
|
||||
(
|
||||
pytest.param(XAIResponsesAPIConfig(), "https://api.x.ai/v1/responses", id="xai"),
|
||||
pytest.param(HostedVLLMResponsesAPIConfig(), "http://vllm.example.com/v1/responses", id="hosted_vllm"),
|
||||
),
|
||||
)
|
||||
def test_compact_request_forwards_every_param_for_providers_not_known_to_reject_them(
|
||||
config: OpenAIResponsesAPIConfig, api_base: str
|
||||
) -> None:
|
||||
_, data = config.transform_compact_response_api_request(
|
||||
model="some-model",
|
||||
input="hi",
|
||||
response_api_optional_request_params={"metadata": {"ticket": "qa-1"}, "user": "end-user-1", "temperature": 0.2},
|
||||
api_base=api_base,
|
||||
litellm_params=GenericLiteLLMParams(drop_params=True),
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert {key: data[key] for key in data if key != "input"} == {
|
||||
"model": "some-model",
|
||||
"metadata": {"ticket": "qa-1"},
|
||||
"user": "end-user-1",
|
||||
"temperature": 0.2,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("config", "api_base"), _COMPACT_CONFIGS)
|
||||
def test_compact_request_keeps_stream_under_drop_params(config: OpenAIResponsesAPIConfig, api_base: str) -> None:
|
||||
_, data = config.transform_compact_response_api_request(
|
||||
model="gpt-5.4-mini",
|
||||
input="hi",
|
||||
response_api_optional_request_params={"stream": True, "stream_options": {"include_obfuscation": False}},
|
||||
api_base=api_base,
|
||||
litellm_params=GenericLiteLLMParams(drop_params=True),
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert {key: data[key] for key in data if key != "input"} == {"model": "gpt-5.4-mini", "stream": True}
|
||||
|
|
|
|||
402
tests/unit/llms/openai_like/test_streamlake_provider.py
Normal file
402
tests/unit/llms/openai_like/test_streamlake_provider.py
Normal file
|
|
@ -0,0 +1,402 @@
|
|||
import json
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.llms.openai_like.dynamic_config import create_config_class
|
||||
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
|
||||
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
|
||||
from litellm.types.utils import ModelResponse, Usage
|
||||
|
||||
STREAMLAKE_BASE_URL: Final = "https://vanchin.streamlake.ai/api/gateway/v1/endpoints"
|
||||
STREAMLAKE_CHAT_URL: Final = f"{STREAMLAKE_BASE_URL}/chat/completions"
|
||||
|
||||
REPO_ROOT: Final = Path(__file__).parents[4]
|
||||
|
||||
|
||||
class TestStreamlakeProviderConfig:
|
||||
def test_streamlake_provider_resolution(self) -> None:
|
||||
model, provider, api_key, api_base = get_llm_provider(
|
||||
model="streamlake/ep-test-endpoint",
|
||||
custom_llm_provider=None,
|
||||
api_base=None,
|
||||
api_key="sk-test",
|
||||
)
|
||||
|
||||
assert model == "ep-test-endpoint"
|
||||
assert provider == "streamlake"
|
||||
assert api_base == STREAMLAKE_BASE_URL
|
||||
assert api_key == "sk-test"
|
||||
|
||||
def test_streamlake_api_base_override(self) -> None:
|
||||
model, provider, api_key, api_base = get_llm_provider(
|
||||
model="streamlake/ep-test-endpoint",
|
||||
custom_llm_provider=None,
|
||||
api_base="https://custom.streamlake.example/v1",
|
||||
api_key="sk-test",
|
||||
)
|
||||
|
||||
assert model == "ep-test-endpoint"
|
||||
assert provider == "streamlake"
|
||||
assert api_base == "https://custom.streamlake.example/v1"
|
||||
assert api_key == "sk-test"
|
||||
|
||||
def test_streamlake_url_autodetection(self) -> None:
|
||||
_, provider, _, api_base = get_llm_provider(
|
||||
model="some-public-model",
|
||||
custom_llm_provider=None,
|
||||
api_base=STREAMLAKE_BASE_URL,
|
||||
api_key=None,
|
||||
)
|
||||
assert provider == "streamlake"
|
||||
assert api_base == STREAMLAKE_BASE_URL
|
||||
|
||||
def test_streamlake_api_key_resolved_from_env(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("STREAMLAKE_API_KEY", "sk-env-key")
|
||||
|
||||
_, provider, api_key, _ = get_llm_provider(
|
||||
model="streamlake/ep-test-endpoint",
|
||||
custom_llm_provider=None,
|
||||
api_base=None,
|
||||
api_key=None,
|
||||
)
|
||||
assert provider == "streamlake"
|
||||
assert api_key == "sk-env-key"
|
||||
|
||||
def test_streamlake_validate_environment_sets_bearer(self) -> None:
|
||||
provider: Final = JSONProviderRegistry.get("streamlake")
|
||||
assert provider is not None
|
||||
config: Final = create_config_class(provider)()
|
||||
|
||||
headers: Final = config.validate_environment(
|
||||
headers={},
|
||||
model="ep-test-endpoint",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
api_key="sk-resolved-key",
|
||||
api_base=None,
|
||||
)
|
||||
assert headers["Authorization"] == "Bearer sk-resolved-key"
|
||||
|
||||
def test_streamlake_complete_url_appends_chat_completions(self) -> None:
|
||||
provider: Final = JSONProviderRegistry.get("streamlake")
|
||||
assert provider is not None
|
||||
config: Final = create_config_class(provider)()
|
||||
|
||||
url: Final = config.get_complete_url(
|
||||
api_base=None,
|
||||
api_key="sk-test",
|
||||
model="ep-test-endpoint",
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
)
|
||||
assert url == STREAMLAKE_CHAT_URL
|
||||
|
||||
@pytest.mark.parametrize("cap_param", ("max_tokens", "max_completion_tokens"))
|
||||
def test_streamlake_token_cap_sent_as_max_completion_tokens(self, cap_param: str) -> None:
|
||||
provider: Final = JSONProviderRegistry.get("streamlake")
|
||||
assert provider is not None
|
||||
config: Final = create_config_class(provider)()
|
||||
|
||||
optional_params: Final = config.map_openai_params(
|
||||
non_default_params={cap_param: 1024},
|
||||
optional_params={},
|
||||
model="ep-test-endpoint",
|
||||
drop_params=False,
|
||||
)
|
||||
assert optional_params == {"max_completion_tokens": 1024}
|
||||
|
||||
def test_streamlake_chat_params_pass_through(self) -> None:
|
||||
provider: Final = JSONProviderRegistry.get("streamlake")
|
||||
assert provider is not None
|
||||
config: Final = create_config_class(provider)()
|
||||
|
||||
optional_params: Final = config.map_openai_params(
|
||||
non_default_params={"temperature": 0.2, "top_p": 0.9, "stream": True},
|
||||
optional_params={},
|
||||
model="ep-test-endpoint",
|
||||
drop_params=False,
|
||||
)
|
||||
assert optional_params == {"temperature": 0.2, "top_p": 0.9, "stream": True}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"is_async, api_base",
|
||||
(
|
||||
(False, None),
|
||||
(False, "https://custom.streamlake.example/v1"),
|
||||
(False, "https://custom.streamlake.example/v1/"),
|
||||
(True, None),
|
||||
(True, "https://custom.streamlake.example/v1"),
|
||||
(True, "https://custom.streamlake.example/v1/"),
|
||||
),
|
||||
ids=(
|
||||
"sync-default",
|
||||
"sync-override",
|
||||
"sync-override-trailing-slash",
|
||||
"async-default",
|
||||
"async-override",
|
||||
"async-override-trailing-slash",
|
||||
),
|
||||
)
|
||||
async def test_streamlake_completion_uses_openai_compatible_http(
|
||||
is_async: bool, api_base: str | None, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
response_body: Final[dict[str, object]] = {
|
||||
"id": "chatcmpl-streamlake-test",
|
||||
"object": "chat.completion",
|
||||
"created": 1710000000,
|
||||
"model": "ep-test-endpoint",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "StreamLake response"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5},
|
||||
}
|
||||
expected_url: Final[str] = f"{(api_base or STREAMLAKE_BASE_URL).rstrip('/')}/chat/completions"
|
||||
expected_request_body: Final[dict[str, object]] = {
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"model": "ep-test-endpoint",
|
||||
"max_completion_tokens": 512,
|
||||
}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
assert request.method == "POST"
|
||||
assert str(request.url) == expected_url
|
||||
assert request.headers["authorization"] == "Bearer sk-streamlake-test"
|
||||
assert request.headers["content-type"] == "application/json"
|
||||
assert json.loads(request.content) == expected_request_body
|
||||
return httpx.Response(200, json=response_body, request=request)
|
||||
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
with respx.mock(assert_all_called=True) as respx_mock:
|
||||
route: Final = respx_mock.post(expected_url).mock(side_effect=handler)
|
||||
if is_async:
|
||||
response: Final = await litellm.acompletion(
|
||||
model="streamlake/ep-test-endpoint",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
api_key="sk-streamlake-test",
|
||||
api_base=api_base,
|
||||
max_tokens=512,
|
||||
)
|
||||
else:
|
||||
response: Final = litellm.completion(
|
||||
model="streamlake/ep-test-endpoint",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
api_key="sk-streamlake-test",
|
||||
api_base=api_base,
|
||||
max_tokens=512,
|
||||
)
|
||||
|
||||
assert route.call_count == 1
|
||||
assert len(response.choices) == 1
|
||||
assert response.choices[0].message.role == "assistant"
|
||||
assert response.choices[0].message.content == "StreamLake response"
|
||||
assert response.usage is not None
|
||||
assert response.usage.prompt_tokens == 3
|
||||
assert response.usage.completion_tokens == 2
|
||||
assert response.usage.total_tokens == 5
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
def test_endpoint_id_call_forwards_tools_and_reasoning_effort_to_streamlake(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
tools: Final = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"parameters": {"type": "object", "properties": {"city": {"type": "string"}}},
|
||||
},
|
||||
}
|
||||
]
|
||||
response_body: Final[dict[str, object]] = {
|
||||
"id": "chatcmpl-streamlake-tools",
|
||||
"object": "chat.completion",
|
||||
"created": 1710000000,
|
||||
"model": "glm-5.3",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4},
|
||||
}
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
with respx.mock(assert_all_called=True) as respx_mock:
|
||||
route: Final = respx_mock.post(STREAMLAKE_CHAT_URL).mock(return_value=httpx.Response(200, json=response_body))
|
||||
litellm.completion(
|
||||
model="streamlake/ep-test-endpoint",
|
||||
messages=[{"role": "user", "content": "weather in Paris?"}],
|
||||
api_key="sk-streamlake-test",
|
||||
tools=tools,
|
||||
tool_choice="auto",
|
||||
reasoning_effort="low",
|
||||
)
|
||||
|
||||
sent: Final = json.loads(route.calls.last.request.content)
|
||||
assert sent["tools"] == tools
|
||||
assert sent["tool_choice"] == "auto"
|
||||
assert sent["reasoning_effort"] == "low"
|
||||
|
||||
|
||||
def _streamlake_keys(cost_map: dict[str, object]) -> tuple[str, ...]:
|
||||
return tuple(key for key in cost_map if key.startswith("streamlake/"))
|
||||
|
||||
|
||||
def test_root_and_backup_cost_maps_carry_the_same_streamlake_rows() -> None:
|
||||
root: Final = json.loads((REPO_ROOT / "model_prices_and_context_window.json").read_text())
|
||||
backup: Final = json.loads((REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json").read_text())
|
||||
|
||||
assert _streamlake_keys(root)
|
||||
assert {key: root[key] for key in _streamlake_keys(root)} == {key: backup[key] for key in _streamlake_keys(backup)}
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
def test_endpoint_id_call_is_priced_from_the_lowercase_model_streamlake_returns() -> None:
|
||||
keys: Final = _streamlake_keys(litellm.model_cost)
|
||||
assert keys
|
||||
|
||||
for key in keys:
|
||||
info: Final = litellm.get_model_info(model=key)
|
||||
response: Final = ModelResponse(
|
||||
model=key.split("/", 1)[1].lower(),
|
||||
usage=Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500),
|
||||
)
|
||||
cost: Final = litellm.completion_cost(
|
||||
completion_response=response,
|
||||
model="streamlake/ep-test-endpoint",
|
||||
custom_llm_provider="streamlake",
|
||||
)
|
||||
expected: Final = 1000 * info["input_cost_per_token"] + 500 * info["output_cost_per_token"]
|
||||
assert cost == pytest.approx(expected), key
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
def test_named_model_params_follow_the_row_and_fall_back_to_the_provider_default() -> None:
|
||||
provider: Final = JSONProviderRegistry.get("streamlake")
|
||||
assert provider is not None
|
||||
config: Final = create_config_class(provider)()
|
||||
keys: Final = _streamlake_keys(litellm.model_cost)
|
||||
assert keys
|
||||
|
||||
for key in keys:
|
||||
row: Final = litellm.model_cost[key]
|
||||
supported: Final = config.get_supported_openai_params(model=key.split("/", 1)[1].lower())
|
||||
assert ("tools" in supported) is (row.get("supports_function_calling") is not False), key
|
||||
assert ("reasoning_effort" in supported) is (row.get("supports_reasoning") is not False), key
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
def test_streamlake_wildcard_expands_to_every_streamlake_row() -> None:
|
||||
expanded: Final = get_known_models_from_wildcard("streamlake/*")
|
||||
assert set(expanded) == set(_streamlake_keys(litellm.model_cost))
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
def test_cache_control_marks_reach_streamlake(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
system_block: Final = {"type": "text", "text": "You are terse.", "cache_control": {"type": "ephemeral"}}
|
||||
tool: Final = {
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}},
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
response_body: Final[dict[str, object]] = {
|
||||
"id": "chatcmpl-streamlake-cache",
|
||||
"object": "chat.completion",
|
||||
"created": 1710000000,
|
||||
"model": "qwen3.8-27b",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4},
|
||||
}
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
with respx.mock(assert_all_called=True) as respx_mock:
|
||||
route: Final = respx_mock.post(STREAMLAKE_CHAT_URL).mock(return_value=httpx.Response(200, json=response_body))
|
||||
litellm.completion(
|
||||
model="streamlake/qwen3.8-27b",
|
||||
messages=[{"role": "system", "content": [system_block]}, {"role": "user", "content": "hi"}],
|
||||
api_key="sk-streamlake-test",
|
||||
tools=[tool],
|
||||
)
|
||||
|
||||
sent: Final = json.loads(route.calls.last.request.content)
|
||||
assert sent["messages"][0]["content"] == [system_block]
|
||||
assert sent["tools"][0]["cache_control"] == {"type": "ephemeral"}
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
@pytest.mark.parametrize("model", ("kimi-k2.6", "qwen3.6-27b", "qwen3.6-35b-a3b"))
|
||||
def test_reasoning_effort_is_dropped_for_models_streamlake_lists_without_reasoning(
|
||||
model: str, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
# StreamLake's Models Overview (https://www.streamlake.ai/document/DOC/mh1gbfvrdn6hpbzxixv, read 2026-10-28)
|
||||
# gives these models no Reasoning tag, and kimi-k2.6 answers 400 to reasoning_effort under an 8193 token cap
|
||||
response_body: Final[dict[str, object]] = {
|
||||
"id": "chatcmpl-streamlake-reasoning",
|
||||
"object": "chat.completion",
|
||||
"created": 1710000000,
|
||||
"model": model,
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4},
|
||||
}
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
with respx.mock(assert_all_called=True) as respx_mock:
|
||||
route: Final = respx_mock.post(STREAMLAKE_CHAT_URL).mock(return_value=httpx.Response(200, json=response_body))
|
||||
litellm.completion(
|
||||
model=f"streamlake/{model}",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
api_key="sk-streamlake-test",
|
||||
reasoning_effort="low",
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
assert "reasoning_effort" not in json.loads(route.calls.last.request.content)
|
||||
with pytest.raises(litellm.UnsupportedParamsError):
|
||||
litellm.completion(
|
||||
model=f"streamlake/{model}",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
api_key="sk-streamlake-test",
|
||||
reasoning_effort="low",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
def test_streamlake_rows_with_a_cache_read_price_declare_prompt_caching() -> None:
|
||||
cached_rows: Final = tuple(
|
||||
key for key in _streamlake_keys(litellm.model_cost) if "cache_read_input_token_cost" in litellm.model_cost[key]
|
||||
)
|
||||
assert cached_rows
|
||||
assert [key for key in cached_rows if litellm.model_cost[key].get("supports_prompt_caching") is not True] == []
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
@pytest.mark.parametrize("model", ("qwen3.8-27b", "qwen3.8-2.4t-a95b"))
|
||||
@pytest.mark.parametrize("cache_type", ("ephemeral", None))
|
||||
def test_qwen3_8_explicit_cache_hits_and_prefix_hits_use_their_own_read_price(
|
||||
model: str, cache_type: str | None
|
||||
) -> None:
|
||||
# StreamLake's price page (https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce, read 2026-10-28) lists
|
||||
# separate PrefixCache and Cache Read prices for Qwen3.8, and its usage tags only explicit hits with cache_type
|
||||
rate_key: Final = "explicit_cache_read_input_token_cost" if cache_type else "cache_read_input_token_cost"
|
||||
row: Final = litellm.model_cost[f"streamlake/{model}"]
|
||||
details: Final = {"cached_tokens": 4000} | ({"cache_type": cache_type} if cache_type else {})
|
||||
response: Final = ModelResponse(
|
||||
model=model,
|
||||
usage=Usage(prompt_tokens=5000, completion_tokens=100, total_tokens=5100, prompt_tokens_details=details),
|
||||
)
|
||||
|
||||
cost: Final = litellm.completion_cost(
|
||||
completion_response=response, model=f"streamlake/{model}", custom_llm_provider="streamlake"
|
||||
)
|
||||
|
||||
expected: Final = 4000 * row[rate_key] + 1000 * row["input_cost_per_token"] + 100 * row["output_cost_per_token"]
|
||||
assert cost == pytest.approx(expected)
|
||||
|
|
@ -12,7 +12,7 @@ load_dotenv()
|
|||
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
|
||||
from copy import deepcopy
|
||||
from types import SimpleNamespace
|
||||
from typing import Final
|
||||
from typing import Final, Literal
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest, litellm
|
||||
|
|
@ -721,6 +721,51 @@ async def test_can_key_call_model_wildcard_access(key_models, model, expect_to_w
|
|||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def shipped_routing_rules() -> Iterator[None]:
|
||||
from litellm.litellm_core_utils.fallback_generalizations import (
|
||||
get_fallback_generalization_rules,
|
||||
set_fallback_generalizations,
|
||||
)
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
|
||||
previous_rules: Final = list(get_fallback_generalization_rules())
|
||||
set_fallback_generalizations(GetModelCostMap.load_local_model_cost_map()["fallback_generalizations"]["rules"])
|
||||
yield
|
||||
set_fallback_generalizations(previous_rules)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("object_type", ["team", "project"])
|
||||
@pytest.mark.parametrize(
|
||||
"allowed_models, model, expect_to_work",
|
||||
[
|
||||
(["typesafe/*"], "jev-latest", True),
|
||||
(["typesafe/*"], "jev-1.14.0", True),
|
||||
(["jev-latest"], "jev-latest", True),
|
||||
(["openai/*"], "jev-latest", False),
|
||||
(["typesafe/*"], "foo/jev-latest", False),
|
||||
(["typesafe/*"], "jev-internal-router", False),
|
||||
(["typesafe/*"], "jev-2.0-internal", False),
|
||||
(["typesafe/*"], "jev-1.0", False),
|
||||
],
|
||||
)
|
||||
def test_typesafe_provider_wildcard_covers_bare_jev_model(
|
||||
shipped_routing_rules: None,
|
||||
object_type: Literal["team", "project"],
|
||||
allowed_models: list[str],
|
||||
model: str,
|
||||
expect_to_work: bool,
|
||||
) -> None:
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy.auth.auth_checks import can_object_call_model
|
||||
|
||||
if expect_to_work:
|
||||
assert can_object_call_model(model=model, llm_router=None, models=allowed_models, object_type=object_type)
|
||||
return
|
||||
with pytest.raises(ProxyException):
|
||||
can_object_call_model(model=model, llm_router=None, models=allowed_models, object_type=object_type)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"key_models, model, expect_to_work",
|
||||
[
|
||||
|
|
|
|||
|
|
@ -277,7 +277,7 @@ def _free_port() -> int:
|
|||
|
||||
|
||||
def _fake_pooler(
|
||||
tmp_path: Path,
|
||||
directory: Path,
|
||||
port: int,
|
||||
exit_immediately: bool = False,
|
||||
port_file: Path | None = None,
|
||||
|
|
@ -288,13 +288,13 @@ def _fake_pooler(
|
|||
"""An executable that listens like PgBouncer: on the TCP port first, then on ``.s.PGSQL.<port>`` in the socket dir.
|
||||
|
||||
Port and socket dir come from the ini it is given, else from ``port`` and
|
||||
``tmp_path``. With ``port_file`` each start reads the port from that file
|
||||
``directory``. With ``port_file`` each start reads the port from that file
|
||||
instead. ``bind_delay_seconds`` holds the bind back, like a slow start.
|
||||
``--version`` prints ``version_banner``. With ``auth_log`` it appends the
|
||||
``auth_file`` it reads at startup and on every SIGHUP, one line per read,
|
||||
like PgBouncer loading its credentials.
|
||||
"""
|
||||
script: Final = tmp_path / "fake-pgbouncer"
|
||||
script: Final = directory / "fake-pgbouncer"
|
||||
script.write_text(
|
||||
textwrap.dedent(
|
||||
f"""\
|
||||
|
|
@ -316,7 +316,7 @@ def _fake_pooler(
|
|||
port = ini.getint("pgbouncer", "listen_port", fallback={port})
|
||||
if not {port_file is None!r}:
|
||||
port = int(pathlib.Path({str(port_file)!r}).read_text())
|
||||
socket_dir = ini.get("pgbouncer", "unix_socket_dir", fallback={str(tmp_path)!r})
|
||||
socket_dir = ini.get("pgbouncer", "unix_socket_dir", fallback={str(directory)!r})
|
||||
socket_path = f"{{socket_dir}}/.s.PGSQL.{{port}}"
|
||||
time.sleep({bind_delay_seconds!r})
|
||||
listener = socket.socket()
|
||||
|
|
@ -357,10 +357,10 @@ def _wait_until(condition: Callable[[], bool], timeout_seconds: float = 5.0) ->
|
|||
|
||||
|
||||
class TestPgBouncerProcess:
|
||||
def test_start_waits_for_the_listener_and_stop_ends_it(self, tmp_path: Path):
|
||||
def test_start_waits_for_the_listener_and_stop_ends_it(self, socket_dir: Path):
|
||||
port: Final = _free_port()
|
||||
pooler: Final = PgBouncerProcess(
|
||||
argv=(str(_fake_pooler(tmp_path, port)),), port=port, socket_path=unix_socket_path(tmp_path, port)
|
||||
argv=(str(_fake_pooler(socket_dir, port)),), port=port, socket_path=unix_socket_path(socket_dir, port)
|
||||
)
|
||||
assert pooler.start() is None
|
||||
assert _listening(port)
|
||||
|
|
@ -371,12 +371,12 @@ class TestPgBouncerProcess:
|
|||
with pytest.raises(ProcessLookupError):
|
||||
os.kill(pid, 0)
|
||||
|
||||
def test_a_crashed_pooler_is_restarted_with_a_new_pid(self, tmp_path: Path):
|
||||
def test_a_crashed_pooler_is_restarted_with_a_new_pid(self, socket_dir: Path):
|
||||
port: Final = _free_port()
|
||||
pooler: Final = PgBouncerProcess(
|
||||
argv=(str(_fake_pooler(tmp_path, port)),),
|
||||
argv=(str(_fake_pooler(socket_dir, port)),),
|
||||
port=port,
|
||||
socket_path=unix_socket_path(tmp_path, port),
|
||||
socket_path=unix_socket_path(socket_dir, port),
|
||||
restart_delay_seconds=0.1,
|
||||
)
|
||||
assert pooler.start() is None
|
||||
|
|
@ -388,17 +388,17 @@ class TestPgBouncerProcess:
|
|||
assert _wait_until(lambda: not _listening(port))
|
||||
|
||||
def test_a_failed_restart_is_retried_until_the_pooler_is_back(
|
||||
self, tmp_path: Path, caplog: pytest.LogCaptureFixture
|
||||
self, socket_dir: Path, caplog: pytest.LogCaptureFixture
|
||||
):
|
||||
port: Final = _free_port()
|
||||
script: Final = _fake_pooler(tmp_path, port)
|
||||
script: Final = _fake_pooler(socket_dir, port)
|
||||
pooler: Final = PgBouncerProcess(
|
||||
argv=(str(script),), port=port, socket_path=unix_socket_path(tmp_path, port), restart_delay_seconds=0.1
|
||||
argv=(str(script),), port=port, socket_path=unix_socket_path(socket_dir, port), restart_delay_seconds=0.1
|
||||
)
|
||||
assert pooler.start() is None
|
||||
first_pid: Final = pooler.pid
|
||||
assert first_pid is not None
|
||||
hidden: Final = script.rename(tmp_path / "hidden")
|
||||
hidden: Final = script.rename(socket_dir / "hidden")
|
||||
with caplog.at_level(logging.ERROR, logger=verbose_proxy_logger.name):
|
||||
os.kill(first_pid, signal.SIGKILL)
|
||||
assert _wait_until(lambda: any("could not be restarted" in record.message for record in caplog.records))
|
||||
|
|
@ -408,15 +408,17 @@ class TestPgBouncerProcess:
|
|||
pooler.stop()
|
||||
assert _wait_until(lambda: not _listening(port))
|
||||
|
||||
def test_a_replacement_that_never_listens_is_replaced_again(self, tmp_path: Path, caplog: pytest.LogCaptureFixture):
|
||||
def test_a_replacement_that_never_listens_is_replaced_again(
|
||||
self, socket_dir: Path, caplog: pytest.LogCaptureFixture
|
||||
):
|
||||
port: Final = _free_port()
|
||||
port_file: Final = tmp_path / "port"
|
||||
port_file: Final = socket_dir / "port"
|
||||
port_file.write_text(str(port))
|
||||
script: Final = _fake_pooler(tmp_path, port, port_file=port_file)
|
||||
script: Final = _fake_pooler(socket_dir, port, port_file=port_file)
|
||||
pooler: Final = PgBouncerProcess(
|
||||
argv=(str(script),),
|
||||
port=port,
|
||||
socket_path=unix_socket_path(tmp_path, port),
|
||||
socket_path=unix_socket_path(socket_dir, port),
|
||||
restart_delay_seconds=0.1,
|
||||
ready_timeout_seconds=2.0,
|
||||
)
|
||||
|
|
@ -435,12 +437,12 @@ class TestPgBouncerProcess:
|
|||
pooler.stop()
|
||||
assert _wait_until(lambda: not _listening(port))
|
||||
|
||||
def test_stopping_during_the_restart_delay_leaves_no_pooler_behind(self, tmp_path: Path):
|
||||
def test_stopping_during_the_restart_delay_leaves_no_pooler_behind(self, socket_dir: Path):
|
||||
port: Final = _free_port()
|
||||
pooler: Final = PgBouncerProcess(
|
||||
argv=(str(_fake_pooler(tmp_path, port)),),
|
||||
argv=(str(_fake_pooler(socket_dir, port)),),
|
||||
port=port,
|
||||
socket_path=unix_socket_path(tmp_path, port),
|
||||
socket_path=unix_socket_path(socket_dir, port),
|
||||
restart_delay_seconds=0.3,
|
||||
)
|
||||
assert pooler.start() is None
|
||||
|
|
@ -453,12 +455,12 @@ class TestPgBouncerProcess:
|
|||
assert not _listening(port)
|
||||
assert pooler.pid == first_pid
|
||||
|
||||
def test_a_stopped_pooler_is_not_restarted(self, tmp_path: Path, caplog: pytest.LogCaptureFixture):
|
||||
def test_a_stopped_pooler_is_not_restarted(self, socket_dir: Path, caplog: pytest.LogCaptureFixture):
|
||||
port: Final = _free_port()
|
||||
pooler: Final = PgBouncerProcess(
|
||||
argv=(str(_fake_pooler(tmp_path, port)),),
|
||||
argv=(str(_fake_pooler(socket_dir, port)),),
|
||||
port=port,
|
||||
socket_path=unix_socket_path(tmp_path, port),
|
||||
socket_path=unix_socket_path(socket_dir, port),
|
||||
restart_delay_seconds=0.1,
|
||||
)
|
||||
assert pooler.start() is None
|
||||
|
|
@ -468,31 +470,31 @@ class TestPgBouncerProcess:
|
|||
assert not _listening(port)
|
||||
assert caplog.records == []
|
||||
|
||||
def test_a_pooler_that_exits_during_startup_is_reported(self, tmp_path: Path):
|
||||
def test_a_pooler_that_exits_during_startup_is_reported(self, socket_dir: Path):
|
||||
port: Final = _free_port()
|
||||
pooler: Final = PgBouncerProcess(
|
||||
argv=(str(_fake_pooler(tmp_path, port, exit_immediately=True)),),
|
||||
argv=(str(_fake_pooler(socket_dir, port, exit_immediately=True)),),
|
||||
port=port,
|
||||
socket_path=unix_socket_path(tmp_path, port),
|
||||
socket_path=unix_socket_path(socket_dir, port),
|
||||
)
|
||||
outcome: Final = pooler.start()
|
||||
assert isinstance(outcome, PgBouncerError)
|
||||
assert "status 3" in outcome.reason
|
||||
|
||||
def test_a_missing_binary_is_reported(self, tmp_path: Path):
|
||||
def test_a_missing_binary_is_reported(self, socket_dir: Path):
|
||||
outcome: Final = PgBouncerProcess(
|
||||
argv=("/nonexistent/pgbouncer",), port=_free_port(), socket_path=tmp_path / "sock"
|
||||
argv=("/nonexistent/pgbouncer",), port=_free_port(), socket_path=socket_dir / "sock"
|
||||
).start()
|
||||
assert isinstance(outcome, PgBouncerError)
|
||||
assert "/nonexistent/pgbouncer" in outcome.reason
|
||||
|
||||
def test_a_port_owned_by_someone_else_is_refused_before_spawning(self, tmp_path: Path):
|
||||
def test_a_port_owned_by_someone_else_is_refused_before_spawning(self, socket_dir: Path):
|
||||
with socket.socket() as squatter:
|
||||
squatter.bind(("127.0.0.1", 0))
|
||||
squatter.listen()
|
||||
port: Final = _bound_port(squatter)
|
||||
pooler: Final = PgBouncerProcess(
|
||||
argv=(str(_fake_pooler(tmp_path, port)),), port=port, socket_path=unix_socket_path(tmp_path, port)
|
||||
argv=(str(_fake_pooler(socket_dir, port)),), port=port, socket_path=unix_socket_path(socket_dir, port)
|
||||
)
|
||||
outcome: Final = pooler.start()
|
||||
assert isinstance(outcome, PgBouncerError)
|
||||
|
|
@ -500,13 +502,13 @@ class TestPgBouncerProcess:
|
|||
assert pooler.pid is None
|
||||
|
||||
def test_a_replacement_waits_until_a_squatter_leaves_the_port(
|
||||
self, tmp_path: Path, caplog: pytest.LogCaptureFixture
|
||||
self, socket_dir: Path, caplog: pytest.LogCaptureFixture
|
||||
):
|
||||
port: Final = _free_port()
|
||||
pooler: Final = PgBouncerProcess(
|
||||
argv=(str(_fake_pooler(tmp_path, port)),),
|
||||
argv=(str(_fake_pooler(socket_dir, port)),),
|
||||
port=port,
|
||||
socket_path=unix_socket_path(tmp_path, port),
|
||||
socket_path=unix_socket_path(socket_dir, port),
|
||||
restart_delay_seconds=0.5,
|
||||
)
|
||||
assert pooler.start() is None
|
||||
|
|
@ -524,12 +526,12 @@ class TestPgBouncerProcess:
|
|||
pooler.stop()
|
||||
assert _wait_until(lambda: not _listening(port))
|
||||
|
||||
def test_a_listener_that_grabs_the_port_after_the_spawn_is_not_taken_for_the_pooler(self, tmp_path: Path):
|
||||
def test_a_listener_that_grabs_the_port_after_the_spawn_is_not_taken_for_the_pooler(self, socket_dir: Path):
|
||||
port: Final = _free_port()
|
||||
pooler: Final = PgBouncerProcess(
|
||||
argv=(str(_fake_pooler(tmp_path, port, bind_delay_seconds=0.5)),),
|
||||
argv=(str(_fake_pooler(socket_dir, port, bind_delay_seconds=0.5)),),
|
||||
port=port,
|
||||
socket_path=unix_socket_path(tmp_path, port),
|
||||
socket_path=unix_socket_path(socket_dir, port),
|
||||
ready_timeout_seconds=3.0,
|
||||
)
|
||||
with socket.socket() as squatter, ThreadPoolExecutor(max_workers=1) as starter:
|
||||
|
|
@ -542,12 +544,12 @@ class TestPgBouncerProcess:
|
|||
assert isinstance(outcome, PgBouncerError)
|
||||
assert "exited with status 1" in outcome.reason
|
||||
|
||||
def test_a_port_served_by_a_stranger_while_the_pooler_is_still_starting_is_reported(self, tmp_path: Path):
|
||||
def test_a_port_served_by_a_stranger_while_the_pooler_is_still_starting_is_reported(self, socket_dir: Path):
|
||||
port: Final = _free_port()
|
||||
pooler: Final = PgBouncerProcess(
|
||||
argv=(str(_fake_pooler(tmp_path, port, bind_delay_seconds=30.0)),),
|
||||
argv=(str(_fake_pooler(socket_dir, port, bind_delay_seconds=30.0)),),
|
||||
port=port,
|
||||
socket_path=unix_socket_path(tmp_path, port),
|
||||
socket_path=unix_socket_path(socket_dir, port),
|
||||
ready_timeout_seconds=0.5,
|
||||
)
|
||||
with socket.socket() as squatter, ThreadPoolExecutor(max_workers=1) as starter:
|
||||
|
|
@ -564,12 +566,12 @@ class TestPgBouncerProcess:
|
|||
with pytest.raises(ProcessLookupError):
|
||||
os.kill(pid, 0)
|
||||
|
||||
def test_a_pooler_that_never_listens_times_out(self, tmp_path: Path):
|
||||
def test_a_pooler_that_never_listens_times_out(self, socket_dir: Path):
|
||||
port: Final = _free_port()
|
||||
pooler: Final = PgBouncerProcess(
|
||||
argv=(str(_fake_pooler(tmp_path, _free_port())),),
|
||||
argv=(str(_fake_pooler(socket_dir, _free_port())),),
|
||||
port=port,
|
||||
socket_path=unix_socket_path(tmp_path, port),
|
||||
socket_path=unix_socket_path(socket_dir, port),
|
||||
ready_timeout_seconds=0.5,
|
||||
)
|
||||
outcome: Final = pooler.start()
|
||||
|
|
@ -704,10 +706,10 @@ def _runtime_dir_listening_on(port: int) -> Path:
|
|||
|
||||
|
||||
class TestStartInContainerPgBouncer:
|
||||
def test_returns_the_loopback_url_once_the_pooler_listens(self, tmp_path: Path):
|
||||
def test_returns_the_loopback_url_once_the_pooler_listens(self, socket_tempdir: Path):
|
||||
port: Final = _free_port()
|
||||
auth_log: Final = tmp_path / "auth.log"
|
||||
binary: Final = _fake_pooler(tmp_path, port, auth_log=auth_log)
|
||||
auth_log: Final = socket_tempdir / "auth.log"
|
||||
binary: Final = _fake_pooler(socket_tempdir, port, auth_log=auth_log)
|
||||
settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(binary))
|
||||
pooled: Final = start_in_container_pgbouncer(settings, "postgresql://app:pw@db/litellm?connection_limit=5")
|
||||
assert isinstance(pooled, str), pooled
|
||||
|
|
@ -723,9 +725,9 @@ class TestStartInContainerPgBouncer:
|
|||
assert auth_log.read_text() == repr(f'"app" "pw"\n"litellm_pgbouncer" "{parsed.password}"\n') + "\n"
|
||||
|
||||
@pytest.mark.filterwarnings("ignore:This process .* is multi-threaded:DeprecationWarning")
|
||||
def test_a_forked_worker_exiting_leaves_the_pooler_and_its_files_to_the_parent(self, tmp_path: Path):
|
||||
def test_a_forked_worker_exiting_leaves_the_pooler_and_its_files_to_the_parent(self, socket_tempdir: Path):
|
||||
port: Final = _free_port()
|
||||
settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pooler(tmp_path, port)))
|
||||
settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pooler(socket_tempdir, port)))
|
||||
exit_hooks: Final[list[Callable[[], None]]] = []
|
||||
pooled: Final = start_in_container_pgbouncer(
|
||||
settings, "postgresql://app:pw@db/litellm", register_exit_hook=exit_hooks.append
|
||||
|
|
@ -766,10 +768,10 @@ class TestStartInContainerPgBouncer:
|
|||
assert "IAM_TOKEN_DB_AUTH" in outcome.reason
|
||||
assert not _listening(port)
|
||||
|
||||
def test_token_auth_mints_the_first_token_into_the_auth_file_before_the_pooler_starts(self, tmp_path: Path):
|
||||
def test_token_auth_mints_the_first_token_into_the_auth_file_before_the_pooler_starts(self, socket_tempdir: Path):
|
||||
port: Final = _free_port()
|
||||
auth_log: Final = tmp_path / "auth.log"
|
||||
binary: Final = _fake_pooler(tmp_path, port, auth_log=auth_log)
|
||||
auth_log: Final = socket_tempdir / "auth.log"
|
||||
binary: Final = _fake_pooler(socket_tempdir, port, auth_log=auth_log)
|
||||
settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(binary))
|
||||
token: Final = _entra_jwt(datetime.now(tz=timezone.utc) + timedelta(hours=1))
|
||||
pooled: Final = start_in_container_pgbouncer(
|
||||
|
|
@ -798,25 +800,25 @@ class TestStartInContainerPgBouncer:
|
|||
assert "no Azure credential" in outcome.reason
|
||||
assert not _listening(port)
|
||||
|
||||
def test_a_renewed_token_is_written_and_picked_up_by_the_running_and_by_a_restarted_pooler(self, tmp_path: Path):
|
||||
def test_a_renewed_token_is_written_and_picked_up_by_the_running_and_by_a_restarted_pooler(self, socket_dir: Path):
|
||||
port: Final = _free_port()
|
||||
auth_log: Final = tmp_path / "auth.log"
|
||||
auth_log: Final = socket_dir / "auth.log"
|
||||
plan: Final = plan_pgbouncer(
|
||||
"postgresql://app@db/litellm", PgBouncerSettings(enabled=True, port=port), tmp_path, None
|
||||
"postgresql://app@db/litellm", PgBouncerSettings(enabled=True, port=port), socket_dir, None
|
||||
)
|
||||
assert isinstance(plan, PgBouncerPlan), plan
|
||||
ini_path: Final = write_pgbouncer_ini(plan, tmp_path, None)
|
||||
write_userlist(plan.userlist("first"), tmp_path, None)
|
||||
ini_path: Final = write_pgbouncer_ini(plan, socket_dir, None)
|
||||
write_userlist(plan.userlist("first"), socket_dir, None)
|
||||
pooler: Final = PgBouncerProcess(
|
||||
argv=(str(_fake_pooler(tmp_path, port, auth_log=auth_log)), str(ini_path)),
|
||||
argv=(str(_fake_pooler(socket_dir, port, auth_log=auth_log)), str(ini_path)),
|
||||
port=port,
|
||||
socket_path=unix_socket_path(tmp_path, port),
|
||||
socket_path=unix_socket_path(socket_dir, port),
|
||||
restart_delay_seconds=0.1,
|
||||
)
|
||||
assert pooler.start() is None
|
||||
first_pid: Final = pooler.pid
|
||||
assert first_pid is not None
|
||||
install_pgbouncer_token(plan, tmp_path, None, pooler, "second")
|
||||
install_pgbouncer_token(plan, socket_dir, None, pooler, "second")
|
||||
assert _wait_until(lambda: auth_log.read_text().count("\n") == 2)
|
||||
os.kill(first_pid, signal.SIGKILL)
|
||||
assert _wait_until(lambda: pooler.pid not in (None, first_pid) and _listening(port))
|
||||
|
|
@ -837,9 +839,9 @@ class TestStartInContainerPgBouncer:
|
|||
assert "1.19" in outcome.reason
|
||||
assert not _listening(port)
|
||||
|
||||
def test_the_first_version_that_dies_on_a_failed_tcp_bind_is_accepted(self, tmp_path: Path):
|
||||
def test_the_first_version_that_dies_on_a_failed_tcp_bind_is_accepted(self, socket_tempdir: Path):
|
||||
port: Final = _free_port()
|
||||
binary: Final = _fake_pooler(tmp_path, port, version_banner="PgBouncer 1.19.0")
|
||||
binary: Final = _fake_pooler(socket_tempdir, port, version_banner="PgBouncer 1.19.0")
|
||||
settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(binary))
|
||||
pooled: Final = start_in_container_pgbouncer(settings, "postgresql://app:pw@db/litellm")
|
||||
assert isinstance(pooled, str), pooled
|
||||
|
|
|
|||
|
|
@ -4834,6 +4834,83 @@ class TestPanwAirsLatestRoleMessageOnlyEveryRequestShape:
|
|||
None,
|
||||
id="function_call_output",
|
||||
),
|
||||
pytest.param(
|
||||
(
|
||||
{"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_1",
|
||||
"output": [{"type": "input_text", "text": "tool "}, {"type": "input_text", "text": "result"}],
|
||||
},
|
||||
),
|
||||
None,
|
||||
id="function_call_output_text_parts",
|
||||
),
|
||||
pytest.param(
|
||||
(
|
||||
{"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_1",
|
||||
"output": [
|
||||
{"type": "input_text", "text": "tool "},
|
||||
{"type": "input_image", "image_url": "https://example.com/chart.png"},
|
||||
{"type": "input_text", "text": "result"},
|
||||
],
|
||||
},
|
||||
),
|
||||
None,
|
||||
id="function_call_output_text_and_image_parts",
|
||||
),
|
||||
pytest.param(
|
||||
(
|
||||
{"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"},
|
||||
{"type": "function_call_output", "output": "orphan tool result"},
|
||||
{"type": "function_call_output", "call_id": "call_1", "output": "tool result"},
|
||||
),
|
||||
None,
|
||||
id="function_call_output_without_call_id",
|
||||
),
|
||||
pytest.param(
|
||||
(
|
||||
{"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"},
|
||||
{"type": "tool_result", "call_id": "call_1", "output": "tool result"},
|
||||
),
|
||||
None,
|
||||
id="tool_result",
|
||||
),
|
||||
pytest.param(
|
||||
(
|
||||
{"type": "computer_call", "call_id": "cu_1", "id": "cu_1", "action": {"type": "screenshot"}},
|
||||
{
|
||||
"type": "computer_call_output",
|
||||
"call_id": "cu_1",
|
||||
"output": {"type": "computer_screenshot", "image_url": "https://example.com/screen.png"},
|
||||
},
|
||||
),
|
||||
None,
|
||||
id="computer_call_output",
|
||||
),
|
||||
pytest.param(
|
||||
(
|
||||
{"type": "computer_call", "call_id": "cu_1", "id": "cu_1", "action": {"type": "screenshot"}},
|
||||
{
|
||||
"type": "computer_call_output",
|
||||
"call_id": "cu_1",
|
||||
"output": {"type": "computer_screenshot", "file_id": "file-screen"},
|
||||
},
|
||||
),
|
||||
None,
|
||||
id="computer_call_output_file_id",
|
||||
),
|
||||
pytest.param(
|
||||
(
|
||||
{"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"},
|
||||
{"type": "function_call_output", "call_id": "call_1", "output": ""},
|
||||
),
|
||||
None,
|
||||
id="function_call_output_empty",
|
||||
),
|
||||
pytest.param(
|
||||
({"type": "reasoning", "id": "rs_1", "summary": [{"type": "summary_text", "text": "thinking"}]},),
|
||||
None,
|
||||
|
|
@ -4898,6 +4975,25 @@ class TestPanwAirsLatestRoleMessageOnlyEveryRequestShape:
|
|||
|
||||
assert [call.kwargs["content"] for call in mock_api.call_args_list] == [self.LATEST]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_flag_true_with_skip_tool_still_scans_only_the_latest_turn_on_responses(self) -> None:
|
||||
from litellm.llms.openai.responses.guardrail_translation.handler import (
|
||||
OpenAIResponsesHandler,
|
||||
)
|
||||
|
||||
handler = make_handler(experimental_use_latest_role_message_only=True)
|
||||
handler.skip_tool_message_in_guardrail = True
|
||||
request_data = self._responses_request(
|
||||
{"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"},
|
||||
{"type": "function_call_output", "call_id": "call_1", "output": "tool result"},
|
||||
{"role": "user", "content": self.LATEST},
|
||||
)
|
||||
patcher, mock_api = self._scan(handler)
|
||||
with patcher:
|
||||
await OpenAIResponsesHandler().process_input_messages(data=request_data, guardrail_to_apply=handler)
|
||||
|
||||
assert [call.kwargs["content"] for call in mock_api.call_args_list] == [self.LATEST]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_flag_false_responses_scans_instructions_and_full_history(self) -> None:
|
||||
from litellm.llms.openai.responses.guardrail_translation.handler import (
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from litellm.proxy.guardrails._content_utils import (
|
|||
build_inspection_messages,
|
||||
has_non_string_content,
|
||||
is_non_conversational_call_type,
|
||||
is_non_conversational_request,
|
||||
is_string_batch_input,
|
||||
iter_message_text,
|
||||
iter_tool_json_text,
|
||||
|
|
@ -819,6 +820,30 @@ def test_is_non_conversational_call_type_defaults_to_inspecting_unknown_call_typ
|
|||
assert is_non_conversational_call_type("some_future_call_type") is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("call_type", "endpoint", "expected"),
|
||||
[
|
||||
("aembedding", None, True),
|
||||
("allm_passthrough_route", "embeddings", True),
|
||||
("allm_passthrough_route", "/v1/embeddings/", True),
|
||||
("allm_passthrough_route", "chat/completions", False),
|
||||
("allm_passthrough_route", "model/amazon.titan-embed-text-v2:0/invoke", False),
|
||||
("allm_passthrough_route", "fine_tuned_embeddings_jobs", False),
|
||||
("allm_passthrough_route", "chat/completions#embeddings", False),
|
||||
("allm_passthrough_route", "chat/completions?x=embeddings", False),
|
||||
("allm_passthrough_route", "chat/completions;embeddings", False),
|
||||
("allm_passthrough_route", None, False),
|
||||
("acompletion", "embeddings", False),
|
||||
],
|
||||
)
|
||||
def test_is_non_conversational_request_skips_only_embeddings_requests(
|
||||
call_type: str, endpoint: str | None, expected: bool
|
||||
):
|
||||
data: Final[dict[str, object]] = {"endpoint": endpoint} if endpoint is not None else {}
|
||||
|
||||
assert is_non_conversational_request(call_type, data) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tool_input",
|
||||
(
|
||||
|
|
|
|||
|
|
@ -12,17 +12,19 @@ inspection by the relevant guardrail hook:
|
|||
not just ``choices[0]``.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, Dict, Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
from httpx import Request, Response
|
||||
|
||||
import litellm
|
||||
from litellm import DualCache
|
||||
from litellm.constants import DEFAULT_OPENAI_MODERATIONS_MODEL
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.types.utils import Choices, Message, ModelResponse
|
||||
from litellm.types.utils import CallTypesLiteral, Choices, Message, ModelResponse
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
|
@ -905,3 +907,78 @@ async def test_google_text_moderation_inspects_multimodal_content(user_api_key):
|
|||
)
|
||||
|
||||
assert seen_documents == ["hello world"]
|
||||
|
||||
|
||||
class _FinanceFlaggingLanguageClient:
|
||||
def __init__(self) -> None:
|
||||
self.moderated_texts: Final[list[str]] = [] # mutable-ok: records what reached Google
|
||||
|
||||
def moderate_text(self, request: SimpleNamespace) -> SimpleNamespace:
|
||||
self.moderated_texts.append(request.document.content)
|
||||
return SimpleNamespace(moderation_categories=[SimpleNamespace(name="Finance", confidence=0.885)])
|
||||
|
||||
|
||||
def _google_text_moderation_flagging_finance():
|
||||
from enterprise.enterprise_hooks.google_text_moderation import (
|
||||
ENTERPRISE_GoogleTextModeration,
|
||||
)
|
||||
|
||||
guard: Final = ENTERPRISE_GoogleTextModeration.__new__(ENTERPRISE_GoogleTextModeration)
|
||||
guard.client = _FinanceFlaggingLanguageClient()
|
||||
guard.language_document = lambda content, type_: SimpleNamespace(content=content)
|
||||
guard.moderate_text_request = lambda document: SimpleNamespace(document=document)
|
||||
guard.document_type = "PLAIN_TEXT"
|
||||
guard.finance_confidence_threshold = 0.8
|
||||
return guard
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("call_type", "data"),
|
||||
[
|
||||
("aembedding", {"model": "text-embedding-ada-002", "input": "stocks"}),
|
||||
("embedding", {"model": "text-embedding-ada-002", "input": ["stocks", "mortgage"]}),
|
||||
("allm_passthrough_route", {"model": "gigachat-embeddings", "endpoint": "embeddings", "input": ["stocks"]}),
|
||||
],
|
||||
)
|
||||
async def test_google_text_moderation_leaves_embedding_input_unmoderated(
|
||||
user_api_key: UserAPIKeyAuth, call_type: CallTypesLiteral, data: dict[str, object]
|
||||
):
|
||||
guard: Final = _google_text_moderation_flagging_finance()
|
||||
|
||||
await guard.async_moderation_hook(data=data, user_api_key_dict=user_api_key, call_type=call_type)
|
||||
|
||||
assert guard.client.moderated_texts == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("call_type", "data"),
|
||||
[
|
||||
("acompletion", {"model": "gpt-4o", "messages": [{"role": "user", "content": "stocks"}]}),
|
||||
("anthropic_messages", {"model": "claude", "messages": [{"role": "user", "content": "stocks"}]}),
|
||||
("aresponses", {"model": "gpt-4o", "input": "stocks"}),
|
||||
(
|
||||
"allm_passthrough_route",
|
||||
{"model": "gigachat", "endpoint": "chat/completions", "messages": [{"role": "user", "content": "stocks"}]},
|
||||
),
|
||||
(
|
||||
"allm_passthrough_route",
|
||||
{
|
||||
"model": "gigachat",
|
||||
"endpoint": "chat/completions#embeddings",
|
||||
"messages": [{"role": "user", "content": "stocks"}],
|
||||
},
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_google_text_moderation_still_blocks_conversations(
|
||||
user_api_key: UserAPIKeyAuth, call_type: CallTypesLiteral, data: dict[str, object]
|
||||
):
|
||||
guard: Final = _google_text_moderation_flagging_finance()
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await guard.async_moderation_hook(data=data, user_api_key_dict=user_api_key, call_type=call_type)
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
assert guard.client.moderated_texts == ["stocks"]
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import json
|
|||
import logging
|
||||
from collections.abc import Mapping
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Final, cast
|
||||
from unittest.mock import AsyncMock, MagicMock, create_autospec, patch
|
||||
|
|
@ -3062,10 +3063,10 @@ async def _no_fallback(line: bytes) -> None:
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_log_success_event_hands_the_sidecar_a_compact_event_and_skips_the_pipeline(tmp_path):
|
||||
async def test_async_log_success_event_hands_the_sidecar_a_compact_event_and_skips_the_pipeline(socket_path: Path):
|
||||
handler = _RecordingHandler()
|
||||
consumer = SpendEventConsumer(handler)
|
||||
address = UnixAddress(path=str(tmp_path / "spend.sock"))
|
||||
address = UnixAddress(path=str(socket_path))
|
||||
server = await consumer.serve(address)
|
||||
producer = SpendEventProducer(
|
||||
address=address, on_unavailable="fallback", buffer_size=10, connect_timeout=1.0, fallback=_no_fallback
|
||||
|
|
|
|||
|
|
@ -11,11 +11,14 @@ import contextlib
|
|||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, AsyncIterator, Callable, Dict, Iterator, List, Optional
|
||||
from typing import TYPE_CHECKING, Any, AsyncIterator, Callable, Dict, Final, Iterator, List, Optional
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm import Router
|
||||
|
||||
# Repo root, anchored to this file (not CWD) so the path is correct no
|
||||
# matter where pytest is invoked from. With the project installed via
|
||||
# uv this is defensive — `litellm` already resolves through site-packages
|
||||
|
|
@ -536,3 +539,25 @@ def reset_login_throttle(monkeypatch):
|
|||
_drop_throttle_keys()
|
||||
yield _drop_throttle_keys
|
||||
_drop_throttle_keys()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def openai_only_router(monkeypatch: pytest.MonkeyPatch) -> Router:
|
||||
from litellm import Router
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-4",
|
||||
"litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-fake"},
|
||||
"model_info": {"id": "gpt-4-id"},
|
||||
}
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
monkeypatch.setattr(proxy_server, "llm_model_list", router.model_list)
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", None)
|
||||
monkeypatch.setattr(proxy_server, "general_settings", {})
|
||||
monkeypatch.setattr(proxy_server, "user_model", None)
|
||||
return router
|
||||
|
|
|
|||
|
|
@ -9,15 +9,20 @@ Pins (PR2):
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.integrations.custom_secret_manager import CustomSecretManager
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy import utils as proxy_utils
|
||||
from litellm.proxy.utils import create_model_info_response
|
||||
from litellm.types.router import DeploymentModelListingInfo
|
||||
from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem
|
||||
|
||||
from .conftest import normalize # type: ignore[import-not-found]
|
||||
|
||||
|
|
@ -237,6 +242,7 @@ def test_anthropic_display_name_resolved_via_internal_team_key(
|
|||
return_value=[
|
||||
{
|
||||
"model_name": internal_name,
|
||||
"litellm_params": {"model": "openai/gpt-4"},
|
||||
"model_info": {
|
||||
"team_id": "team-1",
|
||||
"team_public_model_name": "gpt-4-team",
|
||||
|
|
@ -320,6 +326,7 @@ def test_anthropic_format_returns_public_team_model_name(
|
|||
return_value=[
|
||||
{
|
||||
"model_name": internal_name,
|
||||
"litellm_params": {"model": "openai/gpt-4"},
|
||||
"model_info": {
|
||||
"team_id": "team-1",
|
||||
"team_public_model_name": "gpt-4-team",
|
||||
|
|
@ -494,7 +501,11 @@ def test_codex_service_tiers_resolved_via_internal_team_key(client, auth_as, pat
|
|||
|
||||
patched_models.get_model_list = MagicMock(
|
||||
return_value=[
|
||||
{"model_name": internal_name, "model_info": {"team_id": "team-1", "team_public_model_name": "gpt-4-team"}}
|
||||
{
|
||||
"model_name": internal_name,
|
||||
"litellm_params": {"model": "openai/gpt-4"},
|
||||
"model_info": {"team_id": "team-1", "team_public_model_name": "gpt-4-team"},
|
||||
}
|
||||
]
|
||||
)
|
||||
patched_models.get_model_names = MagicMock(return_value=[internal_name])
|
||||
|
|
@ -514,3 +525,206 @@ def test_codex_service_tiers_resolved_via_internal_team_key(client, auth_as, pat
|
|||
assert response.status_code == 200
|
||||
(entry,) = response.json()["models"]
|
||||
assert (entry["slug"], [tier["id"] for tier in entry["service_tiers"]]) == ("gpt-4-team", ["ultrafast"])
|
||||
|
||||
|
||||
def _priced_jev_models() -> frozenset[str]:
|
||||
return frozenset(name.removeprefix("typesafe/") for name in litellm.model_cost if name.startswith("typesafe/"))
|
||||
|
||||
|
||||
async def _admin_picker_models() -> set[str]:
|
||||
admin: Final = UserAPIKeyAuth(api_key="sk-test", user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN)
|
||||
response: Final = await proxy_server.model_list(
|
||||
user_api_key_dict=admin, return_wildcard_routes=True, include_model_access_groups=True, scope="expand"
|
||||
)
|
||||
return {model["id"] for model in response["data"]}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scope_expand_offers_typesafe_models_when_typesafe_is_configured(
|
||||
openai_only_router: Router, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake")
|
||||
jev_models: Final = _priced_jev_models()
|
||||
|
||||
picker_models: Final = await _admin_picker_models()
|
||||
|
||||
assert jev_models
|
||||
assert {"gpt-4", "typesafe/*", *jev_models} <= picker_models
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scope_expand_still_lists_models_when_the_typesafe_key_reference_cannot_resolve(
|
||||
openai_only_router: Router, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.delenv("LITELLM_QA_UNSET_TYPESAFE_TOKEN", raising=False)
|
||||
monkeypatch.setenv("TYPESAFE_API_KEY", "oidc/env/LITELLM_QA_UNSET_TYPESAFE_TOKEN")
|
||||
|
||||
picker_models: Final = await _admin_picker_models()
|
||||
|
||||
assert {"gpt-4", "typesafe/*"} <= picker_models
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scope_expand_offers_no_typesafe_models_without_typesafe_credentials(
|
||||
openai_only_router: Router, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.delenv("TYPESAFE_API_KEY", raising=False)
|
||||
|
||||
picker_models: Final = await _admin_picker_models()
|
||||
|
||||
assert "gpt-4" in picker_models
|
||||
assert picker_models.isdisjoint({"typesafe/*", *_priced_jev_models()})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scope_expand_does_not_read_the_secret_manager_for_typesafe_credentials(
|
||||
openai_only_router: Router, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
secret_manager: Final = MagicMock(spec=CustomSecretManager)
|
||||
secret_manager.sync_read_secret.return_value = None
|
||||
monkeypatch.delenv("TYPESAFE_API_KEY", raising=False)
|
||||
monkeypatch.setattr(litellm, "secret_manager_client", secret_manager)
|
||||
monkeypatch.setattr(litellm, "_key_management_system", KeyManagementSystem.CUSTOM)
|
||||
monkeypatch.setattr(litellm, "_key_management_settings", KeyManagementSettings(access_mode="read_only"))
|
||||
|
||||
picker_models: Final = await _admin_picker_models()
|
||||
|
||||
secrets_read: Final = {
|
||||
call.kwargs["secret_name"]
|
||||
for call in (*secret_manager.sync_read_secret.call_args_list, *secret_manager.async_read_secret.call_args_list)
|
||||
}
|
||||
assert "gpt-4" in picker_models
|
||||
assert "TYPESAFE_API_KEY" not in secrets_read
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_default_model_list_does_not_offer_typesafe_models(
|
||||
openai_only_router: Router, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake")
|
||||
admin: Final = UserAPIKeyAuth(api_key="sk-test", user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN)
|
||||
|
||||
response: Final = await proxy_server.model_list(user_api_key_dict=admin)
|
||||
|
||||
assert {model["id"] for model in response["data"]}.isdisjoint({"typesafe/*", *_priced_jev_models()})
|
||||
|
||||
|
||||
async def _key_dialog_models(user_role: LitellmUserRoles, team_id: str | None) -> set[str]:
|
||||
caller: Final = UserAPIKeyAuth(api_key="sk-test", user_id="u", user_role=user_role)
|
||||
response: Final = await proxy_server.model_list(
|
||||
user_api_key_dict=caller, return_wildcard_routes=True, include_model_access_groups=True, team_id=team_id
|
||||
)
|
||||
return {model["id"] for model in response["data"]}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_key_dialog_without_a_team_offers_typesafe_models_to_an_admin(
|
||||
openai_only_router: Router, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake")
|
||||
|
||||
dialog_models: Final = await _key_dialog_models(LitellmUserRoles.PROXY_ADMIN, team_id=None)
|
||||
|
||||
assert {"gpt-4", "typesafe/*", *_priced_jev_models()} <= dialog_models
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("scope", ["expand", None])
|
||||
async def test_pickers_offer_typesafe_names_once_and_no_bare_name_for_registered_typesafe_deployments(
|
||||
monkeypatch: pytest.MonkeyPatch, scope: str | None
|
||||
) -> None:
|
||||
monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake")
|
||||
jev_models: Final = _priced_jev_models()
|
||||
monkeypatch.setattr(litellm, "model_cost", {**litellm.model_cost})
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{"model_name": "typesafe/*", "litellm_params": {"model": "typesafe/*", "api_key": "sk-typesafe-fake"}},
|
||||
{
|
||||
"model_name": "ticket-router",
|
||||
"litellm_params": {"model": "typesafe/my-classifier", "api_key": "sk-typesafe-fake"},
|
||||
},
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
monkeypatch.setattr(proxy_server, "llm_model_list", router.model_list)
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", None)
|
||||
monkeypatch.setattr(proxy_server, "general_settings", {})
|
||||
monkeypatch.setattr(proxy_server, "user_model", None)
|
||||
admin: Final = UserAPIKeyAuth(api_key="sk-test", user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN)
|
||||
|
||||
response: Final = await proxy_server.model_list(
|
||||
user_api_key_dict=admin, return_wildcard_routes=True, include_model_access_groups=True, scope=scope
|
||||
)
|
||||
|
||||
listed: Final = [model["id"] for model in response["data"]]
|
||||
assert {"ticket-router", *jev_models} <= set(listed)
|
||||
assert listed.count("typesafe/*") == 1
|
||||
assert set(listed).isdisjoint({"*", "my-classifier"})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_key_dialog_offers_no_typesafe_models_to_an_internal_user(
|
||||
openai_only_router: Router, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake")
|
||||
|
||||
dialog_models: Final = await _key_dialog_models(LitellmUserRoles.INTERNAL_USER, team_id=None)
|
||||
|
||||
assert "gpt-4" in dialog_models
|
||||
assert dialog_models.isdisjoint({"typesafe/*", *_priced_jev_models()})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_key_dialog_for_a_team_offers_no_typesafe_models_the_team_lacks(
|
||||
openai_only_router: Router, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake")
|
||||
|
||||
dialog_models: Final = await _key_dialog_models(LitellmUserRoles.PROXY_ADMIN, team_id="team-without-typesafe")
|
||||
|
||||
assert dialog_models.isdisjoint({"typesafe/*", *_priced_jev_models()})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scope_expand_lists_no_typesafe_models_to_a_client_not_asking_for_allowlist_names(
|
||||
openai_only_router: Router, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake")
|
||||
admin: Final = UserAPIKeyAuth(api_key="sk-test", user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN)
|
||||
|
||||
response: Final = await proxy_server.model_list(user_api_key_dict=admin, scope="expand")
|
||||
|
||||
listed: Final = {model["id"] for model in response["data"]}
|
||||
assert "gpt-4" in listed
|
||||
assert listed.isdisjoint({"typesafe/*", *_priced_jev_models()})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("scope", "return_wildcard_routes", "only_model_access_groups"),
|
||||
[
|
||||
pytest.param(None, False, True, id="access-group-dropdown"),
|
||||
pytest.param(None, None, None, id="dashboard-list-without-wildcard-routes"),
|
||||
pytest.param("expand", None, None, id="expanded-list-without-wildcard-routes"),
|
||||
pytest.param(None, True, True, id="access-groups-only-with-wildcard-routes"),
|
||||
],
|
||||
)
|
||||
async def test_dashboard_lists_not_asking_for_allowlist_names_offer_no_typesafe_models(
|
||||
openai_only_router: Router,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
scope: str | None,
|
||||
return_wildcard_routes: bool | None,
|
||||
only_model_access_groups: bool | None,
|
||||
) -> None:
|
||||
monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake")
|
||||
admin: Final = UserAPIKeyAuth(api_key="sk-test", user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN)
|
||||
|
||||
response: Final = await proxy_server.model_list(
|
||||
user_api_key_dict=admin,
|
||||
include_model_access_groups=True,
|
||||
return_wildcard_routes=return_wildcard_routes,
|
||||
only_model_access_groups=only_model_access_groups,
|
||||
scope=scope,
|
||||
)
|
||||
|
||||
assert {model["id"] for model in response["data"]}.isdisjoint({"typesafe/*", *_priced_jev_models()})
|
||||
|
|
|
|||
|
|
@ -1141,6 +1141,25 @@ def test_get_supported_endpoints_chat_completions_present(reset_endpoints_cache)
|
|||
assert len(chat["providers"]) > 0
|
||||
|
||||
|
||||
def test_get_supported_endpoints_lists_streamlake_on_chat_messages_and_responses(
|
||||
reset_endpoints_cache: None,
|
||||
) -> None:
|
||||
app: Final[FastAPI] = FastAPI()
|
||||
app.include_router(router)
|
||||
client: Final[TestClient] = TestClient(app)
|
||||
with client:
|
||||
response: Final = client.get("/public/endpoints")
|
||||
assert response.status_code == 200, response.text
|
||||
|
||||
streamlake_endpoints: Final = {
|
||||
endpoint["key"]
|
||||
for endpoint in response.json()["endpoints"]
|
||||
if any(provider["slug"] == "streamlake" for provider in endpoint["providers"])
|
||||
}
|
||||
assert {"chat_completions", "messages", "responses"} <= streamlake_endpoints
|
||||
assert "embeddings" not in streamlake_endpoints
|
||||
|
||||
|
||||
def test_get_supported_endpoints_display_names_have_no_slug_suffix(
|
||||
reset_endpoints_cache,
|
||||
):
|
||||
|
|
|
|||
|
|
@ -177,9 +177,9 @@ def test_settings_read_the_documented_env(monkeypatch):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_events_reach_the_sidecar_once_and_in_order(tmp_path: Path):
|
||||
async def test_events_reach_the_sidecar_once_and_in_order(socket_path: Path):
|
||||
fallback: Final = _Fallback()
|
||||
async with _Sidecar(tmp_path / "spend.sock") as sidecar:
|
||||
async with _Sidecar(socket_path) as sidecar:
|
||||
producer: Final = _producer(sidecar.path, fallback)
|
||||
outcomes: Final = [await producer.publish(f"event-{i}\n".encode()) for i in range(20)]
|
||||
await producer.close(drain_timeout=5.0)
|
||||
|
|
@ -193,9 +193,9 @@ async def test_events_reach_the_sidecar_once_and_in_order(tmp_path: Path):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unreachable_sidecar_falls_back_in_process_and_backs_off(tmp_path: Path):
|
||||
async def test_unreachable_sidecar_falls_back_in_process_and_backs_off(socket_path: Path):
|
||||
fallback: Final = _Fallback()
|
||||
producer: Final = _producer(tmp_path / "missing.sock", fallback)
|
||||
producer: Final = _producer(socket_path.with_name("missing.sock"), fallback)
|
||||
first: Final = await producer.publish(b"event-1\n")
|
||||
await asyncio.sleep(0.05)
|
||||
second: Final = await producer.publish(b"event-2\n")
|
||||
|
|
@ -210,11 +210,11 @@ async def test_unreachable_sidecar_falls_back_in_process_and_backs_off(tmp_path:
|
|||
|
||||
@pytest.mark.parametrize("loop_factory", [asyncio.new_event_loop, uvloop.new_event_loop], ids=["asyncio", "uvloop"])
|
||||
def test_sidecar_hang_up_falls_back_instead_of_losing_events(
|
||||
tmp_path: Path, loop_factory: Callable[[], asyncio.AbstractEventLoop]
|
||||
socket_path: Path, loop_factory: Callable[[], asyncio.AbstractEventLoop]
|
||||
):
|
||||
async def scenario() -> tuple[list[bytes], list[bytes], tuple[int, int, int]]:
|
||||
fallback: Final = _Fallback()
|
||||
sidecar: Final = _Sidecar(tmp_path / "spend.sock")
|
||||
sidecar: Final = _Sidecar(socket_path)
|
||||
async with sidecar:
|
||||
producer: Final = _producer(sidecar.path, fallback)
|
||||
await producer.publish(b"event-1\n")
|
||||
|
|
@ -236,7 +236,7 @@ def test_sidecar_hang_up_falls_back_instead_of_losing_events(
|
|||
|
||||
@pytest.mark.parametrize("loop_factory", [asyncio.new_event_loop, uvloop.new_event_loop], ids=["asyncio", "uvloop"])
|
||||
def test_mid_stream_crash_never_bills_an_event_on_both_sides(
|
||||
tmp_path: Path, loop_factory: Callable[[], asyncio.AbstractEventLoop]
|
||||
socket_path: Path, loop_factory: Callable[[], asyncio.AbstractEventLoop]
|
||||
):
|
||||
"""Events large enough to straddle the kernel buffer, a sidecar that reads some and then drops the socket: a
|
||||
failed write may only fall back when the sidecar cannot have read the whole line."""
|
||||
|
|
@ -244,7 +244,7 @@ def test_mid_stream_crash_never_bills_an_event_on_both_sides(
|
|||
|
||||
async def scenario() -> tuple[list[bytes], list[bytes], tuple[int, int, int]]:
|
||||
fallback: Final = _Fallback()
|
||||
sidecar: Final = _CrashingSidecar(tmp_path / "spend.sock", lines_before_crash=3)
|
||||
sidecar: Final = _CrashingSidecar(socket_path, lines_before_crash=3)
|
||||
async with sidecar:
|
||||
producer: Final = _producer(sidecar.path, fallback)
|
||||
for event in events:
|
||||
|
|
@ -266,11 +266,11 @@ def test_mid_stream_crash_never_bills_an_event_on_both_sides(
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_drain_timeout_hands_the_in_flight_event_to_fallback(tmp_path: Path):
|
||||
async def test_drain_timeout_hands_the_in_flight_event_to_fallback(socket_path: Path):
|
||||
"""A sidecar that stops reading leaves one event half-written; cancelling the writer must not lose it."""
|
||||
fallback: Final = _Fallback()
|
||||
stuck: Final = b"x" * (4 * 1024 * 1024) + b"\n"
|
||||
async with _Sidecar(tmp_path / "spend.sock", reads=False) as sidecar:
|
||||
async with _Sidecar(socket_path, reads=False) as sidecar:
|
||||
producer: Final = _producer(sidecar.path, fallback)
|
||||
assert await producer.publish(stuck) == "queued"
|
||||
await asyncio.sleep(0.1)
|
||||
|
|
@ -282,10 +282,10 @@ async def test_drain_timeout_hands_the_in_flight_event_to_fallback(tmp_path: Pat
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_shutdown_lets_the_writer_finish_a_fallback_already_in_progress(tmp_path: Path):
|
||||
async def test_shutdown_lets_the_writer_finish_a_fallback_already_in_progress(socket_path: Path):
|
||||
"""Cancelling the writer while it runs the pipeline in-process must neither lose nor repeat that event."""
|
||||
fallback: Final = _GatedFallback()
|
||||
producer: Final = _producer(tmp_path / "missing.sock", fallback)
|
||||
producer: Final = _producer(socket_path.with_name("missing.sock"), fallback)
|
||||
assert await producer.publish(b"event-1\n") == "queued"
|
||||
await asyncio.wait_for(fallback.started.wait(), 5.0)
|
||||
closing: Final = asyncio.ensure_future(producer.close(drain_timeout=0.05))
|
||||
|
|
@ -299,10 +299,10 @@ async def test_shutdown_lets_the_writer_finish_a_fallback_already_in_progress(tm
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_shutdown_does_not_replay_an_event_the_kernel_already_took(tmp_path: Path):
|
||||
async def test_shutdown_does_not_replay_an_event_the_kernel_already_took(socket_path: Path):
|
||||
"""Cancelling a drain whose bytes already left the process must not run the event a second time in-process."""
|
||||
fallback: Final = _Fallback()
|
||||
async with _Sidecar(tmp_path / "spend.sock") as sidecar:
|
||||
async with _Sidecar(socket_path) as sidecar:
|
||||
producer: Final = _producer(sidecar.path, fallback, open_connection=_open_with_stalled_drain)
|
||||
assert await producer.publish(b"event-1\n") == "queued"
|
||||
await asyncio.sleep(0.1)
|
||||
|
|
@ -316,9 +316,9 @@ async def test_shutdown_does_not_replay_an_event_the_kernel_already_took(tmp_pat
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_drop_policy_counts_instead_of_running_in_process(tmp_path: Path):
|
||||
async def test_drop_policy_counts_instead_of_running_in_process(socket_path: Path):
|
||||
fallback: Final = _Fallback()
|
||||
producer: Final = _producer(tmp_path / "missing.sock", fallback, on_unavailable="drop")
|
||||
producer: Final = _producer(socket_path.with_name("missing.sock"), fallback, on_unavailable="drop")
|
||||
await producer.publish(b"event-1\n")
|
||||
await producer.close(drain_timeout=5.0)
|
||||
assert await producer.publish(b"event-2\n") == "dropped"
|
||||
|
|
@ -328,9 +328,9 @@ async def test_drop_policy_counts_instead_of_running_in_process(tmp_path: Path):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_full_buffer_applies_the_unavailable_policy_immediately(tmp_path: Path):
|
||||
async def test_full_buffer_applies_the_unavailable_policy_immediately(socket_path: Path):
|
||||
fallback: Final = _Fallback()
|
||||
async with _Sidecar(tmp_path / "spend.sock") as sidecar:
|
||||
async with _Sidecar(socket_path) as sidecar:
|
||||
producer: Final = _producer(sidecar.path, fallback, buffer_size=2)
|
||||
outcomes: Final = [await producer.publish(f"event-{i}\n".encode()) for i in range(3)]
|
||||
await producer.close(drain_timeout=5.0)
|
||||
|
|
@ -342,9 +342,9 @@ async def test_full_buffer_applies_the_unavailable_policy_immediately(tmp_path:
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_close_flushes_buffered_events_then_refuses_new_ones(tmp_path: Path):
|
||||
async def test_close_flushes_buffered_events_then_refuses_new_ones(socket_path: Path):
|
||||
fallback: Final = _Fallback()
|
||||
async with _Sidecar(tmp_path / "spend.sock") as sidecar:
|
||||
async with _Sidecar(socket_path) as sidecar:
|
||||
producer: Final = _producer(sidecar.path, fallback)
|
||||
for i in range(50):
|
||||
await producer.publish(f"event-{i}\n".encode())
|
||||
|
|
|
|||
|
|
@ -49,14 +49,14 @@ class _Fallback:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("transport", ["unix", "tcp"])
|
||||
async def test_consumer_handles_each_producer_line_once_in_order(tmp_path: Path, transport: str):
|
||||
async def test_consumer_handles_each_producer_line_once_in_order(socket_path: Path, transport: str):
|
||||
handler: Final = _Handler(fail_on=b"event-3\n")
|
||||
consumer: Final = SpendEventConsumer(handler)
|
||||
server: Final = await consumer.serve(
|
||||
UnixAddress(path=str(tmp_path / "spend.sock")) if transport == "unix" else TcpAddress("127.0.0.1", 0)
|
||||
UnixAddress(path=str(socket_path)) if transport == "unix" else TcpAddress("127.0.0.1", 0)
|
||||
)
|
||||
address: Final = (
|
||||
UnixAddress(path=str(tmp_path / "spend.sock"))
|
||||
UnixAddress(path=str(socket_path))
|
||||
if transport == "unix"
|
||||
else TcpAddress("127.0.0.1", server.sockets[0].getsockname()[1])
|
||||
)
|
||||
|
|
@ -74,10 +74,10 @@ async def test_consumer_handles_each_producer_line_once_in_order(tmp_path: Path,
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_consumer_discards_a_truncated_trailing_event(tmp_path: Path):
|
||||
async def test_consumer_discards_a_truncated_trailing_event(socket_path: Path):
|
||||
handler: Final = _Handler()
|
||||
consumer: Final = SpendEventConsumer(handler)
|
||||
address: Final = UnixAddress(path=str(tmp_path / "spend.sock"))
|
||||
address: Final = UnixAddress(path=str(socket_path))
|
||||
server: Final = await consumer.serve(address)
|
||||
_, writer = await open_collector_connection(address, timeout=1.0)
|
||||
writer.write(b"whole\npartial-without-newline")
|
||||
|
|
@ -93,9 +93,9 @@ async def test_consumer_discards_a_truncated_trailing_event(tmp_path: Path):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_drain_reports_producers_still_connected_after_the_timeout(tmp_path: Path):
|
||||
async def test_drain_reports_producers_still_connected_after_the_timeout(socket_path: Path):
|
||||
consumer: Final = SpendEventConsumer(_Handler())
|
||||
address: Final = UnixAddress(path=str(tmp_path / "spend.sock"))
|
||||
address: Final = UnixAddress(path=str(socket_path))
|
||||
server: Final = await consumer.serve(address)
|
||||
_, writer = await open_collector_connection(address, timeout=1.0)
|
||||
await asyncio.sleep(0.05)
|
||||
|
|
@ -108,11 +108,11 @@ async def test_drain_reports_producers_still_connected_after_the_timeout(tmp_pat
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_graceful_stop_hands_the_producer_over_to_its_fallback_without_losing_events(tmp_path: Path):
|
||||
async def test_graceful_stop_hands_the_producer_over_to_its_fallback_without_losing_events(socket_path: Path):
|
||||
handler: Final = _Handler()
|
||||
fallback: Final = _Fallback()
|
||||
consumer: Final = SpendEventConsumer(handler)
|
||||
address: Final = UnixAddress(path=str(tmp_path / "spend.sock"))
|
||||
address: Final = UnixAddress(path=str(socket_path))
|
||||
server: Final = await consumer.serve(address)
|
||||
producer: Final = SpendEventProducer(
|
||||
address=address, on_unavailable="fallback", buffer_size=100, connect_timeout=1.0, fallback=fallback
|
||||
|
|
@ -134,7 +134,7 @@ async def test_graceful_stop_hands_the_producer_over_to_its_fallback_without_los
|
|||
|
||||
@pytest.mark.parametrize("loop_factory", [asyncio.new_event_loop, uvloop.new_event_loop], ids=["asyncio", "uvloop"])
|
||||
def test_drain_still_hands_over_live_producers_when_another_connection_already_died(
|
||||
tmp_path: Path, loop_factory: Callable[[], asyncio.AbstractEventLoop]
|
||||
socket_path: Path, loop_factory: Callable[[], asyncio.AbstractEventLoop]
|
||||
):
|
||||
"""A transport the loop force-closed under a busy handler must not abort the half-close of the others."""
|
||||
|
||||
|
|
@ -145,7 +145,7 @@ def test_drain_still_hands_over_live_producers_when_another_connection_already_d
|
|||
await release.wait()
|
||||
|
||||
consumer: Final = SpendEventConsumer(slow_handler)
|
||||
address: Final = UnixAddress(path=str(tmp_path / "spend.sock"))
|
||||
address: Final = UnixAddress(path=str(socket_path))
|
||||
server: Final = await consumer.serve(address)
|
||||
_, dead = await open_collector_connection(address, timeout=1.0)
|
||||
dead.write(b"stuck\n")
|
||||
|
|
|
|||
|
|
@ -22,7 +22,9 @@ import json
|
|||
import asyncio
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
from litellm import RateLimitError, Timeout, completion, completion_cost, embedding
|
||||
|
|
@ -3857,3 +3859,104 @@ async def test_model_list_return_wildcard_routes_defaults_to_general_setting(
|
|||
|
||||
assert "gpt-6.1-sol" in listed, listed
|
||||
assert ("openai/*" in listed) is wildcard_listed, listed
|
||||
|
||||
|
||||
_DEBUG_LOG_SENTINEL: Final = "sentinel-request-body-7f3a"
|
||||
_LONG_PROMPT: Final = f"{_DEBUG_LOG_SENTINEL} " + "Keep edits minimal and explain them briefly. " * 20
|
||||
_TOOL_PARAMETERS: Final = {"type": "object", "properties": {"cmd": {"type": "string"}}}
|
||||
|
||||
|
||||
class _MessageCollector(logging.Handler):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(level=logging.DEBUG)
|
||||
self.messages: list[str] = [] # mutable-ok: a logging.Handler collects records as they are emitted
|
||||
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
self.messages.append(record.getMessage())
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"route, path_suffix, body, upstream_json",
|
||||
[
|
||||
(
|
||||
"/v1/chat/completions",
|
||||
"/chat/completions",
|
||||
{
|
||||
"model": "test_openai_models",
|
||||
"messages": [{"role": "user", "content": _LONG_PROMPT}],
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {"name": "shell", "description": _LONG_PROMPT, "parameters": _TOOL_PARAMETERS},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "chatcmpl-1",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "gpt-3.5-turbo",
|
||||
"choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "hi"}}],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
},
|
||||
),
|
||||
(
|
||||
"/v1/responses",
|
||||
"/responses",
|
||||
{
|
||||
"model": "test_openai_models",
|
||||
"instructions": _LONG_PROMPT,
|
||||
"input": "hi",
|
||||
"tools": [
|
||||
{"type": "function", "name": "shell", "description": _LONG_PROMPT, "parameters": _TOOL_PARAMETERS}
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "resp_1",
|
||||
"object": "response",
|
||||
"created_at": 1,
|
||||
"status": "completed",
|
||||
"model": "gpt-3.5-turbo",
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_1",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "hi", "annotations": []}],
|
||||
}
|
||||
],
|
||||
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
|
||||
},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_detailed_debug_logs_request_body_only_as_received_and_as_sent(
|
||||
client_no_auth: TestClient,
|
||||
respx_mock: respx.MockRouter,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
route: str,
|
||||
path_suffix: str,
|
||||
body: Mapping[str, object],
|
||||
upstream_json: Mapping[str, object],
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
respx_mock.post(url__regex=rf".*{path_suffix}$").mock(return_value=httpx.Response(200, json=upstream_json))
|
||||
collector: Final = _MessageCollector()
|
||||
loggers: Final = tuple(logging.getLogger(name) for name in ("LiteLLM", "LiteLLM Router", "LiteLLM Proxy"))
|
||||
original_levels: Final = tuple(lg.level for lg in loggers)
|
||||
for lg in loggers:
|
||||
lg.addHandler(collector)
|
||||
lg.setLevel(logging.DEBUG)
|
||||
try:
|
||||
response: Final = client_no_auth.post(route, json=body)
|
||||
finally:
|
||||
for lg, level in zip(loggers, original_levels):
|
||||
lg.removeHandler(collector)
|
||||
lg.setLevel(level)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
body_dumps: Final = [message for message in collector.messages if _DEBUG_LOG_SENTINEL in message]
|
||||
assert len(body_dumps) == 2, [message[:120] for message in body_dumps]
|
||||
assert body_dumps[0].startswith("Request received by LiteLLM:")
|
||||
assert "POST Request Sent from LiteLLM" in body_dumps[1]
|
||||
|
|
|
|||
|
|
@ -111,6 +111,8 @@ def test_media_normalization_preserves_formats_references_and_metadata(kind: str
|
|||
"keep",
|
||||
),
|
||||
([{"type": "input_image", "file_id": "file-only"}], '[{"type": "input_image", "file_id": "file-only"}]'),
|
||||
({"ids": {7}}, "{'ids': {7}}"),
|
||||
([{"type": "unknown", "ids": {7}}], "[{'type': 'unknown', 'ids': {7}}]"),
|
||||
],
|
||||
)
|
||||
def test_tool_output_without_usable_media_keeps_its_existing_fallback(output: object, expected: str) -> None:
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ before response.completed, and that every event of a bridged stream carries the
|
|||
spend tracking stores, so a follow-up previous_response_id still finds the conversation.
|
||||
"""
|
||||
|
||||
import itertools
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from typing import Final
|
||||
|
|
@ -23,6 +24,7 @@ from litellm.responses.litellm_completion_transformation.streaming_iterator impo
|
|||
from litellm.responses.utils import ResponsesAPIRequestUtils
|
||||
from litellm.types.llms.openai import (
|
||||
BaseLiteLLMOpenAIResponseObject,
|
||||
OutputTextDeltaEvent,
|
||||
ResponsesAPIStreamEvents,
|
||||
)
|
||||
from litellm.types.responses.main import build_web_search_call
|
||||
|
|
@ -1658,3 +1660,34 @@ async def test_a_compaction_block_without_its_summary_streams_no_compaction_item
|
|||
|
||||
assert [event["item"]["type"] for event in events if "item" in event] == ["message", "message"]
|
||||
assert [item["type"] for item in events[-1]["response"]["output"]] == ["message"]
|
||||
|
||||
|
||||
def test_the_partial_response_billed_on_disconnect_carries_the_logging_objects_bill_as_hidden_response_cost() -> None:
|
||||
from litellm.litellm_core_utils.hidden_params import get_hidden_params
|
||||
|
||||
wrapper: Final = _FakeStreamWrapper([_chunk("Hello"), _chunk(" world", finish_reason="stop")])
|
||||
wrapper.logging_obj.response_cost_calculator.return_value = 0.00032
|
||||
iterator: Final = LiteLLMCompletionStreamingIterator(
|
||||
model="claude-haiku-4-5",
|
||||
litellm_custom_stream_wrapper=wrapper, # pyright: ignore[reportArgumentType] # the fake stands in for CustomStreamWrapper, as in _build_iterator
|
||||
request_input="What is the weather in San Francisco?",
|
||||
responses_api_request={},
|
||||
custom_llm_provider="anthropic",
|
||||
litellm_metadata={},
|
||||
)
|
||||
relayed: Final = tuple(
|
||||
itertools.takewhile(lambda event: not isinstance(event, OutputTextDeltaEvent), iterator)
|
||||
)
|
||||
assert relayed, "the stream must relay at least one chunk before the client goes away"
|
||||
|
||||
partial: Final = iterator.partial_response_on_disconnect()
|
||||
|
||||
assert partial is not None
|
||||
hidden: Final = get_hidden_params(partial.response)
|
||||
assert hidden is not None and hidden["response_cost"] == 0.00032
|
||||
priced: Final = wrapper.logging_obj.response_cost_calculator.call_args.kwargs["result"]
|
||||
assert priced.id == partial.response.id
|
||||
assert (priced.usage.prompt_tokens, priced.usage.completion_tokens) == (
|
||||
partial.response.usage.input_tokens,
|
||||
partial.response.usage.output_tokens,
|
||||
), "the bill is priced on the partial's own usage"
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ from litellm.exceptions import MidStreamFallbackError
|
|||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
|
||||
from litellm.responses.streaming_iterator import(
|
||||
from litellm.responses.streaming_iterator import (
|
||||
CachedResponsesAPIStreamingIterator,
|
||||
MockResponsesAPIStreamingIterator,
|
||||
ResponsesAPIStreamingIterator,
|
||||
|
|
@ -845,25 +845,33 @@ def test_stamp_responses_usage_cost_stamps_computed_cost():
|
|||
|
||||
response = _responses_api_response_with_usage()
|
||||
logging_obj = Mock(spec=LiteLLMLoggingObj)
|
||||
logging_obj.model_call_details = {"custom_llm_provider": "openai"}
|
||||
logging_obj.response_cost_calculator.return_value = 0.000704
|
||||
|
||||
stamp_responses_usage_cost(response, logging_obj)
|
||||
|
||||
assert getattr(response.usage, "cost", None) == pytest.approx(0.000704)
|
||||
logging_obj.response_cost_calculator.assert_called_once_with(result=response)
|
||||
priced = logging_obj.response_cost_calculator.call_args.kwargs["result"]
|
||||
assert priced.usage.prompt_tokens == response.usage.input_tokens, (
|
||||
"priced in the chat usage shape the spend row uses"
|
||||
)
|
||||
|
||||
|
||||
def test_stamp_responses_usage_cost_keeps_provider_reported_cost():
|
||||
def test_stamp_responses_usage_cost_bills_the_spend_row_price_over_the_provider_reported_cost():
|
||||
from litellm.litellm_core_utils.core_helpers import response_cost_in_hidden_params
|
||||
from litellm.responses.streaming_iterator import stamp_responses_usage_cost
|
||||
|
||||
response = _responses_api_response_with_usage()
|
||||
setattr(response.usage, "cost", 0.5)
|
||||
logging_obj = Mock(spec=LiteLLMLoggingObj)
|
||||
logging_obj.model_call_details = {"custom_llm_provider": "openai"}
|
||||
logging_obj.response_cost_calculator.return_value = 0.00032
|
||||
|
||||
stamp_responses_usage_cost(response, logging_obj)
|
||||
|
||||
assert getattr(response.usage, "cost", None) == pytest.approx(0.5)
|
||||
logging_obj.response_cost_calculator.assert_not_called()
|
||||
assert getattr(response.usage, "cost", None) == pytest.approx(0.00032), "the client sees what the row bills"
|
||||
priced = logging_obj.response_cost_calculator.call_args.kwargs["result"]
|
||||
assert response_cost_in_hidden_params(priced) == pytest.approx(0.5), "the reported cost reaches the calculator"
|
||||
|
||||
|
||||
def _unvalidated_response_with_dict_usage(usage: dict) -> ResponsesAPIResponse:
|
||||
|
|
@ -881,6 +889,7 @@ def _unvalidated_response_with_dict_usage(usage: dict) -> ResponsesAPIResponse:
|
|||
|
||||
def test_stamp_responses_usage_cost_keeps_provider_cost_from_dict_usage():
|
||||
from litellm.responses.streaming_iterator import stamp_responses_usage_cost
|
||||
|
||||
response = _unvalidated_response_with_dict_usage(
|
||||
{
|
||||
"input_tokens": 29,
|
||||
|
|
@ -891,26 +900,29 @@ def test_stamp_responses_usage_cost_keeps_provider_cost_from_dict_usage():
|
|||
}
|
||||
)
|
||||
logging_obj = Mock(spec=LiteLLMLoggingObj)
|
||||
logging_obj.model_call_details = {"custom_llm_provider": "openai"}
|
||||
logging_obj.response_cost_calculator.return_value = 3e-05
|
||||
|
||||
stamp_responses_usage_cost(response, logging_obj)
|
||||
|
||||
assert isinstance(response.usage, ResponseAPIUsage)
|
||||
assert response.usage.cost == pytest.approx(3e-05)
|
||||
assert response.usage.output_tokens_details.reasoning_tokens == 117
|
||||
logging_obj.response_cost_calculator.assert_not_called()
|
||||
|
||||
|
||||
def test_stamp_responses_usage_cost_computes_cost_for_dict_usage_without_cost():
|
||||
from litellm.responses.streaming_iterator import stamp_responses_usage_cost
|
||||
|
||||
response = _unvalidated_response_with_dict_usage({"input_tokens": 29, "output_tokens": 120, "total_tokens": 149})
|
||||
logging_obj = Mock(spec=LiteLLMLoggingObj)
|
||||
logging_obj.model_call_details = {"custom_llm_provider": "openai"}
|
||||
logging_obj.response_cost_calculator.return_value = 0.000704
|
||||
|
||||
stamp_responses_usage_cost(response, logging_obj)
|
||||
|
||||
assert isinstance(response.usage, ResponseAPIUsage)
|
||||
assert response.usage.cost == pytest.approx(0.000704)
|
||||
logging_obj.response_cost_calculator.assert_called_once_with(result=response)
|
||||
logging_obj.response_cost_calculator.assert_called_once()
|
||||
|
||||
|
||||
def test_stamp_responses_usage_cost_survives_calculator_failure():
|
||||
|
|
@ -918,6 +930,7 @@ def test_stamp_responses_usage_cost_survives_calculator_failure():
|
|||
|
||||
response = _responses_api_response_with_usage()
|
||||
logging_obj = Mock(spec=LiteLLMLoggingObj)
|
||||
logging_obj.model_call_details = {"custom_llm_provider": "openai"}
|
||||
logging_obj.response_cost_calculator.side_effect = RuntimeError("cost map unavailable")
|
||||
|
||||
stamp_responses_usage_cost(response, logging_obj)
|
||||
|
|
@ -1207,9 +1220,7 @@ async def test_completed_event_survives_a_failing_usage_estimate():
|
|||
falls back to usage None."""
|
||||
malformed_input: Final = [{"type": "message", "role": "user", "content": 42}]
|
||||
with pytest.raises(ValueError, match="Invalid content type"):
|
||||
_estimate_usage_from_text(
|
||||
"gpt-4o-mini", malformed_input, {"input": malformed_input}, "hello world"
|
||||
)
|
||||
_estimate_usage_from_text("gpt-4o-mini", malformed_input, {"input": malformed_input}, "hello world")
|
||||
|
||||
response = _responses_api_response_without_usage()
|
||||
iterator = _make_iterator(
|
||||
|
|
@ -1305,7 +1316,14 @@ async def test_completed_event_with_a_dict_response_is_typed_and_billed():
|
|||
assert usage.input_tokens > 0
|
||||
assert usage.output_tokens > 0
|
||||
assert usage.cost == pytest.approx(0.000704)
|
||||
logging_obj.response_cost_calculator.assert_any_call(result=completed_response)
|
||||
priced: Final = [
|
||||
call.kwargs["result"]
|
||||
for call in logging_obj.response_cost_calculator.call_args_list
|
||||
if isinstance(call.kwargs.get("result"), ResponsesAPIResponse)
|
||||
]
|
||||
assert [(r.id, r.usage.prompt_tokens) for r in priced] == [(completed_response.id, usage.input_tokens)], (
|
||||
"priced once, in the chat usage shape the spend row uses"
|
||||
)
|
||||
|
||||
|
||||
def test_billed_terminal_response_keeps_a_response_that_already_has_usage():
|
||||
|
|
@ -1375,6 +1393,7 @@ def _vcr_outcome_gate(request, vcr):
|
|||
yield
|
||||
record_vcr_outcome(request, vcr)
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def setup_and_teardown():
|
||||
"""
|
||||
|
|
@ -1392,6 +1411,7 @@ def setup_and_teardown():
|
|||
loop.close()
|
||||
asyncio.set_event_loop(None)
|
||||
|
||||
|
||||
class _FakeLoggingObj:
|
||||
def __init__(self):
|
||||
self.success_calls = 0
|
||||
|
|
@ -1432,10 +1452,12 @@ class _FakeLoggingObj:
|
|||
self.completion_start_time = completion_start_time
|
||||
self.model_call_details["completion_start_time"] = completion_start_time
|
||||
|
||||
|
||||
async def _await_tasks_spawned_since(existing: set["asyncio.Task[object]"]) -> None:
|
||||
spawned: Final = asyncio.all_tasks() - existing
|
||||
await asyncio.wait_for(asyncio.gather(*spawned, return_exceptions=True), timeout=5)
|
||||
|
||||
|
||||
def _make_completed_response(response_id: str = "resp_test") -> ResponseCompletedEvent:
|
||||
return ResponseCompletedEvent(
|
||||
type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED,
|
||||
|
|
@ -1463,6 +1485,7 @@ def _make_completed_response(response_id: str = "resp_test") -> ResponseComplete
|
|||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
@pytest.mark.asyncio
|
||||
async def test_log_background_task_failure_logs_task_exceptions(monkeypatch):
|
||||
|
|
@ -1485,6 +1508,7 @@ async def test_log_background_task_failure_logs_task_exceptions(monkeypatch):
|
|||
task.exception(),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
@pytest.mark.asyncio
|
||||
async def test_log_background_task_failure_ignores_cancelled_tasks(monkeypatch):
|
||||
|
|
@ -1500,6 +1524,7 @@ async def test_log_background_task_failure_ignores_cancelled_tasks(monkeypatch):
|
|||
|
||||
error_logger.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
def test_content_part_done_event_supports_refusal_and_reasoning_text():
|
||||
refusal_event = streaming_module._build_content_part_done_event(
|
||||
|
|
@ -1527,6 +1552,7 @@ def test_content_part_done_event_supports_refusal_and_reasoning_text():
|
|||
assert reasoning_event.part.reasoning == "because"
|
||||
assert unsupported_event is None
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
def test_dump_response_object_handles_model_and_unknown_values():
|
||||
response = ResponsesAPIResponse(
|
||||
|
|
@ -1542,6 +1568,7 @@ def test_dump_response_object_handles_model_and_unknown_values():
|
|||
assert streaming_module._dump_response_object({"type": "message"}) == {"type": "message"}
|
||||
assert streaming_module._dump_response_object(object()) == {}
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_streaming_triggers_hooks(monkeypatch):
|
||||
|
|
@ -1598,6 +1625,7 @@ async def test_responses_streaming_triggers_hooks(monkeypatch):
|
|||
assert seen["request_data"].get("litellm_params") is not None
|
||||
assert seen["call_type"] == CallTypes.responses
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_streaming_calls_post_streaming_deployment_hook(monkeypatch):
|
||||
|
|
@ -1638,6 +1666,7 @@ async def test_responses_streaming_calls_post_streaming_deployment_hook(monkeypa
|
|||
# reset callbacks
|
||||
litellm.callbacks = original_callbacks
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_streaming_failure_triggers_failure_handlers():
|
||||
|
|
@ -1667,6 +1696,7 @@ async def test_responses_streaming_failure_triggers_failure_handlers():
|
|||
assert logging_obj.failure_calls >= 1
|
||||
assert logging_obj.async_failure_calls >= 1
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
def test_process_chunk_requires_provider_config():
|
||||
iterator = ResponsesAPIStreamingIterator(
|
||||
|
|
@ -1681,6 +1711,7 @@ def test_process_chunk_requires_provider_config():
|
|||
with pytest.raises(ValueError, match="responses_api_provider_config is required"):
|
||||
iterator._process_chunk(json.dumps({"type": "response.completed"}))
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
def test_process_chunk_wraps_encrypted_content_with_model_id():
|
||||
openai_types = streaming_module._get_openai_response_types()
|
||||
|
|
@ -1715,6 +1746,7 @@ def test_process_chunk_wraps_encrypted_content_with_model_id():
|
|||
assert event.item.encrypted_content.startswith("litellm_enc:")
|
||||
assert event.item.encrypted_content.endswith(";ciphertext")
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
def test_process_chunk_completed_response_updates_id_and_usage_cost(monkeypatch):
|
||||
original_include_cost = litellm.include_cost_in_streaming_usage
|
||||
|
|
@ -1768,6 +1800,7 @@ def test_process_chunk_completed_response_updates_id_and_usage_cost(monkeypatch)
|
|||
assert event.response.usage.cost == 1.23
|
||||
completion_handler.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
def test_process_chunk_failed_response_triggers_failure_logging(monkeypatch):
|
||||
openai_types = streaming_module._get_openai_response_types()
|
||||
|
|
@ -1803,6 +1836,7 @@ def test_process_chunk_failed_response_triggers_failure_logging(monkeypatch):
|
|||
assert iterator.completed_response is event
|
||||
failure_handler.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_logging_failed_response_uses_response_error_message():
|
||||
|
|
@ -1835,6 +1869,7 @@ async def test_handle_logging_failed_response_uses_response_error_message():
|
|||
assert logging_obj.failure_calls == 1
|
||||
assert logging_obj.async_failure_calls == 1
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
def test_process_chunk_returns_none_for_invalid_json_and_non_dict_payload():
|
||||
class _NoopConfig:
|
||||
|
|
@ -1853,6 +1888,7 @@ def test_process_chunk_returns_none_for_invalid_json_and_non_dict_payload():
|
|||
assert iterator._process_chunk("not-json") is None
|
||||
assert iterator._process_chunk(json.dumps(["not", "a", "dict"])) is None
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
def test_process_chunk_cost_annotation_failure_is_nonfatal(monkeypatch):
|
||||
original_include_cost = litellm.include_cost_in_streaming_usage
|
||||
|
|
@ -1900,6 +1936,7 @@ def test_process_chunk_cost_annotation_failure_is_nonfatal(monkeypatch):
|
|||
assert event.response.usage.cost is None
|
||||
completion_handler.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
def test_get_completed_response_object_accepts_direct_response():
|
||||
logging_obj = _FakeLoggingObj()
|
||||
|
|
@ -1916,6 +1953,7 @@ def test_get_completed_response_object_accepts_direct_response():
|
|||
|
||||
assert iterator._get_completed_response_object() is direct_response
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_streaming_completed_event_persists_async_cache():
|
||||
|
|
@ -1964,6 +2002,7 @@ async def test_responses_streaming_completed_event_persists_async_cache():
|
|||
assert json.loads(litellm.cache.async_add_cache.call_args.args[0])["id"] == iterator.completed_response.response.id
|
||||
litellm.cache = original_cache
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
def test_responses_streaming_completed_event_persists_sync_cache():
|
||||
logging_obj = _FakeLoggingObj()
|
||||
|
|
@ -2009,6 +2048,7 @@ def test_responses_streaming_completed_event_persists_sync_cache():
|
|||
assert json.loads(litellm.cache.add_cache.call_args.args[0])["id"] == iterator.completed_response.response.id
|
||||
litellm.cache = original_cache
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
def test_log_completed_response_sync_direct_path(monkeypatch):
|
||||
hook_calls = {"post_call": 0, "metadata": 0}
|
||||
|
|
@ -2050,6 +2090,7 @@ def test_log_completed_response_sync_direct_path(monkeypatch):
|
|||
assert hook_calls["post_call"] == 1
|
||||
assert hook_calls["metadata"] == 1
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
def test_log_completed_response_falls_back_when_model_validate_fails(monkeypatch):
|
||||
class _BadSerializableResponse:
|
||||
|
|
@ -2079,6 +2120,7 @@ def test_log_completed_response_falls_back_when_model_validate_fails(monkeypatch
|
|||
assert logging_obj.success_calls == 1
|
||||
assert logging_obj.async_success_calls == 1
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
@pytest.mark.parametrize(
|
||||
"scenario",
|
||||
|
|
@ -2143,6 +2185,7 @@ def test_persist_completed_response_to_cache_guard_branches(monkeypatch, scenari
|
|||
expected_cached_flag = scenario == "already_cached"
|
||||
assert iterator._completed_response_cached is expected_cached_flag
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
def test_build_synthetic_response_events_covers_annotations_function_calls_and_refusals():
|
||||
original_include_cost = litellm.include_cost_in_streaming_usage
|
||||
|
|
@ -2201,6 +2244,7 @@ def test_build_synthetic_response_events_covers_annotations_function_calls_and_r
|
|||
assert "response.function_call_arguments.done" in event_types
|
||||
assert event_types[-1] == "response.completed"
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
@pytest.mark.asyncio
|
||||
async def test_mock_responses_streaming_iterator_async_iteration_logs_completion(
|
||||
|
|
@ -2251,6 +2295,7 @@ async def test_mock_responses_streaming_iterator_async_iteration_logs_completion
|
|||
assert hook_calls["post_call"] == 1
|
||||
assert hook_calls["metadata"] == 1
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
def test_mock_responses_streaming_iterator_sync_iteration_logs_completion(monkeypatch):
|
||||
hook_calls = {"post_call": 0, "metadata": 0}
|
||||
|
|
@ -2296,6 +2341,7 @@ def test_mock_responses_streaming_iterator_sync_iteration_logs_completion(monkey
|
|||
assert hook_calls["post_call"] == 1
|
||||
assert hook_calls["metadata"] == 1
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
@pytest.mark.asyncio
|
||||
async def test_cached_responses_stream_async_hit_triggers_success_callbacks(
|
||||
|
|
@ -2355,6 +2401,7 @@ async def test_cached_responses_stream_async_hit_triggers_success_callbacks(
|
|||
litellm.cache.add_cache.assert_not_called()
|
||||
litellm.cache = original_cache
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
def test_cached_responses_stream_sync_hit_triggers_success_callbacks(monkeypatch):
|
||||
hook_calls = {"post_call": 0, "metadata": 0}
|
||||
|
|
@ -2453,7 +2500,9 @@ def test_faked_stream_bridged_to_chat_sends_tool_arguments_once():
|
|||
tool_call_deltas: Final = tuple(
|
||||
tool_call
|
||||
for chunk in chunks
|
||||
for tool_call in (chunk.choices[0].delta.tool_calls or ()) # comprehension-ok: flattening each chunk's tool calls
|
||||
for tool_call in (
|
||||
chunk.choices[0].delta.tool_calls or ()
|
||||
) # comprehension-ok: flattening each chunk's tool calls
|
||||
)
|
||||
joined_arguments: Final = "".join(tool_call.function.arguments or "" for tool_call in tool_call_deltas)
|
||||
joined_text: Final = "".join(chunk.choices[0].delta.content or "" for chunk in chunks)
|
||||
|
|
@ -2509,7 +2558,6 @@ def test_faked_stream_opens_items_in_progress_and_empty_like_a_real_stream():
|
|||
assert [part.text for part in added_parts] == [""]
|
||||
|
||||
|
||||
|
||||
def _answer_with_reasoning_refusal_and_custom_tool_call() -> ResponsesAPIResponse:
|
||||
return ResponsesAPIResponse(
|
||||
id="resp_synthetic",
|
||||
|
|
@ -2643,7 +2691,9 @@ async def test_shutdown_drain_waits_for_a_pending_logging_task():
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_shutdown_drain_gives_up_on_a_stuck_logging_task_at_its_timeout():
|
||||
stuck: Final = streaming_module.spawn_logging_task(asyncio.get_running_loop(), asyncio.Event().wait(), task_name="stuck")
|
||||
stuck: Final = streaming_module.spawn_logging_task(
|
||||
asyncio.get_running_loop(), asyncio.Event().wait(), task_name="stuck"
|
||||
)
|
||||
|
||||
await asyncio.wait_for(streaming_module.drain_logging_tasks(timeout=0.05), timeout=1)
|
||||
|
||||
|
|
|
|||
|
|
@ -87,7 +87,7 @@ def assert_native_request(
|
|||
def native_response(status: int) -> bytes:
|
||||
if status == 429:
|
||||
return b'{"error":"native-rate-limit"}'
|
||||
return b'{"output":{"message":{"content":[{"text":"native-transcription"}]}}}'
|
||||
return b'{"output":{"message":{"content":[{"text":"native-transcription"}]}},"usage":{"inputTokens":390,"outputTokens":60}}'
|
||||
|
||||
|
||||
def load_native(native_path: Path) -> object:
|
||||
|
|
@ -130,6 +130,9 @@ def assert_success(route: str, response: object) -> None:
|
|||
raise TypeError(f"{route} returned {type(response).__name__}, expected dict")
|
||||
if response["text"] != "native-transcription":
|
||||
raise AssertionError(f"{route} returned {response['text']!r}, expected 'native-transcription'")
|
||||
expected_usage: Final = {"type": "tokens", "input_tokens": 390, "output_tokens": 60, "total_tokens": 450}
|
||||
if response.get("usage") != expected_usage:
|
||||
raise AssertionError(f"{route} returned usage {response.get('usage')!r}, expected {expected_usage!r}")
|
||||
|
||||
|
||||
def assert_rate_limit(route: str, error: BaseException) -> None:
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ from litellm.rust_bridge import configuration
|
|||
from litellm.rust_bridge._native import RustBridgeDeclined, RustUpstreamError
|
||||
from litellm.rust_bridge.public_call import NativeCall
|
||||
from litellm.rust_bridge.transcription.native import NATIVE_ATRANSCRIPTION, NATIVE_TRANSCRIPTION
|
||||
from litellm.types.utils import TranscriptionUsageTokensObject
|
||||
|
||||
MODEL: Final = "bedrock/mistral.voxtral-mini-3b-2507"
|
||||
AUDIO_FILE: Final = ("audio.wav", b"audio", "audio/wav")
|
||||
|
|
@ -59,6 +60,14 @@ class AsyncBridge:
|
|||
return {"text": "async rust"}
|
||||
|
||||
|
||||
class UsageBridge:
|
||||
def __call__(self, call: NativeCall) -> dict[str, object]:
|
||||
return {
|
||||
"text": "rust",
|
||||
"usage": {"type": "tokens", "input_tokens": 390, "output_tokens": 60, "total_tokens": 450},
|
||||
}
|
||||
|
||||
|
||||
def dispatch_sync() -> litellm.TranscriptionResponse:
|
||||
return BedrockAudioTranscriptionRustDispatch().audio_transcriptions(
|
||||
model=MODEL,
|
||||
|
|
@ -196,3 +205,99 @@ async def test_async_missing_native_binding_raises_without_python_fallback() ->
|
|||
optional_params={},
|
||||
timeout=None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("region_kwargs", "region_env", "priced_model"),
|
||||
[
|
||||
({}, {}, MODEL),
|
||||
({"aws_region_name": "ap-southeast-2"}, {}, "bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507"),
|
||||
({}, {"AWS_REGION_NAME": "eu-west-2"}, "bedrock/eu-west-2/mistral.voxtral-mini-3b-2507"),
|
||||
({}, {"AWS_REGION": "ap-northeast-1"}, "bedrock/ap-northeast-1/mistral.voxtral-mini-3b-2507"),
|
||||
(
|
||||
{"aws_region_name": "ap-southeast-2"},
|
||||
{"AWS_REGION_NAME": "eu-west-2"},
|
||||
"bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_bedrock_transcription_bills_the_converse_token_usage(
|
||||
region_kwargs: dict[str, str],
|
||||
region_env: dict[str, str],
|
||||
priced_model: str,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
for name in ("AWS_REGION_NAME", "AWS_REGION"):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
for name, value in region_env.items():
|
||||
monkeypatch.setenv(name, value)
|
||||
NATIVE_TRANSCRIPTION.override(UsageBridge())
|
||||
|
||||
response: Final = litellm.transcription(model=MODEL, file=AUDIO_FILE, **region_kwargs)
|
||||
|
||||
model_info: Final = litellm.get_model_info(priced_model)
|
||||
expected_cost: Final = 390 * (model_info.get("input_cost_per_token") or 0.0) + 60 * (
|
||||
model_info.get("output_cost_per_token") or 0.0
|
||||
)
|
||||
assert response.usage == TranscriptionUsageTokensObject(
|
||||
type="tokens", input_tokens=390, output_tokens=60, total_tokens=450
|
||||
)
|
||||
assert expected_cost > 0
|
||||
assert litellm.completion_cost(
|
||||
completion_response=response, model=MODEL, call_type="transcription"
|
||||
) == pytest.approx(expected_cost)
|
||||
|
||||
|
||||
def test_bedrock_transcription_forwards_language_and_text_formats_to_rust() -> None:
|
||||
bridge: Final = SyncBridge()
|
||||
NATIVE_TRANSCRIPTION.override(bridge)
|
||||
|
||||
litellm.transcription(model=MODEL, file=AUDIO_FILE, language="fr", response_format="text", prompt="Names")
|
||||
|
||||
forwarded: Final = bridge.calls[0].resolved["optional_params"]
|
||||
assert {key: forwarded.get(key) for key in ("language", "response_format", "prompt")} == {
|
||||
"language": "fr",
|
||||
"response_format": "text",
|
||||
"prompt": "Names",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("sent", "expected"), [("0.2", 0.2), ("0", 0.0), (0.5, 0.5), (1, 1.0)])
|
||||
def test_bedrock_transcription_forwards_form_temperature_as_a_number(sent: object, expected: float) -> None:
|
||||
bridge: Final = SyncBridge()
|
||||
NATIVE_TRANSCRIPTION.override(bridge)
|
||||
|
||||
litellm.transcription(model=MODEL, file=AUDIO_FILE, temperature=sent)
|
||||
|
||||
forwarded: Final = bridge.calls[0].resolved["optional_params"]["temperature"]
|
||||
assert (type(forwarded), forwarded) == (float, expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"unsupported",
|
||||
[
|
||||
{"response_format": "srt"},
|
||||
{"response_format": "verbose_json"},
|
||||
{"timestamp_granularities": ["word"]},
|
||||
{"temperature": "warm"},
|
||||
{"temperature": "inf"},
|
||||
{"temperature": True},
|
||||
],
|
||||
)
|
||||
def test_bedrock_transcription_rejects_params_it_cannot_honor(unsupported: dict[str, object]) -> None:
|
||||
bridge: Final = SyncBridge()
|
||||
NATIVE_TRANSCRIPTION.override(bridge)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError) as raised:
|
||||
litellm.transcription(model=MODEL, file=AUDIO_FILE, language="fr", **unsupported)
|
||||
|
||||
assert raised.value.status_code == 400
|
||||
assert next(iter(unsupported)) in str(raised.value)
|
||||
assert bridge.calls == ()
|
||||
|
||||
dropped: Final = litellm.transcription(model=MODEL, file=AUDIO_FILE, language="fr", drop_params=True, **unsupported)
|
||||
|
||||
assert dropped.text == "rust"
|
||||
forwarded: Final = bridge.calls[0].resolved["optional_params"]
|
||||
assert forwarded["language"] == "fr"
|
||||
assert forwarded.keys().isdisjoint(unsupported)
|
||||
|
|
|
|||
|
|
@ -7,9 +7,11 @@ regardless of the routing strategy being used.
|
|||
|
||||
import asyncio
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
|
|
@ -724,3 +726,159 @@ class TestDailyLimits:
|
|||
router.discard()
|
||||
|
||||
assert served == ["a", "b", "b", "b"]
|
||||
|
||||
|
||||
_LOGGED_TOKENS: Final = TypeAdapter[float | None](float | None)
|
||||
|
||||
|
||||
async def _tpd_tokens_once_logged(cache: DualCache, key: str) -> int:
|
||||
for _ in range(200):
|
||||
logged: Final = _LOGGED_TOKENS.validate_python(
|
||||
cache.in_memory_cache.get_cache(key) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # InMemoryCache.get_cache is untyped
|
||||
)
|
||||
if logged is not None:
|
||||
return int(logged)
|
||||
await asyncio.sleep(0.01)
|
||||
pytest.fail(f"the success callback never logged tpd tokens under {key}")
|
||||
|
||||
|
||||
RESET_AT_EIGHT = datetime(2026, 10, 28, 8, 0, 0, tzinfo=timezone.utc)
|
||||
DAILY_RESET_DEPLOYMENT = {
|
||||
**RPD_DEPLOYMENT,
|
||||
"litellm_params": {"model": "gpt-4.1-mini", "rpd": 2, "daily_reset_time": "08:00"},
|
||||
}
|
||||
|
||||
|
||||
class TestDailyResetTime:
|
||||
def test_rpd_window_starts_over_at_daily_reset_time_not_midnight(self):
|
||||
clock = Clock(RESET_AT_EIGHT - timedelta(hours=9))
|
||||
check = ModelRateLimitingCheck(dual_cache=DualCache(), clock=clock)
|
||||
|
||||
assert _pre_call(check, DAILY_RESET_DEPLOYMENT) == DAILY_RESET_DEPLOYMENT
|
||||
clock.now = RESET_AT_EIGHT - timedelta(hours=7)
|
||||
assert _pre_call(check, DAILY_RESET_DEPLOYMENT) == DAILY_RESET_DEPLOYMENT
|
||||
clock.now = RESET_AT_EIGHT - timedelta(minutes=1)
|
||||
with pytest.raises(litellm.RateLimitError, match="RPD limit=2, current usage=3") as exc_info:
|
||||
_pre_call(check, DAILY_RESET_DEPLOYMENT)
|
||||
assert exc_info.value.response.headers["retry-after"] == "60"
|
||||
|
||||
clock.now = RESET_AT_EIGHT
|
||||
assert _pre_call(check, DAILY_RESET_DEPLOYMENT) == DAILY_RESET_DEPLOYMENT
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_rpd_window_starts_over_at_daily_reset_time_not_midnight(self):
|
||||
clock = Clock(RESET_AT_EIGHT - timedelta(hours=9))
|
||||
check = ModelRateLimitingCheck(dual_cache=DualCache(), clock=clock)
|
||||
|
||||
assert await _async_pre_call(check, DAILY_RESET_DEPLOYMENT) == DAILY_RESET_DEPLOYMENT
|
||||
clock.now = RESET_AT_EIGHT - timedelta(hours=7)
|
||||
assert await _async_pre_call(check, DAILY_RESET_DEPLOYMENT) == DAILY_RESET_DEPLOYMENT
|
||||
clock.now = RESET_AT_EIGHT - timedelta(minutes=1)
|
||||
with pytest.raises(litellm.RateLimitError, match="RPD limit=2, current usage=3") as exc_info:
|
||||
await _async_pre_call(check, DAILY_RESET_DEPLOYMENT)
|
||||
assert exc_info.value.response.headers["retry-after"] == "60"
|
||||
|
||||
clock.now = RESET_AT_EIGHT
|
||||
assert await _async_pre_call(check, DAILY_RESET_DEPLOYMENT) == DAILY_RESET_DEPLOYMENT
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("now", "seconds_to_reset"),
|
||||
[
|
||||
(RESET_AT_EIGHT + timedelta(seconds=1), 86399),
|
||||
(datetime(2026, 10, 28, 23, 59, 30, tzinfo=timezone.utc), 28830),
|
||||
(datetime(2026, 10, 29, 7, 30, 0, tzinfo=timezone.utc), 1800),
|
||||
],
|
||||
)
|
||||
def test_daily_429_retry_after_is_seconds_until_daily_reset_time(self, now: datetime, seconds_to_reset: int):
|
||||
deployment = {
|
||||
**RPD_DEPLOYMENT,
|
||||
"litellm_params": {"model": "gpt-4.1-mini", "rpd": 0, "daily_reset_time": "08:00"},
|
||||
}
|
||||
check = ModelRateLimitingCheck(dual_cache=DualCache(), clock=Clock(now))
|
||||
|
||||
with pytest.raises(litellm.RateLimitError) as exc_info:
|
||||
_pre_call(check, deployment)
|
||||
|
||||
assert exc_info.value.response.headers["retry-after"] == str(seconds_to_reset)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tokens_logged_before_daily_reset_time_do_not_count_after_it(self):
|
||||
deployment = {
|
||||
**RPD_DEPLOYMENT,
|
||||
"litellm_params": {"model": "gpt-4.1-mini", "tpd": 100, "daily_reset_time": "08:00"},
|
||||
}
|
||||
clock = Clock(RESET_AT_EIGHT - timedelta(minutes=1))
|
||||
check = ModelRateLimitingCheck(dual_cache=DualCache(), clock=clock)
|
||||
await _async_log_success(
|
||||
check,
|
||||
{
|
||||
"litellm_params": {"daily_reset_time": "08:00", "metadata": {}},
|
||||
"standard_logging_object": {
|
||||
"model_id": "daily-test-id",
|
||||
"total_tokens": 120,
|
||||
"hidden_params": {"litellm_model_name": "gpt-4.1-mini"},
|
||||
},
|
||||
},
|
||||
)
|
||||
with pytest.raises(litellm.RateLimitError, match="TPD limit=100, current usage=120"):
|
||||
await _async_pre_call(check, deployment)
|
||||
|
||||
clock.now = RESET_AT_EIGHT
|
||||
assert await _async_pre_call(check, deployment) == deployment
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"deployment",
|
||||
[
|
||||
{"litellm_params": {"model": "gpt-4.1-mini"}, "model_info": {"id": "x", "rpd": 0}, "model_name": "m"},
|
||||
{"litellm_params": {"model": "gpt-4.1-mini"}, "model_info": {"id": "x"}, "model_name": "m", "rpd": 0},
|
||||
],
|
||||
)
|
||||
def test_rpd_outside_litellm_params_is_not_a_limit(self, deployment: dict[str, object]):
|
||||
check = ModelRateLimitingCheck(dual_cache=DualCache(), clock=Clock(DAY_ONE))
|
||||
|
||||
assert _pre_call(check, deployment) == deployment
|
||||
|
||||
@pytest.mark.parametrize("daily_reset_time", ["8am", "08:00:30", "08:00+05:00", "25:00"])
|
||||
def test_router_rejects_a_daily_reset_time_that_is_not_hh_mm(self, daily_reset_time: str):
|
||||
with pytest.raises(Exception, match="daily_reset_time"):
|
||||
Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "daily-group",
|
||||
"litellm_params": {"model": "gpt-4.1-mini", "rpd": 2, "daily_reset_time": daily_reset_time},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_daily_reset_time_sent_in_the_request_does_not_move_the_tokens_out_of_the_deployment_window(self):
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "daily-group",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4.1-mini",
|
||||
"api_key": "k",
|
||||
"mock_response": "a",
|
||||
"tpd": 100,
|
||||
},
|
||||
"model_info": {"id": "daily-test-id"},
|
||||
}
|
||||
]
|
||||
)
|
||||
check = ModelRateLimitingCheck(dual_cache=router.cache, clock=Clock(RESET_AT_EIGHT - timedelta(minutes=1)))
|
||||
router.optional_callbacks = [check]
|
||||
litellm.logging_callback_manager.add_litellm_callback(check)
|
||||
try:
|
||||
await router.acompletion( # pyright: ignore[reportUnknownMemberType] # Router.acompletion kwargs are untyped
|
||||
model="daily-group", messages=[{"role": "user", "content": "hi"}], daily_reset_time="08:00"
|
||||
)
|
||||
logged = await _tpd_tokens_once_logged(router.cache, "daily-test-id:openai/gpt-4.1-mini:tpd:2026-10-28")
|
||||
finally:
|
||||
router.discard()
|
||||
|
||||
assert logged > 0
|
||||
day_before: Final = _LOGGED_TOKENS.validate_python(
|
||||
router.cache.in_memory_cache.get_cache("daily-test-id:openai/gpt-4.1-mini:tpd:2026-10-27") # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # InMemoryCache.get_cache is untyped
|
||||
)
|
||||
assert day_before is None
|
||||
|
|
|
|||
|
|
@ -10883,3 +10883,23 @@ def test_get_optional_params_forwards_a_tool_config_cache_point_to_a_downstream_
|
|||
)
|
||||
|
||||
assert optional_params["extra_body"] == {"kept": True, "cache_control_injection_points": [point]}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("provider", "expected_warnings"),
|
||||
(("streamlake", 1), ("veniceai", 1), ("my-own-llm", 0)),
|
||||
)
|
||||
def test_custom_provider_named_like_a_built_in_warns_once(
|
||||
provider: str, expected_warnings: int, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "custom_provider_map", [{"provider": provider, "custom_handler": litellm.CustomLLM()}])
|
||||
monkeypatch.setattr(litellm, "provider_list", list(litellm.provider_list))
|
||||
monkeypatch.setattr(litellm, "_custom_providers", [])
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger=verbose_logger.name):
|
||||
litellm.utils.custom_llm_setup()
|
||||
litellm.utils.custom_llm_setup()
|
||||
|
||||
warnings: Final = [r for r in caplog.records if f"provider '{provider}'" in r.getMessage()]
|
||||
assert len(warnings) == expected_warnings
|
||||
assert provider in litellm._custom_providers
|
||||
|
|
|
|||
|
|
@ -156,6 +156,7 @@ OPTION_NAMES: Final = (
|
|||
"tpm",
|
||||
"rpd",
|
||||
"tpd",
|
||||
"daily_reset_time",
|
||||
"itpm",
|
||||
"otpm",
|
||||
"default_api_key_rpm_limit",
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { useSyncExternalStore } from "react";
|
||||
import { act, renderWithProviders, screen, waitFor, within } from "../../../../../tests/test-utils";
|
||||
import { act, fireEvent, renderWithProviders, screen, waitFor, within } from "../../../../../tests/test-utils";
|
||||
import type { OnUrlUpdateFunction } from "nuqs/adapters/testing";
|
||||
import userEvent, { PointerEventsCheckLevel } from "@testing-library/user-event";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
|
@ -80,6 +80,13 @@ vi.mock("@/app/(dashboard)/hooks/providers/useProviderFields", () => ({
|
|||
default_model_placeholder: "jev-latest",
|
||||
credential_fields: [{ key: "api_key", label: "API Key", field_type: "password", required: false }],
|
||||
},
|
||||
{
|
||||
provider: "StreamLake",
|
||||
provider_display_name: "StreamLake",
|
||||
litellm_provider: "streamlake",
|
||||
default_model_placeholder: "streamlake/<model-or-endpoint>",
|
||||
credential_fields: [{ key: "api_key", label: "API Key", field_type: "password", required: false }],
|
||||
},
|
||||
],
|
||||
isLoading: false,
|
||||
error: null,
|
||||
|
|
@ -278,6 +285,28 @@ describe("AddModelPanel submit payload contract", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("creates a StreamLake model from a public model selection and API key", async () => {
|
||||
mockUseModelCostMap.mockReturnValue({ data: { "streamlake/GLM-5.3": { litellm_provider: "streamlake" } } });
|
||||
const { user, submit } = await setup();
|
||||
|
||||
await user.click(screen.getByRole("combobox", { name: /provider/i }));
|
||||
await user.click(await screen.findByText("StreamLake"));
|
||||
await user.click(screen.getByRole("combobox", { name: "Select models" }));
|
||||
await user.click(await screen.findByText("streamlake/GLM-5.3"));
|
||||
fireEvent.change(screen.getByLabelText("API Key"), { target: { value: "sk-streamlake-test" } });
|
||||
await submit();
|
||||
|
||||
expect(lastCreatedModel()).toStrictEqual({
|
||||
model_name: "streamlake/GLM-5.3",
|
||||
litellm_params: {
|
||||
model: "streamlake/GLM-5.3",
|
||||
custom_llm_provider: "streamlake",
|
||||
api_key: "sk-streamlake-test",
|
||||
},
|
||||
model_info: { ...baseModelInfo },
|
||||
});
|
||||
});
|
||||
|
||||
it("registers four more keys as undefined once Advanced Settings opens", async () => {
|
||||
const { openAdvanced, fillRequired, submit } = await setup();
|
||||
await fillRequired();
|
||||
|
|
|
|||
|
|
@ -222,6 +222,7 @@ describe("provider_info_helpers", () => {
|
|||
Providers.PG_VECTOR,
|
||||
Providers.PREDIBASE,
|
||||
Providers.Sail,
|
||||
Providers.StreamLake,
|
||||
Providers.WANDB,
|
||||
Providers.ZAI,
|
||||
Providers.ZeroGPU,
|
||||
|
|
@ -462,6 +463,19 @@ describe("provider_info_helpers", () => {
|
|||
expect(result).toEqual(["gpt-3.5-turbo", "gpt-4"]);
|
||||
});
|
||||
|
||||
it("should return StreamLake models and exclude models from other providers", () => {
|
||||
const modelMap = {
|
||||
"streamlake/GLM-5.3": { litellm_provider: "streamlake" },
|
||||
"streamlake/DeepSeek-V3": { litellm_provider: "streamlake" },
|
||||
"gpt-4o": { litellm_provider: "openai" },
|
||||
};
|
||||
const result = getProviderModels(Providers.StreamLake, modelMap);
|
||||
|
||||
expect(provider_map.StreamLake).toBe("streamlake");
|
||||
expect(result).toEqual(["streamlake/GLM-5.3", "streamlake/DeepSeek-V3"]);
|
||||
expect(result).not.toContain("gpt-4o");
|
||||
});
|
||||
|
||||
it("should return models whose litellm_provider is a prefix-anchored variant of the provider", () => {
|
||||
const modelMap = {
|
||||
"anthropic-text-model": { litellm_provider: "anthropic_text" },
|
||||
|
|
|
|||
|
|
@ -177,6 +177,7 @@ export enum Providers {
|
|||
Snowflake = "Snowflake",
|
||||
Soniox = "Soniox",
|
||||
StrandsDecider = "Strands Decider",
|
||||
StreamLake = "StreamLake",
|
||||
TEXT_COMPLETION_CODESTRAL = "Text-Completion-Codestral",
|
||||
Tencent = "Tencent",
|
||||
TogetherAI = "TogetherAI",
|
||||
|
|
@ -306,6 +307,7 @@ export const provider_map: Record<string, string> = {
|
|||
Snowflake: "snowflake",
|
||||
Soniox: "soniox",
|
||||
StrandsDecider: "strands_decider",
|
||||
StreamLake: "streamlake",
|
||||
TEXT_COMPLETION_CODESTRAL: "text-completion-codestral",
|
||||
Tencent: "tencent",
|
||||
TogetherAI: "together_ai",
|
||||
|
|
@ -494,6 +496,7 @@ const providerPlaceholderMap: Partial<Record<Providers, string>> = {
|
|||
[Providers.Tencent]: "tencent/deepseek-v4-pro",
|
||||
[Providers.TypeSafe]: "typesafe/jev-latest",
|
||||
[Providers.Nace]: "nace/drex-latest",
|
||||
[Providers.StreamLake]: "streamlake/<model-or-endpoint>",
|
||||
[Providers.Vertex_AI]: "gemini-pro",
|
||||
[Providers.VolcEngine]: "volcengine/<any-model-on-volcengine>",
|
||||
[Providers.Voyage]: "voyage/",
|
||||
|
|
|
|||
4
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
4
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -35397,6 +35397,8 @@ export interface components {
|
|||
cost_per_second?: number | null;
|
||||
/** Custom Llm Provider */
|
||||
custom_llm_provider?: string | null;
|
||||
/** Daily Reset Time */
|
||||
daily_reset_time?: string | null;
|
||||
/** Default Api Key Rpm Limit */
|
||||
default_api_key_rpm_limit?: number | null;
|
||||
/** Default Api Key Tpm Limit */
|
||||
|
|
@ -50793,6 +50795,8 @@ export interface components {
|
|||
cost_per_second?: number | null;
|
||||
/** Custom Llm Provider */
|
||||
custom_llm_provider?: string | null;
|
||||
/** Daily Reset Time */
|
||||
daily_reset_time?: string | null;
|
||||
/** Default Api Key Rpm Limit */
|
||||
default_api_key_rpm_limit?: number | null;
|
||||
/** Default Api Key Tpm Limit */
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue