From 57f05e56f6550ee8fa4a1207f2ed1efca277d87b Mon Sep 17 00:00:00 2001 From: David Koh <124323762+david-koh@users.noreply.github.com> Date: Thu, 29 Oct 2026 11:23:15 +0900 Subject: [PATCH 01/10] fix(bedrock): omit system prompt from audio transcription and report its token usage (#41565) * fix(bedrock): omit system prompt from audio transcription * fix(bedrock): report Converse token usage on audio transcription Bedrock transcriptions returned usage null and were billed at $0. Map Converse inputTokens, outputTokens and totalTokens into the transcription usage so spend uses the model's per-token price. input_token_details is optional because Converse does not split audio and text tokens. Also switch the updated request-shape test to rstest. * fix(bedrock): default transcription temperature to 0 and price ap-southeast-2 Voxtral regionally Voxtral wrapped some transcripts in quotes at its default sampling temperature. In direct Converse calls in ap-southeast-2, 0 of 48 calls at temperature 0 were framed against 7 of 72 at the default, and instruction wording made no clear difference, so the request now sends temperature 0 when the caller sets none, matching OpenAI's transcription default. Calls in ap-southeast-2 billed at the us-east-1 price. The cost map gains bedrock/ap-southeast-2 Voxtral rows with all four service tiers, the Bedrock transcription response carries the deployment region in its hidden params, and the transcription token cost uses the region-resolved model key. The bridge now builds the response with model_copy instead of assigning usage after construction, and the native wheel test checks the usage the compiled route returns. * fix(bedrock): forward transcription language, reject unsupported formats, and bill env and Tokyo/London regions get_optional_params_transcription had no Bedrock branch, so language, response_format and timestamp_granularities were dropped before the Rust transform ever saw them. Bedrock now forwards language, prompt and temperature, accepts response_format json or text (the only shapes it can return), and answers 400 for srt, vtt, verbose_json or timestamp_granularities unless drop_params is set. The billing region now falls back to AWS_REGION_NAME and then AWS_REGION when aws_region_name is unset, the same order the Rust resolver uses to pick the endpoint, so an env-configured region gets its regional price. Adds ap-northeast-1 and eu-west-2 Voxtral rows (standard, batch, flex and priority) from the AWS Bedrock offer files, the other two regions AWS serves Voxtral audio in. * fix(bedrock): send transcription temperature as a number and name the rejected param The proxy keeps multipart form values as strings, so -F temperature=0.2 reached Converse as "0.2" and Bedrock answered 400. The Bedrock mapper now converts temperature to a float and answers 400 (or drops it under drop_params) when it is not a finite number. get_optional_params_transcription copied prompt and temperature back from the raw call after the Bedrock branch ran, which undid that conversion and any drop. The Bedrock branch now owns every OpenAI transcription param. An unsupported param such as timestamp_granularities now gets a 400 that names it instead of the generic "user/encoding format" message. * refactor(bedrock): return unsupported transcription params as a value and raise at the param step * fix(bedrock): drop the retired max_tokens field from the regional voxtral price rows --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../formats/audio_transcription/response.rs | 23 +++- .../src/bedrock/audio_transcription/mod.rs | 83 +++++++++---- litellm/cost_calculator.py | 2 +- .../usage_object_transformation.py | 11 +- .../bedrock/audio_transcription/__init__.py | 75 +++++++++++- ...odel_prices_and_context_window_backup.json | 114 ++++++++++++++++++ litellm/types/utils.py | 2 +- litellm/utils.py | 19 ++- model_prices_and_context_window.json | 114 ++++++++++++++++++ .../rust_bridge/native_route_wheel_test.py | 5 +- .../test_audio_transcription_rust_bridge.py | 105 ++++++++++++++++ 11 files changed, 516 insertions(+), 37 deletions(-) diff --git a/litellm-rust/crates/llms-types/src/formats/audio_transcription/response.rs b/litellm-rust/crates/llms-types/src/formats/audio_transcription/response.rs index 624eb7be4e4..211d4d0f648 100644 --- a/litellm-rust/crates/llms-types/src/formats/audio_transcription/response.rs +++ b/litellm-rust/crates/llms-types/src/formats/audio_transcription/response.rs @@ -1,14 +1,31 @@ use serde_json::Value; +#[macro_rules_attribute::apply(crate::wire_type)] +pub struct AudioTranscriptionUsage { + pub input_tokens: u64, + pub output_tokens: u64, + pub total_tokens: u64, +} + #[macro_rules_attribute::apply(crate::wire_type)] pub struct AudioTranscriptionResponseData { pub text: String, + pub usage: Option, } impl AudioTranscriptionResponseData { pub fn into_json(self) -> Value { - serde_json::json!({ - "text": self.text, - }) + match self.usage { + None => serde_json::json!({"text": self.text}), + Some(usage) => serde_json::json!({ + "text": self.text, + "usage": { + "type": "tokens", + "input_tokens": usage.input_tokens, + "output_tokens": usage.output_tokens, + "total_tokens": usage.total_tokens, + }, + }), + } } } diff --git a/litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs b/litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs index 0f8703f223b..1c976c1f73d 100644 --- a/litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs +++ b/litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs @@ -6,8 +6,8 @@ use litellm_auth_aws::{ }; use litellm_core_utils::core_helpers::json_type_name; use litellm_llms_types::{ - formats::audio_transcription::AudioTranscriptionResponseData, - providers::bedrock::ConverseResponse, + formats::audio_transcription::{AudioTranscriptionResponseData, AudioTranscriptionUsage}, + providers::bedrock::{ConverseResponse, ConverseUsage}, }; use serde::Deserialize; use serde_json::{Map, Value, json}; @@ -101,10 +101,10 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig { if let Some(prompt) = optional_string(&optional_params, "prompt") { instruction.push_str(&format!(" Additional context: {prompt}")); } - let mut inference_config = Map::from_iter([("maxTokens".to_string(), json!(4096))]); - if let Some(temperature) = optional_params.get("temperature") { - inference_config.insert("temperature".to_string(), temperature.clone()); - } + let temperature = optional_params + .get("temperature") + .cloned() + .unwrap_or(json!(0)); Ok(AudioTranscriptionRequestData { body: json!({ "messages": [{ @@ -114,8 +114,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig { {"text": instruction} ] }], - "system": [{"text": "You are a transcription assistant."}], - "inferenceConfig": inference_config, + "inferenceConfig": {"maxTokens": 4096, "temperature": temperature}, }), }) } @@ -125,6 +124,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig { _model: &str, response_json: Value, ) -> Result { + let reports_usage = response_json.get("usage").is_some(); let response: ConverseResponse = serde_json::from_value(response_json).map_err(|error| { Error::InvalidResponse(ErrorDetail::invalid("Converse response", error)) @@ -137,6 +137,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig { } Ok(AudioTranscriptionResponseData { text: response.content_text(), + usage: reports_usage.then(|| transcription_usage(&response.usage)), }) } @@ -191,22 +192,41 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig { } } +fn transcription_usage(usage: &ConverseUsage) -> AudioTranscriptionUsage { + AudioTranscriptionUsage { + input_tokens: usage.input_tokens, + output_tokens: usage.output_tokens, + total_tokens: usage + .total_tokens + .unwrap_or(usage.input_tokens + usage.output_tokens), + } +} + #[cfg(test)] mod tests { use super::*; + use rstest::rstest; fn no_env(_: &str) -> Option { None } - #[test] - fn request_matches_python_shape() { - let params = Map::from_iter([ - ("language".to_string(), json!("en")), - ("prompt".to_string(), json!("Speaker names")), - ("temperature".to_string(), json!(0)), - ("timestamp_granularities".to_string(), json!(["word"])), - ]); + #[rstest] + #[case::caller_temperature(Some(json!(0.4)), json!(0.4))] + #[case::default_temperature(None, json!(0))] + fn request_sends_audio_and_instruction_without_system_block( + #[case] temperature: Option, + #[case] expected_temperature: Value, + ) { + let params = Map::from_iter( + [ + ("language".to_string(), json!("en")), + ("prompt".to_string(), json!("Speaker names")), + ("timestamp_granularities".to_string(), json!(["word"])), + ] + .into_iter() + .chain(temperature.map(|value| ("temperature".to_string(), value))), + ); let params = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG.map_transcription_params(¶ms); let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG .transform_audio_transcription_request( @@ -225,22 +245,39 @@ mod tests { {"text": "Transcribe the audio. Respond with only the transcript. The audio language is en. Additional context: Speaker names"} ] }], - "system": [{"text": "You are a transcription assistant."}], - "inferenceConfig": {"maxTokens": 4096, "temperature": 0} + "inferenceConfig": {"maxTokens": 4096, "temperature": expected_temperature} }) ); } - #[test] - fn response_concatenates_content_blocks() { + #[rstest] + #[case::reported_total( + json!({"inputTokens": 390, "outputTokens": 60, "totalTokens": 450}), + json!({"text": "hello world", "usage": {"type": "tokens", "input_tokens": 390, "output_tokens": 60, "total_tokens": 450}}) + )] + #[case::derived_total( + json!({"inputTokens": 1, "outputTokens": 2}), + json!({"text": "hello world", "usage": {"type": "tokens", "input_tokens": 1, "output_tokens": 2, "total_tokens": 3}}) + )] + fn response_carries_text_and_converse_usage(#[case] usage: Value, #[case] expected: Value) { let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG .transform_audio_transcription_response( "model", - json!({"output": {"message": {"content": [{"text": "hello "}, {"text": "world"}]}}, "usage": {"inputTokens": 1, "outputTokens": 2}}), + json!({"output": {"message": {"content": [{"text": "hello "}, {"text": "world"}]}}, "usage": usage}), ) .expect("response"); - assert_eq!(result.text, "hello world"); - assert_eq!(result.into_json(), json!({"text": "hello world"})); + assert_eq!(result.into_json(), expected); + } + + #[rstest] + fn response_without_usage_reports_none() { + let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG + .transform_audio_transcription_response( + "model", + json!({"output": {"message": {"content": [{"text": "hello"}]}}}), + ) + .expect("response"); + assert_eq!(result.into_json(), json!({"text": "hello"})); } #[test] diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 087d5b5b718..b5dc0ce13de 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -688,7 +688,7 @@ def cost_per_token( elif call_type == "atranscription" or call_type == "transcription": if _transcription_usage_has_token_details(usage_block): return generic_cost_per_token( - model=model_without_prefix, + model=model, usage=usage_block, custom_llm_provider=custom_llm_provider, service_tier=service_tier, diff --git a/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py b/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py index a02c40b7611..78a8cb51bac 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py +++ b/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py @@ -27,13 +27,18 @@ class TranscriptionUsageObjectTransformation: if isinstance(usage_object, TranscriptionUsageDurationObject): return None elif isinstance(usage_object, TranscriptionUsageTokensObject): + input_token_details: Final = usage_object.input_token_details return Usage( prompt_tokens=usage_object.input_tokens, completion_tokens=usage_object.output_tokens, total_tokens=usage_object.total_tokens, - prompt_tokens_details=PromptTokensDetailsWrapper( - text_tokens=usage_object.input_token_details.text_tokens, - audio_tokens=usage_object.input_token_details.audio_tokens, + prompt_tokens_details=( + None + if input_token_details is None + else PromptTokensDetailsWrapper( + text_tokens=input_token_details.text_tokens, + audio_tokens=input_token_details.audio_tokens, + ) ), ) return None diff --git a/litellm/llms/bedrock/audio_transcription/__init__.py b/litellm/llms/bedrock/audio_transcription/__init__.py index 7f7a138398b..bc74e9c38ab 100644 --- a/litellm/llms/bedrock/audio_transcription/__init__.py +++ b/litellm/llms/bedrock/audio_transcription/__init__.py @@ -1,7 +1,11 @@ import base64 +import math +from collections.abc import Mapping +from dataclasses import dataclass from typing import Final import httpx +from pydantic import BaseModel, ConfigDict from litellm.litellm_core_utils.audio_utils.utils import process_audio_file from litellm.rust_bridge import runtime @@ -14,7 +18,72 @@ from litellm.rust_bridge.transcription.native import ( RustAtranscription, RustTranscription, ) -from litellm.types.utils import FileTypes, TranscriptionResponse +from litellm.secret_managers.main import get_secret_str +from litellm.types.utils import FileTypes, TranscriptionResponse, TranscriptionUsageTokensObject + +SUPPORTED_OPENAI_TRANSCRIPTION_PARAMS: Final = ("language", "prompt", "temperature", "response_format") +TEXT_RESPONSE_FORMATS: Final = frozenset({"json", "text"}) + + +def _as_float(value: object) -> float | None: + if isinstance(value, bool) or not isinstance(value, (int, float, str)): + return None + try: + number: Final = float(value) + except ValueError: + return None + return number if math.isfinite(number) else None + + +def _rejection(key: str, value: object) -> str | None: + if key not in SUPPORTED_OPENAI_TRANSCRIPTION_PARAMS: + return f"bedrock transcription does not support {key}." + if key == "response_format" and value not in TEXT_RESPONSE_FORMATS: + return f"bedrock transcription returns only the transcript text, so response_format={value!r} is not supported." + if key == "temperature" and _as_float(value) is None: + return f"bedrock transcription needs a number for temperature, got {value!r}." + return None + + +@dataclass(frozen=True, slots=True) +class UnsupportedTranscriptionParams: + message: str + + +def map_openai_transcription_params( + non_default_params: Mapping[str, object], drop_params: bool +) -> Mapping[str, object] | UnsupportedTranscriptionParams: + rejections: Final = { + key: reason for key, value in non_default_params.items() if (reason := _rejection(key, value)) is not None + } + if rejections and not drop_params: + return UnsupportedTranscriptionParams( + message=f"{' '.join(rejections.values())} To drop it from the call, set `litellm.drop_params = True`." + ) + return { + key: _as_float(value) if key == "temperature" else value + for key, value in non_default_params.items() + if key not in rejections + } + + +class _RustTranscriptionResult(BaseModel): + model_config = ConfigDict(frozen=True) + + text: str + usage: TranscriptionUsageTokensObject | None = None + + +def _transcription_response( + result: Mapping[str, object], optional_params: Mapping[str, object] +) -> TranscriptionResponse: + parsed: Final = _RustTranscriptionResult.model_validate(result) + response: Final = TranscriptionResponse(text=parsed.text).model_copy(update={"usage": parsed.usage}) + region_name: Final = ( + optional_params.get("aws_region_name") or get_secret_str("AWS_REGION_NAME") or get_secret_str("AWS_REGION") + ) + response.hidden_params = {**response.hidden_params, "region_name": region_name} + return response class BedrockAudioTranscriptionRustDispatch: @@ -64,7 +133,7 @@ class BedrockAudioTranscriptionRustDispatch: "timeout_seconds": timeout_to_seconds(timeout), } call: Final = NativeCall(args=(), kwargs=fields, base={}) - return TranscriptionResponse(**rust(call)) + return _transcription_response(rust(call), optional_params) return runtime.run( RouteContext(Route.TRANSCRIPTION, provider=custom_llm_provider, model=model), @@ -97,7 +166,7 @@ class BedrockAudioTranscriptionRustDispatch: "timeout_seconds": timeout_to_seconds(timeout), } call: Final = NativeCall(args=(), kwargs=fields, base={}) - return TranscriptionResponse(**await rust(call)) + return _transcription_response(await rust(call), optional_params) return await runtime.arun( RouteContext(Route.TRANSCRIPTION, provider=custom_llm_provider, model=model), diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 60385d940d8..3ab297425f8 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -41939,6 +41939,120 @@ "supports_system_messages": true, "supports_native_structured_output": true }, + "bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 4.12e-08, + "input_cost_per_token_batches": 2.06e-08, + "input_cost_per_token_flex": 2.06e-08, + "input_cost_per_token_priority": 7.21e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 4.12e-08, + "output_cost_per_token_batches": 2.06e-08, + "output_cost_per_token_flex": 2.06e-08, + "output_cost_per_token_priority": 7.21e-08, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/ap-southeast-2/mistral.voxtral-small-24b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 1.03e-07, + "input_cost_per_token_batches": 5.15e-08, + "input_cost_per_token_flex": 5.15e-08, + "input_cost_per_token_priority": 1.8025e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 3.09e-07, + "output_cost_per_token_batches": 1.545e-07, + "output_cost_per_token_flex": 1.545e-07, + "output_cost_per_token_priority": 5.4075e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/ap-northeast-1/mistral.voxtral-mini-3b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 5e-08, + "input_cost_per_token_batches": 3e-08, + "input_cost_per_token_flex": 3e-08, + "input_cost_per_token_priority": 9e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 5e-08, + "output_cost_per_token_batches": 3e-08, + "output_cost_per_token_flex": 3e-08, + "output_cost_per_token_priority": 9e-08, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-northeast-1/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/ap-northeast-1/mistral.voxtral-small-24b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 1.2e-07, + "input_cost_per_token_batches": 6e-08, + "input_cost_per_token_flex": 6e-08, + "input_cost_per_token_priority": 2.1e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 3.6e-07, + "output_cost_per_token_batches": 1.8e-07, + "output_cost_per_token_flex": 1.8e-07, + "output_cost_per_token_priority": 6.3e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-northeast-1/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/eu-west-2/mistral.voxtral-mini-3b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 6e-08, + "input_cost_per_token_batches": 3e-08, + "input_cost_per_token_flex": 3e-08, + "input_cost_per_token_priority": 1.1e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 6e-08, + "output_cost_per_token_batches": 3e-08, + "output_cost_per_token_flex": 3e-08, + "output_cost_per_token_priority": 1.1e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/eu-west-2/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/eu-west-2/mistral.voxtral-small-24b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 1.6e-07, + "input_cost_per_token_batches": 8e-08, + "input_cost_per_token_flex": 8e-08, + "input_cost_per_token_priority": 2.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 4.7e-07, + "output_cost_per_token_batches": 2.3e-07, + "output_cost_per_token_flex": 2.3e-07, + "output_cost_per_token_priority": 8.2e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/eu-west-2/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, "mistral/codestral-2508": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 3e-07, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index b21bf0e14d4..4855a16a1cf 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2878,7 +2878,7 @@ class TranscriptionUsageTokensObject(LiteLLMBaseModel): input_tokens: int output_tokens: int total_tokens: int - input_token_details: TranscriptionUsageInputTokenDetailsObject + input_token_details: TranscriptionUsageInputTokenDetailsObject | None = None class TranscriptionResponse(OpenAIObject): diff --git a/litellm/utils.py b/litellm/utils.py index be1e11dd39f..699f9b4178f 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4067,7 +4067,9 @@ def get_optional_params_transcription( "timestamp_granularities": None, } - non_default_params = {k: v for k, v in passed_params.items() if (k in default_params and v != default_params[k])} + non_default_params: Final = { + k: v for k, v in passed_params.items() if (k in default_params and v != default_params[k]) + } optional_params = {} ## raise exception if non-default value passed for non-openai/azure embedding calls @@ -4104,6 +4106,19 @@ def get_optional_params_transcription( model=model, drop_params=drop_params if drop_params is not None else False, ) + elif custom_llm_provider == "bedrock": + from litellm.llms.bedrock.audio_transcription import ( + UnsupportedTranscriptionParams, + map_openai_transcription_params, + ) + + match map_openai_transcription_params( + non_default_params, drop_params=drop_params is True or litellm.drop_params is True + ): + case UnsupportedTranscriptionParams(message=message): + raise UnsupportedParamsError(message=message, llm_provider="bedrock") + case mapped_params: + optional_params = dict(mapped_params) elif provider_config is not None: # custom audio transcription config supported_params = provider_config.get_supported_openai_params(model=model) _check_valid_arg(supported_params=supported_params) @@ -4118,7 +4133,7 @@ def get_optional_params_transcription( optional_params=optional_params, passed_params=passed_params, custom_llm_provider=custom_llm_provider, - openai_params=OPENAI_TRANSCRIPTION_PARAMS, + openai_params=list(default_params) if custom_llm_provider == "bedrock" else OPENAI_TRANSCRIPTION_PARAMS, additional_drop_params=kwargs.get("additional_drop_params", None), ) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 60385d940d8..3ab297425f8 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -41939,6 +41939,120 @@ "supports_system_messages": true, "supports_native_structured_output": true }, + "bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 4.12e-08, + "input_cost_per_token_batches": 2.06e-08, + "input_cost_per_token_flex": 2.06e-08, + "input_cost_per_token_priority": 7.21e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 4.12e-08, + "output_cost_per_token_batches": 2.06e-08, + "output_cost_per_token_flex": 2.06e-08, + "output_cost_per_token_priority": 7.21e-08, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/ap-southeast-2/mistral.voxtral-small-24b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 1.03e-07, + "input_cost_per_token_batches": 5.15e-08, + "input_cost_per_token_flex": 5.15e-08, + "input_cost_per_token_priority": 1.8025e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 3.09e-07, + "output_cost_per_token_batches": 1.545e-07, + "output_cost_per_token_flex": 1.545e-07, + "output_cost_per_token_priority": 5.4075e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/ap-northeast-1/mistral.voxtral-mini-3b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 5e-08, + "input_cost_per_token_batches": 3e-08, + "input_cost_per_token_flex": 3e-08, + "input_cost_per_token_priority": 9e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 5e-08, + "output_cost_per_token_batches": 3e-08, + "output_cost_per_token_flex": 3e-08, + "output_cost_per_token_priority": 9e-08, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-northeast-1/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/ap-northeast-1/mistral.voxtral-small-24b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 1.2e-07, + "input_cost_per_token_batches": 6e-08, + "input_cost_per_token_flex": 6e-08, + "input_cost_per_token_priority": 2.1e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 3.6e-07, + "output_cost_per_token_batches": 1.8e-07, + "output_cost_per_token_flex": 1.8e-07, + "output_cost_per_token_priority": 6.3e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-northeast-1/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/eu-west-2/mistral.voxtral-mini-3b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 6e-08, + "input_cost_per_token_batches": 3e-08, + "input_cost_per_token_flex": 3e-08, + "input_cost_per_token_priority": 1.1e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 6e-08, + "output_cost_per_token_batches": 3e-08, + "output_cost_per_token_flex": 3e-08, + "output_cost_per_token_priority": 1.1e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/eu-west-2/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/eu-west-2/mistral.voxtral-small-24b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 1.6e-07, + "input_cost_per_token_batches": 8e-08, + "input_cost_per_token_flex": 8e-08, + "input_cost_per_token_priority": 2.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 4.7e-07, + "output_cost_per_token_batches": 2.3e-07, + "output_cost_per_token_flex": 2.3e-07, + "output_cost_per_token_priority": 8.2e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/eu-west-2/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, "mistral/codestral-2508": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 3e-07, diff --git a/tests/unit/rust_bridge/native_route_wheel_test.py b/tests/unit/rust_bridge/native_route_wheel_test.py index bc3269f5dd9..ed3fe4f6de5 100644 --- a/tests/unit/rust_bridge/native_route_wheel_test.py +++ b/tests/unit/rust_bridge/native_route_wheel_test.py @@ -87,7 +87,7 @@ def assert_native_request( def native_response(status: int) -> bytes: if status == 429: return b'{"error":"native-rate-limit"}' - return b'{"output":{"message":{"content":[{"text":"native-transcription"}]}}}' + return b'{"output":{"message":{"content":[{"text":"native-transcription"}]}},"usage":{"inputTokens":390,"outputTokens":60}}' def load_native(native_path: Path) -> object: @@ -130,6 +130,9 @@ def assert_success(route: str, response: object) -> None: raise TypeError(f"{route} returned {type(response).__name__}, expected dict") if response["text"] != "native-transcription": raise AssertionError(f"{route} returned {response['text']!r}, expected 'native-transcription'") + expected_usage: Final = {"type": "tokens", "input_tokens": 390, "output_tokens": 60, "total_tokens": 450} + if response.get("usage") != expected_usage: + raise AssertionError(f"{route} returned usage {response.get('usage')!r}, expected {expected_usage!r}") def assert_rate_limit(route: str, error: BaseException) -> None: diff --git a/tests/unit/test_audio_transcription_rust_bridge.py b/tests/unit/test_audio_transcription_rust_bridge.py index 715423c59d9..cd94fb32af2 100644 --- a/tests/unit/test_audio_transcription_rust_bridge.py +++ b/tests/unit/test_audio_transcription_rust_bridge.py @@ -11,6 +11,7 @@ from litellm.rust_bridge import configuration from litellm.rust_bridge._native import RustBridgeDeclined, RustUpstreamError from litellm.rust_bridge.public_call import NativeCall from litellm.rust_bridge.transcription.native import NATIVE_ATRANSCRIPTION, NATIVE_TRANSCRIPTION +from litellm.types.utils import TranscriptionUsageTokensObject MODEL: Final = "bedrock/mistral.voxtral-mini-3b-2507" AUDIO_FILE: Final = ("audio.wav", b"audio", "audio/wav") @@ -59,6 +60,14 @@ class AsyncBridge: return {"text": "async rust"} +class UsageBridge: + def __call__(self, call: NativeCall) -> dict[str, object]: + return { + "text": "rust", + "usage": {"type": "tokens", "input_tokens": 390, "output_tokens": 60, "total_tokens": 450}, + } + + def dispatch_sync() -> litellm.TranscriptionResponse: return BedrockAudioTranscriptionRustDispatch().audio_transcriptions( model=MODEL, @@ -196,3 +205,99 @@ async def test_async_missing_native_binding_raises_without_python_fallback() -> optional_params={}, timeout=None, ) + + +@pytest.mark.parametrize( + ("region_kwargs", "region_env", "priced_model"), + [ + ({}, {}, MODEL), + ({"aws_region_name": "ap-southeast-2"}, {}, "bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507"), + ({}, {"AWS_REGION_NAME": "eu-west-2"}, "bedrock/eu-west-2/mistral.voxtral-mini-3b-2507"), + ({}, {"AWS_REGION": "ap-northeast-1"}, "bedrock/ap-northeast-1/mistral.voxtral-mini-3b-2507"), + ( + {"aws_region_name": "ap-southeast-2"}, + {"AWS_REGION_NAME": "eu-west-2"}, + "bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507", + ), + ], +) +def test_bedrock_transcription_bills_the_converse_token_usage( + region_kwargs: dict[str, str], + region_env: dict[str, str], + priced_model: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + for name in ("AWS_REGION_NAME", "AWS_REGION"): + monkeypatch.delenv(name, raising=False) + for name, value in region_env.items(): + monkeypatch.setenv(name, value) + NATIVE_TRANSCRIPTION.override(UsageBridge()) + + response: Final = litellm.transcription(model=MODEL, file=AUDIO_FILE, **region_kwargs) + + model_info: Final = litellm.get_model_info(priced_model) + expected_cost: Final = 390 * (model_info.get("input_cost_per_token") or 0.0) + 60 * ( + model_info.get("output_cost_per_token") or 0.0 + ) + assert response.usage == TranscriptionUsageTokensObject( + type="tokens", input_tokens=390, output_tokens=60, total_tokens=450 + ) + assert expected_cost > 0 + assert litellm.completion_cost( + completion_response=response, model=MODEL, call_type="transcription" + ) == pytest.approx(expected_cost) + + +def test_bedrock_transcription_forwards_language_and_text_formats_to_rust() -> None: + bridge: Final = SyncBridge() + NATIVE_TRANSCRIPTION.override(bridge) + + litellm.transcription(model=MODEL, file=AUDIO_FILE, language="fr", response_format="text", prompt="Names") + + forwarded: Final = bridge.calls[0].resolved["optional_params"] + assert {key: forwarded.get(key) for key in ("language", "response_format", "prompt")} == { + "language": "fr", + "response_format": "text", + "prompt": "Names", + } + + +@pytest.mark.parametrize(("sent", "expected"), [("0.2", 0.2), ("0", 0.0), (0.5, 0.5), (1, 1.0)]) +def test_bedrock_transcription_forwards_form_temperature_as_a_number(sent: object, expected: float) -> None: + bridge: Final = SyncBridge() + NATIVE_TRANSCRIPTION.override(bridge) + + litellm.transcription(model=MODEL, file=AUDIO_FILE, temperature=sent) + + forwarded: Final = bridge.calls[0].resolved["optional_params"]["temperature"] + assert (type(forwarded), forwarded) == (float, expected) + + +@pytest.mark.parametrize( + "unsupported", + [ + {"response_format": "srt"}, + {"response_format": "verbose_json"}, + {"timestamp_granularities": ["word"]}, + {"temperature": "warm"}, + {"temperature": "inf"}, + {"temperature": True}, + ], +) +def test_bedrock_transcription_rejects_params_it_cannot_honor(unsupported: dict[str, object]) -> None: + bridge: Final = SyncBridge() + NATIVE_TRANSCRIPTION.override(bridge) + + with pytest.raises(litellm.BadRequestError) as raised: + litellm.transcription(model=MODEL, file=AUDIO_FILE, language="fr", **unsupported) + + assert raised.value.status_code == 400 + assert next(iter(unsupported)) in str(raised.value) + assert bridge.calls == () + + dropped: Final = litellm.transcription(model=MODEL, file=AUDIO_FILE, language="fr", drop_params=True, **unsupported) + + assert dropped.text == "rust" + forwarded: Final = bridge.calls[0].resolved["optional_params"] + assert forwarded["language"] == "fr" + assert forwarded.keys().isdisjoint(unsupported) From 019a9de603db14849c0157f744eb00f5061c3a61 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 28 Oct 2026 19:30:33 -0700 Subject: [PATCH 02/10] fix(logging): stop dumping the request body at every layer under --detailed_debug (#48973) * fix(logging): stop dumping the request body at every layer under --detailed_debug The proxy dumped the full request body at every layer it passed through (proxy entry, pre-call, router, SDK call echo, param mapping, logging object), so one Codex "hi" wrote over 500KB of debug log. Keep the inbound request line and the outgoing provider request, summarize large values in the SDK call echo, and stop per-request lines that report nothing (callback disable checks with no disable header, cost map name probes, response id decode misses, vector store path misses). * fix(logging): summarize tools in the param mapping debug lines Non-Default params and Final returned optional params still printed full tool definitions, so a Codex-style request repeated them twice more. Use the same size summary as the SDK call echo and cover tools in the test. * perf(logging): build the param mapping debug summary only when debug is on --- .../enterprise_callbacks/callback_controls.py | 3 +- litellm/litellm_core_utils/litellm_logging.py | 3 - litellm/proxy/auth/auth_checks.py | 1 - litellm/proxy/common_request_processing.py | 1 - .../proxy/common_utils/http_parsing_utils.py | 2 - litellm/proxy/litellm_pre_call_utils.py | 6 - litellm/proxy/proxy_server.py | 1 - litellm/responses/utils.py | 6 + litellm/router.py | 34 +++--- litellm/router_utils/search_api_router.py | 3 +- litellm/utils.py | 36 ++++-- .../test_callback_controls.py | 13 ++- tests/unit/proxy/test_proxy_server.py | 103 ++++++++++++++++++ 13 files changed, 166 insertions(+), 46 deletions(-) diff --git a/enterprise/litellm_enterprise/enterprise_callbacks/callback_controls.py b/enterprise/litellm_enterprise/enterprise_callbacks/callback_controls.py index bf99e1a72ca..6a97a9de414 100644 --- a/enterprise/litellm_enterprise/enterprise_callbacks/callback_controls.py +++ b/enterprise/litellm_enterprise/enterprise_callbacks/callback_controls.py @@ -33,9 +33,8 @@ class EnterpriseCallbackControls: try: disabled_callbacks = EnterpriseCallbackControls.get_disabled_callbacks(litellm_params, standard_callback_dynamic_params) - verbose_logger.debug(f"Dynamically disabled callbacks from {X_LITELLM_DISABLE_CALLBACKS}: {disabled_callbacks}") - verbose_logger.debug(f"Checking if {callback} is disabled via headers. Disable callbacks from headers: {disabled_callbacks}") if disabled_callbacks is not None: + verbose_logger.debug(f"Checking if {callback} is in dynamically disabled callbacks: {disabled_callbacks}") ######################################################### # premium user check ######################################################### diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 8b66c27b930..620c8413f29 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1130,9 +1130,6 @@ class Logging(LiteLLMLoggingBaseClass): } self.litellm_request_debug = litellm_params.get("litellm_request_debug", False) self.logger_fn = litellm_params.get("logger_fn", None) - if is_debugging_on() or self.litellm_request_debug: - verbose_logger.debug("self.optional_params: %s", self.optional_params) - self.model_call_details.update( { "model": self.model, diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 448643ef59b..1ead470bbd2 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -8075,7 +8075,6 @@ async def vector_store_access_check( rag_ids: Final = (rag_vector_store_id,) if rag_vector_store_id is not None else () vector_store_ids_to_run: Final = tuple(dict.fromkeys((*registry_ids, *rag_ids))) if not vector_store_ids_to_run: - verbose_proxy_logger.debug("Vector store to run not found, skipping vector store access check") return True ######################################################### diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 89886c43525..792efa480ec 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -2581,7 +2581,6 @@ class ProxyBaseLLMRequestProcessing: # have mutated `self.data` in place, and the audit-trail snapshot taken in # add_litellm_data_to_request predates that mutation. refresh_proxy_server_request_body_snapshot(self.data, guardrails_applied=True) - verbose_proxy_logger.debug("receiving data: %s", self.data) if "messages" in self.data and self.data["messages"]: logging_obj.update_messages(self.data["messages"]) diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index c4c3f0ad634..56cd14d987d 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -845,5 +845,3 @@ def _add_vector_store_id_from_path(request_data: dict, request: Request) -> None "populate_request_with_path_params: Updated request_data with vector_store_ids=%s", request_data.get("vector_store_ids"), ) - else: - verbose_proxy_logger.debug("populate_request_with_path_params: No vector_store_id present in path=%s", path) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 2c05f724374..14d941ff62c 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -3024,13 +3024,7 @@ async def add_litellm_data_to_request( getattr(request.state, "litellm_roi_estimator", False) is True ) - verbose_proxy_logger.debug( - "[PROXY] returned data from litellm_pre_call_utils: %s", - data, - ) - # Team/Project credential overrides from model_config - # Placed after the debug log to avoid leaking credential secrets in logs _apply_credential_overrides_from_model_config( data=data, user_api_key_dict=user_api_key_dict, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 42816e7db61..c04e79d0934 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -18437,7 +18437,6 @@ async def async_queue_request( "body": copy.copy(data), # use copy instead of deepcopy } - verbose_proxy_logger.debug("receiving data: %s", data) data["model"] = ( general_settings.get("completion_model", None) # server default or user_model # model name passed via cli args diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index cf993c69814..05d6a36c1f6 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -766,6 +766,12 @@ class ResponsesAPIRequestUtils: model_id=model_id, response_id=decoded_response_id, ) + except ValueError: + return DecodedResponseId( + custom_llm_provider=None, + model_id=None, + response_id=response_id, + ) except Exception as e: verbose_logger.debug("Error decoding response_id '%s': %s", response_id, e) return DecodedResponseId( diff --git a/litellm/router.py b/litellm/router.py index fc888495e99..0408819e05c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -5668,7 +5668,7 @@ class Router: def _image_generation(self, prompt: str, model: str, **kwargs): model_name: Final = "" try: - verbose_router_logger.debug("Inside _image_generation()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside _image_generation()- model: %s", model) deployment: Final = self.get_available_deployment( model=model, messages=[{"role": "user", "content": "prompt"}], @@ -5730,7 +5730,7 @@ class Router: async def _aimage_generation(self, prompt: str, model: str, **kwargs): model_name = model try: - verbose_router_logger.debug("Inside _image_generation()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside _image_generation()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) deployment: Final = await self.async_get_available_deployment( model=model, @@ -5815,7 +5815,7 @@ class Router: async def _atranscription(self, file: FileTypes, model: str, **kwargs): model_name: Final = model try: - verbose_router_logger.debug("Inside _atranscription()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside _atranscription()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) deployment: Final = await self.async_get_available_deployment( model=model, @@ -5908,7 +5908,7 @@ class Router: async def _aspeech(self, model: str, input: str, voice: str | None = None, **kwargs): model_name: Final = model try: - verbose_router_logger.debug("Inside _aspeech()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside _aspeech()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) deployment: Final = await self.async_get_available_deployment( model=model, @@ -5970,7 +5970,7 @@ class Router: async def _arerank(self, model: str, **kwargs): model_name = None try: - verbose_router_logger.debug("Inside _rerank()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside _rerank()- model: %s", model) deployment: Final = await self.async_get_available_deployment( model=model, specific_deployment=kwargs.pop("specific_deployment", None), @@ -6079,7 +6079,7 @@ class Router: async def _atext_completion(self, model: str, prompt: str, **kwargs): try: - verbose_router_logger.debug("Inside _atext_completion()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside _atext_completion()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) deployment: Final = await self.async_get_available_deployment( model=model, @@ -6149,7 +6149,7 @@ class Router: async def _aadapter_completion(self, adapter_id: str, model: str, **kwargs): try: - verbose_router_logger.debug("Inside _aadapter_completion()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside _aadapter_completion()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) deployment: Final = await self.async_get_available_deployment( model=model, @@ -6243,7 +6243,7 @@ class Router: kwargs=kwargs, metadata_variable_name="litellm_metadata", ) - verbose_router_logger.debug("Inside aguardrail() - guardrail_name: %s; kwargs: %s", guardrail_name, kwargs) + verbose_router_logger.debug("Inside aguardrail() - guardrail_name: %s", guardrail_name) response: Final = await self.async_function_with_fallbacks(**kwargs) return response @@ -6324,9 +6324,7 @@ class Router: controls: Final = per_request_fallback_controls(kwargs) kwargs[MID_STREAM_FALLBACK_CONTROLS_KEY] = controls # rebind-ok: forwarded to every hop self._update_kwargs_before_fallbacks(model=model, kwargs=kwargs, metadata_variable_name="litellm_metadata") - verbose_router_logger.debug( - "Inside ageneric_api_call_with_fallbacks() - model: %s; kwargs: %s", model, kwargs - ) + verbose_router_logger.debug("Inside ageneric_api_call_with_fallbacks() - model: %s", model) response: Final = await self.async_function_with_fallbacks(**kwargs) return response except Exception as e: @@ -7146,9 +7144,7 @@ class Router: handler_name: Final = original_function.__name__ metadata_variable_name: Final = get_router_metadata_variable_name(function_name="generic_api_call") try: - verbose_router_logger.debug( - "Inside _generic_api_call() - handler: %s, model: %s; kwargs: %s", handler_name, model, kwargs - ) + verbose_router_logger.debug("Inside _generic_api_call() - handler: %s, model: %s", handler_name, model) self._update_kwargs_before_fallbacks( model=model, kwargs=kwargs, @@ -7223,7 +7219,7 @@ class Router: def _embedding(self, input: str | list[object], model: str, **kwargs): model_name = None try: - verbose_router_logger.debug("Inside embedding()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside embedding()- model: %s", model) with embedding_deployment_selection(): deployment: Final = self.get_available_deployment( model=model, @@ -7312,7 +7308,7 @@ class Router: async def _aembedding(self, input: str | list[object], model: str, **kwargs): model_name = None try: - verbose_router_logger.debug("Inside _aembedding()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside _aembedding()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) with embedding_deployment_selection(): deployment: Final = await self.async_get_available_deployment( @@ -7396,7 +7392,7 @@ class Router: try: from litellm.router_utils.common_utils import add_model_file_id_mappings - verbose_router_logger.debug("Inside _atext_completion()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside _atext_completion()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) healthy_deployments: Final = await self.async_get_healthy_deployments( model=model, @@ -7634,7 +7630,7 @@ class Router: **kwargs, ) -> LiteLLMBatch: try: - verbose_router_logger.debug("Inside _acreate_batch()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside _acreate_batch()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) with batch_create_routing(): deployment: Final = await self.async_get_available_deployment( @@ -7833,7 +7829,7 @@ class Router: **kwargs, ) -> LiteLLMBatch: try: - verbose_router_logger.debug("Inside _acancel_batch()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside _acancel_batch()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) deployment: Final = await self.async_get_available_deployment( model=model, diff --git a/litellm/router_utils/search_api_router.py b/litellm/router_utils/search_api_router.py index 1cfb311d796..26326a7e6d4 100644 --- a/litellm/router_utils/search_api_router.py +++ b/litellm/router_utils/search_api_router.py @@ -155,10 +155,9 @@ class SearchAPIRouter: available_search_tool_names: Final = [tool.get("search_tool_name") for tool in router_instance.search_tools] verbose_router_logger.debug( - "Inside SearchAPIRouter.async_search_with_fallbacks() - search_tool_name: %s, Available Search Tools: %s, kwargs: %s", + "Inside SearchAPIRouter.async_search_with_fallbacks() - search_tool_name: %s, Available Search Tools: %s", search_tool_name, available_search_tool_names, - kwargs, ) # Use the existing retry/fallback infrastructure diff --git a/litellm/utils.py b/litellm/utils.py index 699f9b4178f..f790966d167 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -304,7 +304,7 @@ except (ImportError, AttributeError, TypeError): # Convert to str (if necessary) claude_json_str = json.dumps(json_data) import importlib.metadata -from collections.abc import AsyncIterator, Callable, Collection, Iterable, Iterator, Mapping, Sequence +from collections.abc import AsyncIterator, Callable, Collection, Iterable, Iterator, Mapping, Sequence, Sized from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypeVar, cast, runtime_checkable from typing_extensions import assert_never @@ -5074,8 +5074,10 @@ def get_optional_params( supported_params: List[str] - supported params from the litellm config """ verbose_logger.info("\nLiteLLM completion() model= %s; provider = %s", model, custom_llm_provider) - verbose_logger.debug("\nLiteLLM: Params passed to completion() %s", passed_params) - verbose_logger.debug("\nLiteLLM: Non-Default params passed to completion() %s", non_default_params) + if verbose_logger.isEnabledFor(logging.DEBUG): + verbose_logger.debug( + "\nLiteLLM: Non-Default params passed to completion() %s", _debug_params_repr(non_default_params) + ) unsupported_params: Final = {} for k in non_default_params: if k not in supported_params: @@ -5616,7 +5618,9 @@ def get_optional_params( additional_drop_params=additional_drop_params, ) if _print_verbose_is_active(): - print_verbose(f"Final returned optional params: {redact_credentials_in_payload(optional_params)}") + print_verbose( + f"Final returned optional params: {_debug_params_repr(redact_credentials_in_payload(optional_params))}" + ) optional_params = apply_openai_param_overrides( optional_params=optional_params, non_default_params=non_default_params, @@ -6625,8 +6629,6 @@ def get_model_info_helper( model=model, custom_llm_provider=custom_llm_provider or declared_authenticating_provider(model) ) - verbose_logger.debug("checking potential_model_names in litellm.model_cost: %s", potential_model_names) - combined_model_name: Final = potential_model_names["combined_model_name"] stripped_model_name: Final = potential_model_names["stripped_model_name"] combined_stripped_model_name: Final = potential_model_names["combined_stripped_model_name"] @@ -6729,6 +6731,7 @@ def get_model_info_helper( key, _model_info = generalization if _model_info is None or key is None: + verbose_logger.debug("model not found in litellm.model_cost, tried: %s", potential_model_names) raise ModelNotMappedError(_model_not_mapped_message(model, custom_llm_provider)) _input_cost_per_token: float | None = _model_info.get("input_cost_per_token") if _input_cost_per_token is None: @@ -8548,6 +8551,23 @@ def get_valid_models( return [] # NON-Blocking +_MAX_INLINE_DEBUG_ARG_CHARS: Final = 500 + + +def _debug_arg_repr(value: object) -> str: + """Short form of a call argument, since the outgoing provider request log already prints the request body.""" + rendered: Final = repr(value) + if len(rendered) <= _MAX_INLINE_DEBUG_ARG_CHARS: + return rendered + if isinstance(value, Sized): + return f"<{type(value).__name__} len={len(value)}>" + return f"<{type(value).__name__}>" + + +def _debug_params_repr(params: Mapping[str, object]) -> str: + return "{" + ", ".join(f"{key!r}: {_debug_arg_repr(value)}" for key, value in params.items()) + "}" + + def print_args_passed_to_litellm(original_function, args, kwargs): if not is_debugging_on(): return @@ -8568,9 +8588,9 @@ def print_args_passed_to_litellm(original_function, args, kwargs): ): return - args_str: Final = ", ".join(map(repr, args)) + args_str: Final = ", ".join(map(_debug_arg_repr, args)) redacted_kwargs: Final = redact_credentials_in_payload(kwargs) - kwargs_str: Final = ", ".join(f"{key}={value!r}" for key, value in redacted_kwargs.items()) + kwargs_str: Final = ", ".join(f"{key}={_debug_arg_repr(value)}" for key, value in redacted_kwargs.items()) print_verbose( "\n", ) # new line before diff --git a/tests/unit/enterprise/enterprise_callbacks/test_callback_controls.py b/tests/unit/enterprise/enterprise_callbacks/test_callback_controls.py index 063e9e0ba7f..9648cb6ffc4 100644 --- a/tests/unit/enterprise/enterprise_callbacks/test_callback_controls.py +++ b/tests/unit/enterprise/enterprise_callbacks/test_callback_controls.py @@ -1,5 +1,5 @@ import unittest.mock as mock -from typing import cast +from typing import Final, cast from unittest.mock import MagicMock, patch import pytest @@ -197,6 +197,17 @@ class TestEnterpriseCallbackControls: ) assert result is False + def test_no_disable_request_logs_nothing(self, caplog: pytest.LogCaptureFixture) -> None: + litellm_params: Final = {"proxy_server_request": {"url": "test", "headers": {"user-agent": "codex"}}} + + with caplog.at_level("DEBUG", logger="LiteLLM"): + result: Final = EnterpriseCallbackControls.is_callback_disabled_dynamically( + DataDogLogger.__new__(DataDogLogger), litellm_params, StandardCallbackDynamicParams() + ) + + assert result is False + assert caplog.records == [] + def test_callback_not_disabled_when_header_none( self, mock_premium_user, mock_request_headers ): diff --git a/tests/unit/proxy/test_proxy_server.py b/tests/unit/proxy/test_proxy_server.py index a26a64c6fb0..976ed4f28a5 100644 --- a/tests/unit/proxy/test_proxy_server.py +++ b/tests/unit/proxy/test_proxy_server.py @@ -22,7 +22,9 @@ import json import asyncio import logging +import httpx import pytest +import respx import litellm from litellm import RateLimitError, Timeout, completion, completion_cost, embedding @@ -3857,3 +3859,104 @@ async def test_model_list_return_wildcard_routes_defaults_to_general_setting( assert "gpt-6.1-sol" in listed, listed assert ("openai/*" in listed) is wildcard_listed, listed + + +_DEBUG_LOG_SENTINEL: Final = "sentinel-request-body-7f3a" +_LONG_PROMPT: Final = f"{_DEBUG_LOG_SENTINEL} " + "Keep edits minimal and explain them briefly. " * 20 +_TOOL_PARAMETERS: Final = {"type": "object", "properties": {"cmd": {"type": "string"}}} + + +class _MessageCollector(logging.Handler): + def __init__(self) -> None: + super().__init__(level=logging.DEBUG) + self.messages: list[str] = [] # mutable-ok: a logging.Handler collects records as they are emitted + + def emit(self, record: logging.LogRecord) -> None: + self.messages.append(record.getMessage()) + + +@pytest.mark.parametrize( + "route, path_suffix, body, upstream_json", + [ + ( + "/v1/chat/completions", + "/chat/completions", + { + "model": "test_openai_models", + "messages": [{"role": "user", "content": _LONG_PROMPT}], + "tools": [ + { + "type": "function", + "function": {"name": "shell", "description": _LONG_PROMPT, "parameters": _TOOL_PARAMETERS}, + } + ], + }, + { + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 1, + "model": "gpt-3.5-turbo", + "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "hi"}}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ), + ( + "/v1/responses", + "/responses", + { + "model": "test_openai_models", + "instructions": _LONG_PROMPT, + "input": "hi", + "tools": [ + {"type": "function", "name": "shell", "description": _LONG_PROMPT, "parameters": _TOOL_PARAMETERS} + ], + }, + { + "id": "resp_1", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-3.5-turbo", + "output": [ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "hi", "annotations": []}], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + }, + ), + ], +) +def test_detailed_debug_logs_request_body_only_as_received_and_as_sent( + client_no_auth: TestClient, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + route: str, + path_suffix: str, + body: Mapping[str, object], + upstream_json: Mapping[str, object], +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post(url__regex=rf".*{path_suffix}$").mock(return_value=httpx.Response(200, json=upstream_json)) + collector: Final = _MessageCollector() + loggers: Final = tuple(logging.getLogger(name) for name in ("LiteLLM", "LiteLLM Router", "LiteLLM Proxy")) + original_levels: Final = tuple(lg.level for lg in loggers) + for lg in loggers: + lg.addHandler(collector) + lg.setLevel(logging.DEBUG) + try: + response: Final = client_no_auth.post(route, json=body) + finally: + for lg, level in zip(loggers, original_levels): + lg.removeHandler(collector) + lg.setLevel(level) + + assert response.status_code == 200, response.text + body_dumps: Final = [message for message in collector.messages if _DEBUG_LOG_SENTINEL in message] + assert len(body_dumps) == 2, [message[:120] for message in body_dumps] + assert body_dumps[0].startswith("Request received by LiteLLM:") + assert "POST Request Sent from LiteLLM" in body_dumps[1] From 4fc2bba1f255f8ae2b55f36d6110dd7a18b38db0 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 28 Oct 2026 19:32:44 -0700 Subject: [PATCH 03/10] fix(stream): the last usage frame carries litellm's bill in usage.cost (#48955) * fix(proxy): copy the stream's hidden response_cost into the client's streamed usage.cost The stream is priced once into hidden response_cost (stream builder for chat and text completions, the Responses iterator for the terminal event) and the proxy egress copies that bill into the streamed usage.cost on a copy of the event. Inside the pipeline usage.cost stays what the upstream reported. Spend rows do not change; the billing fix for upstream-reported usage.cost follows in the next PR cut from #48287. * fix(proxy): type the bridge bill carry and the builder's reported-cost guard for the lint gates * style: format the chat-over-Responses bridge helper signature * refactor(bridge): build the billed Responses event with model_copy instead of a TypeAdapter chain * fix(stream): keep writing the computed cost into usage.cost as on main, the hidden copy is only for the proxy egress * fix(stream): write litellm's bill into the usage frame's usage.cost instead of a hidden field the proxy copies * test(stream): keep main's formatting in test_streaming_handler.py, only the new test changes * refactor(stream): drop the docstring on _stamp_usage_cost, the body says it --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_core_utils/streaming_handler.py | 14 ++++ litellm/responses/streaming_iterator.py | 32 ++++++-- .../test_streaming_handler.py | 16 ++++ .../test_streaming_iterator_transformation.py | 33 ++++++++ .../unit/responses/test_streaming_iterator.py | 78 +++++++++++++++---- 5 files changed, 151 insertions(+), 22 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 2348d8e57d6..f2f06d801a8 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -92,6 +92,18 @@ def _next_sync_or_exhausted(it: Iterator[object]) -> object: return _SYNC_ITER_EXHAUSTED +def _stamp_usage_cost( + usage_chunk: ModelResponse | TextCompletionResponse | ModelResponseStream, + complete_response: ModelResponse | TextCompletionResponse | ModelResponseStream, +) -> None: + bill: Final = (get_hidden_params(complete_response) or {}).get("response_cost") + usage: Final[object] = getattr(usage_chunk, "usage", None) + if usage is None or isinstance(bill, bool) or not isinstance(bill, (int, float)): + return + if bill > 0 or isinstance(getattr(usage, "cost", None), (int, float)): + setattr(usage, "cost", float(bill)) # noqa: B010 # cost is an optional extra on the usage model + + def _stamp_served_service_tier(response: ModelResponseStream, complete_streaming_response: ModelResponse) -> None: served_tier: Final = complete_streaming_response.model_dump().get("service_tier") if isinstance(served_tier, str) and served_tier: @@ -2141,6 +2153,7 @@ class CustomStreamWrapper: getattr(complete_streaming_response, "usage"), ) _stamp_served_service_tier(response, complete_streaming_response) + _stamp_usage_cost(response, complete_streaming_response) try: _cache_copy = complete_streaming_response.model_copy(deep=True) _log_copy = complete_streaming_response.model_copy(deep=True) @@ -2361,6 +2374,7 @@ class CustomStreamWrapper: getattr(complete_streaming_response, "usage"), ) _stamp_served_service_tier(response, complete_streaming_response) + _stamp_usage_cost(response, complete_streaming_response) try: _copy = complete_streaming_response.model_copy(deep=True) except RuntimeError: diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 4e418a91156..5060e03f040 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -39,7 +39,11 @@ from litellm.constants import ( ) from litellm.exceptions import MidStreamFallbackError, RateLimitError from litellm.litellm_core_utils.asyncify import run_async_function -from litellm.litellm_core_utils.core_helpers import process_response_headers +from litellm.litellm_core_utils.core_helpers import ( + process_response_headers, + provider_reported_usage_cost, + set_response_cost_in_hidden_params, +) from litellm.litellm_core_utils.exception_mapping_utils import with_deployment_api_base from litellm.litellm_core_utils.internal_call_metadata import ( is_background_response, @@ -1825,20 +1829,32 @@ def _estimate_usage_safely( def stamp_responses_usage_cost( response_obj: ResponsesAPIResponse | None, logging_obj: LiteLLMLoggingObj | None ) -> None: - if response_obj is None or logging_obj is None: + """The terminal response's ``usage.cost`` is the bill litellm writes to the spend row. It is priced on a copy + shaped the way the success handler shapes the spend row's result (the reported ``usage.cost`` in the + response-cost header, the usage in chat shape), so the client and the row read the same number.""" + if response_obj is None: return usage_obj: Final[ResponseAPIUsage | None] = _usage_as_model(getattr(response_obj, "usage", None)) if usage_obj is None: return - response_obj.usage = usage_obj # rebind-ok: the stamped cost has to ride on the response the client receives - if isinstance(getattr(usage_obj, "cost", None), (int, float)): + response_obj.usage = usage_obj # rebind-ok: the typed usage has to ride on the response the client receives + if logging_obj is None: return + priced: Final = response_obj.model_copy( + update={"usage": ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(usage_obj)} + ) + priced.hidden_params = {} + set_response_cost_in_hidden_params( + priced, provider_reported_usage_cost(usage_obj, logging_obj.model_call_details.get("custom_llm_provider")) + ) try: - cost: Final[float | None] = logging_obj.response_cost_calculator(result=response_obj) - except Exception: + bill: Final = logging_obj.response_cost_calculator(result=priced) + except Exception: # noqa: BLE001 # a pricing failure leaves the client usage as the upstream sent it return - if isinstance(cost, (int, float)) and cost > 0: - setattr(usage_obj, "cost", cost) + if isinstance(bill, bool) or not isinstance(bill, (int, float)): + return + if bill > 0 or isinstance(getattr(usage_obj, "cost", None), (int, float)): + setattr(usage_obj, "cost", float(bill)) def opening_item_payload(payload: Mapping[str, object]) -> Mapping[str, object]: diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index 79978ecfe8e..b580f526a0f 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -6316,6 +6316,22 @@ def test_chunk_creator_preserves_hidden_provider_specific_fields_from_parsed_chu assert assembled._hidden_params["provider_specific_fields"] == {"traffic_type": "ON_DEMAND_FLEX"} +def test_the_usage_chunk_carries_the_bill_the_builder_priced_for_the_spend_row() -> None: + from litellm.litellm_core_utils.streaming_handler import ( + _stamp_usage_cost, # pyright: ignore[reportPrivateUsage] # the helper under test + ) + + complete: Final = ModelResponse(model="omni-model", usage=Usage(prompt_tokens=100, completion_tokens=20)) + complete.hidden_params = {"response_cost": 0.00032} + chunk_usage: Final = Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120) + setattr(chunk_usage, "cost", 0.037) + usage_chunk: Final = ModelResponseStream(id="chunk-1", model="omni-model", choices=[], usage=chunk_usage) + + _stamp_usage_cost(usage_chunk, complete) + + assert usage_chunk.usage is not None and usage_chunk.usage.cost == 0.00032, "the client sees what the row bills" + + def test_chunk_creator_keeps_provider_model_private_across_stream(): from litellm.router_utils.add_retry_fallback_headers import ( get_hidden_params_dict, diff --git a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index 0b971bc1696..d9a0033b179 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -10,6 +10,7 @@ before response.completed, and that every event of a bridged stream carries the spend tracking stores, so a follow-up previous_response_id still finds the conversation. """ +import itertools import json from collections.abc import Callable from typing import Final @@ -23,6 +24,7 @@ from litellm.responses.litellm_completion_transformation.streaming_iterator impo from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ( BaseLiteLLMOpenAIResponseObject, + OutputTextDeltaEvent, ResponsesAPIStreamEvents, ) from litellm.types.responses.main import build_web_search_call @@ -1658,3 +1660,34 @@ async def test_a_compaction_block_without_its_summary_streams_no_compaction_item assert [event["item"]["type"] for event in events if "item" in event] == ["message", "message"] assert [item["type"] for item in events[-1]["response"]["output"]] == ["message"] + + +def test_the_partial_response_billed_on_disconnect_carries_the_logging_objects_bill_as_hidden_response_cost() -> None: + from litellm.litellm_core_utils.hidden_params import get_hidden_params + + wrapper: Final = _FakeStreamWrapper([_chunk("Hello"), _chunk(" world", finish_reason="stop")]) + wrapper.logging_obj.response_cost_calculator.return_value = 0.00032 + iterator: Final = LiteLLMCompletionStreamingIterator( + model="claude-haiku-4-5", + litellm_custom_stream_wrapper=wrapper, # pyright: ignore[reportArgumentType] # the fake stands in for CustomStreamWrapper, as in _build_iterator + request_input="What is the weather in San Francisco?", + responses_api_request={}, + custom_llm_provider="anthropic", + litellm_metadata={}, + ) + relayed: Final = tuple( + itertools.takewhile(lambda event: not isinstance(event, OutputTextDeltaEvent), iterator) + ) + assert relayed, "the stream must relay at least one chunk before the client goes away" + + partial: Final = iterator.partial_response_on_disconnect() + + assert partial is not None + hidden: Final = get_hidden_params(partial.response) + assert hidden is not None and hidden["response_cost"] == 0.00032 + priced: Final = wrapper.logging_obj.response_cost_calculator.call_args.kwargs["result"] + assert priced.id == partial.response.id + assert (priced.usage.prompt_tokens, priced.usage.completion_tokens) == ( + partial.response.usage.input_tokens, + partial.response.usage.output_tokens, + ), "the bill is priced on the partial's own usage" diff --git a/tests/unit/responses/test_streaming_iterator.py b/tests/unit/responses/test_streaming_iterator.py index 664f4dba163..b9454ae6c1a 100644 --- a/tests/unit/responses/test_streaming_iterator.py +++ b/tests/unit/responses/test_streaming_iterator.py @@ -24,7 +24,7 @@ from litellm.exceptions import MidStreamFallbackError from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig -from litellm.responses.streaming_iterator import( +from litellm.responses.streaming_iterator import ( CachedResponsesAPIStreamingIterator, MockResponsesAPIStreamingIterator, ResponsesAPIStreamingIterator, @@ -845,25 +845,33 @@ def test_stamp_responses_usage_cost_stamps_computed_cost(): response = _responses_api_response_with_usage() logging_obj = Mock(spec=LiteLLMLoggingObj) + logging_obj.model_call_details = {"custom_llm_provider": "openai"} logging_obj.response_cost_calculator.return_value = 0.000704 stamp_responses_usage_cost(response, logging_obj) assert getattr(response.usage, "cost", None) == pytest.approx(0.000704) - logging_obj.response_cost_calculator.assert_called_once_with(result=response) + priced = logging_obj.response_cost_calculator.call_args.kwargs["result"] + assert priced.usage.prompt_tokens == response.usage.input_tokens, ( + "priced in the chat usage shape the spend row uses" + ) -def test_stamp_responses_usage_cost_keeps_provider_reported_cost(): +def test_stamp_responses_usage_cost_bills_the_spend_row_price_over_the_provider_reported_cost(): + from litellm.litellm_core_utils.core_helpers import response_cost_in_hidden_params from litellm.responses.streaming_iterator import stamp_responses_usage_cost response = _responses_api_response_with_usage() setattr(response.usage, "cost", 0.5) logging_obj = Mock(spec=LiteLLMLoggingObj) + logging_obj.model_call_details = {"custom_llm_provider": "openai"} + logging_obj.response_cost_calculator.return_value = 0.00032 stamp_responses_usage_cost(response, logging_obj) - assert getattr(response.usage, "cost", None) == pytest.approx(0.5) - logging_obj.response_cost_calculator.assert_not_called() + assert getattr(response.usage, "cost", None) == pytest.approx(0.00032), "the client sees what the row bills" + priced = logging_obj.response_cost_calculator.call_args.kwargs["result"] + assert response_cost_in_hidden_params(priced) == pytest.approx(0.5), "the reported cost reaches the calculator" def _unvalidated_response_with_dict_usage(usage: dict) -> ResponsesAPIResponse: @@ -881,6 +889,7 @@ def _unvalidated_response_with_dict_usage(usage: dict) -> ResponsesAPIResponse: def test_stamp_responses_usage_cost_keeps_provider_cost_from_dict_usage(): from litellm.responses.streaming_iterator import stamp_responses_usage_cost + response = _unvalidated_response_with_dict_usage( { "input_tokens": 29, @@ -891,26 +900,29 @@ def test_stamp_responses_usage_cost_keeps_provider_cost_from_dict_usage(): } ) logging_obj = Mock(spec=LiteLLMLoggingObj) + logging_obj.model_call_details = {"custom_llm_provider": "openai"} + logging_obj.response_cost_calculator.return_value = 3e-05 stamp_responses_usage_cost(response, logging_obj) assert isinstance(response.usage, ResponseAPIUsage) assert response.usage.cost == pytest.approx(3e-05) assert response.usage.output_tokens_details.reasoning_tokens == 117 - logging_obj.response_cost_calculator.assert_not_called() def test_stamp_responses_usage_cost_computes_cost_for_dict_usage_without_cost(): from litellm.responses.streaming_iterator import stamp_responses_usage_cost + response = _unvalidated_response_with_dict_usage({"input_tokens": 29, "output_tokens": 120, "total_tokens": 149}) logging_obj = Mock(spec=LiteLLMLoggingObj) + logging_obj.model_call_details = {"custom_llm_provider": "openai"} logging_obj.response_cost_calculator.return_value = 0.000704 stamp_responses_usage_cost(response, logging_obj) assert isinstance(response.usage, ResponseAPIUsage) assert response.usage.cost == pytest.approx(0.000704) - logging_obj.response_cost_calculator.assert_called_once_with(result=response) + logging_obj.response_cost_calculator.assert_called_once() def test_stamp_responses_usage_cost_survives_calculator_failure(): @@ -918,6 +930,7 @@ def test_stamp_responses_usage_cost_survives_calculator_failure(): response = _responses_api_response_with_usage() logging_obj = Mock(spec=LiteLLMLoggingObj) + logging_obj.model_call_details = {"custom_llm_provider": "openai"} logging_obj.response_cost_calculator.side_effect = RuntimeError("cost map unavailable") stamp_responses_usage_cost(response, logging_obj) @@ -1207,9 +1220,7 @@ async def test_completed_event_survives_a_failing_usage_estimate(): falls back to usage None.""" malformed_input: Final = [{"type": "message", "role": "user", "content": 42}] with pytest.raises(ValueError, match="Invalid content type"): - _estimate_usage_from_text( - "gpt-4o-mini", malformed_input, {"input": malformed_input}, "hello world" - ) + _estimate_usage_from_text("gpt-4o-mini", malformed_input, {"input": malformed_input}, "hello world") response = _responses_api_response_without_usage() iterator = _make_iterator( @@ -1305,7 +1316,14 @@ async def test_completed_event_with_a_dict_response_is_typed_and_billed(): assert usage.input_tokens > 0 assert usage.output_tokens > 0 assert usage.cost == pytest.approx(0.000704) - logging_obj.response_cost_calculator.assert_any_call(result=completed_response) + priced: Final = [ + call.kwargs["result"] + for call in logging_obj.response_cost_calculator.call_args_list + if isinstance(call.kwargs.get("result"), ResponsesAPIResponse) + ] + assert [(r.id, r.usage.prompt_tokens) for r in priced] == [(completed_response.id, usage.input_tokens)], ( + "priced once, in the chat usage shape the spend row uses" + ) def test_billed_terminal_response_keeps_a_response_that_already_has_usage(): @@ -1375,6 +1393,7 @@ def _vcr_outcome_gate(request, vcr): yield record_vcr_outcome(request, vcr) + @pytest.fixture(scope="function") def setup_and_teardown(): """ @@ -1392,6 +1411,7 @@ def setup_and_teardown(): loop.close() asyncio.set_event_loop(None) + class _FakeLoggingObj: def __init__(self): self.success_calls = 0 @@ -1432,10 +1452,12 @@ class _FakeLoggingObj: self.completion_start_time = completion_start_time self.model_call_details["completion_start_time"] = completion_start_time + async def _await_tasks_spawned_since(existing: set["asyncio.Task[object]"]) -> None: spawned: Final = asyncio.all_tasks() - existing await asyncio.wait_for(asyncio.gather(*spawned, return_exceptions=True), timeout=5) + def _make_completed_response(response_id: str = "resp_test") -> ResponseCompletedEvent: return ResponseCompletedEvent( type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, @@ -1463,6 +1485,7 @@ def _make_completed_response(response_id: str = "resp_test") -> ResponseComplete ), ) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_log_background_task_failure_logs_task_exceptions(monkeypatch): @@ -1485,6 +1508,7 @@ async def test_log_background_task_failure_logs_task_exceptions(monkeypatch): task.exception(), ) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_log_background_task_failure_ignores_cancelled_tasks(monkeypatch): @@ -1500,6 +1524,7 @@ async def test_log_background_task_failure_ignores_cancelled_tasks(monkeypatch): error_logger.assert_not_called() + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_content_part_done_event_supports_refusal_and_reasoning_text(): refusal_event = streaming_module._build_content_part_done_event( @@ -1527,6 +1552,7 @@ def test_content_part_done_event_supports_refusal_and_reasoning_text(): assert reasoning_event.part.reasoning == "because" assert unsupported_event is None + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_dump_response_object_handles_model_and_unknown_values(): response = ResponsesAPIResponse( @@ -1542,6 +1568,7 @@ def test_dump_response_object_handles_model_and_unknown_values(): assert streaming_module._dump_response_object({"type": "message"}) == {"type": "message"} assert streaming_module._dump_response_object(object()) == {} + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_responses_streaming_triggers_hooks(monkeypatch): @@ -1598,6 +1625,7 @@ async def test_responses_streaming_triggers_hooks(monkeypatch): assert seen["request_data"].get("litellm_params") is not None assert seen["call_type"] == CallTypes.responses + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_responses_streaming_calls_post_streaming_deployment_hook(monkeypatch): @@ -1638,6 +1666,7 @@ async def test_responses_streaming_calls_post_streaming_deployment_hook(monkeypa # reset callbacks litellm.callbacks = original_callbacks + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_responses_streaming_failure_triggers_failure_handlers(): @@ -1667,6 +1696,7 @@ async def test_responses_streaming_failure_triggers_failure_handlers(): assert logging_obj.failure_calls >= 1 assert logging_obj.async_failure_calls >= 1 + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_process_chunk_requires_provider_config(): iterator = ResponsesAPIStreamingIterator( @@ -1681,6 +1711,7 @@ def test_process_chunk_requires_provider_config(): with pytest.raises(ValueError, match="responses_api_provider_config is required"): iterator._process_chunk(json.dumps({"type": "response.completed"})) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_process_chunk_wraps_encrypted_content_with_model_id(): openai_types = streaming_module._get_openai_response_types() @@ -1715,6 +1746,7 @@ def test_process_chunk_wraps_encrypted_content_with_model_id(): assert event.item.encrypted_content.startswith("litellm_enc:") assert event.item.encrypted_content.endswith(";ciphertext") + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_process_chunk_completed_response_updates_id_and_usage_cost(monkeypatch): original_include_cost = litellm.include_cost_in_streaming_usage @@ -1768,6 +1800,7 @@ def test_process_chunk_completed_response_updates_id_and_usage_cost(monkeypatch) assert event.response.usage.cost == 1.23 completion_handler.assert_called_once() + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_process_chunk_failed_response_triggers_failure_logging(monkeypatch): openai_types = streaming_module._get_openai_response_types() @@ -1803,6 +1836,7 @@ def test_process_chunk_failed_response_triggers_failure_logging(monkeypatch): assert iterator.completed_response is event failure_handler.assert_called_once() + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_handle_logging_failed_response_uses_response_error_message(): @@ -1835,6 +1869,7 @@ async def test_handle_logging_failed_response_uses_response_error_message(): assert logging_obj.failure_calls == 1 assert logging_obj.async_failure_calls == 1 + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_process_chunk_returns_none_for_invalid_json_and_non_dict_payload(): class _NoopConfig: @@ -1853,6 +1888,7 @@ def test_process_chunk_returns_none_for_invalid_json_and_non_dict_payload(): assert iterator._process_chunk("not-json") is None assert iterator._process_chunk(json.dumps(["not", "a", "dict"])) is None + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_process_chunk_cost_annotation_failure_is_nonfatal(monkeypatch): original_include_cost = litellm.include_cost_in_streaming_usage @@ -1900,6 +1936,7 @@ def test_process_chunk_cost_annotation_failure_is_nonfatal(monkeypatch): assert event.response.usage.cost is None completion_handler.assert_called_once() + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_get_completed_response_object_accepts_direct_response(): logging_obj = _FakeLoggingObj() @@ -1916,6 +1953,7 @@ def test_get_completed_response_object_accepts_direct_response(): assert iterator._get_completed_response_object() is direct_response + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_responses_streaming_completed_event_persists_async_cache(): @@ -1964,6 +2002,7 @@ async def test_responses_streaming_completed_event_persists_async_cache(): assert json.loads(litellm.cache.async_add_cache.call_args.args[0])["id"] == iterator.completed_response.response.id litellm.cache = original_cache + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_responses_streaming_completed_event_persists_sync_cache(): logging_obj = _FakeLoggingObj() @@ -2009,6 +2048,7 @@ def test_responses_streaming_completed_event_persists_sync_cache(): assert json.loads(litellm.cache.add_cache.call_args.args[0])["id"] == iterator.completed_response.response.id litellm.cache = original_cache + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_log_completed_response_sync_direct_path(monkeypatch): hook_calls = {"post_call": 0, "metadata": 0} @@ -2050,6 +2090,7 @@ def test_log_completed_response_sync_direct_path(monkeypatch): assert hook_calls["post_call"] == 1 assert hook_calls["metadata"] == 1 + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_log_completed_response_falls_back_when_model_validate_fails(monkeypatch): class _BadSerializableResponse: @@ -2079,6 +2120,7 @@ def test_log_completed_response_falls_back_when_model_validate_fails(monkeypatch assert logging_obj.success_calls == 1 assert logging_obj.async_success_calls == 1 + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.parametrize( "scenario", @@ -2143,6 +2185,7 @@ def test_persist_completed_response_to_cache_guard_branches(monkeypatch, scenari expected_cached_flag = scenario == "already_cached" assert iterator._completed_response_cached is expected_cached_flag + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_build_synthetic_response_events_covers_annotations_function_calls_and_refusals(): original_include_cost = litellm.include_cost_in_streaming_usage @@ -2201,6 +2244,7 @@ def test_build_synthetic_response_events_covers_annotations_function_calls_and_r assert "response.function_call_arguments.done" in event_types assert event_types[-1] == "response.completed" + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_mock_responses_streaming_iterator_async_iteration_logs_completion( @@ -2251,6 +2295,7 @@ async def test_mock_responses_streaming_iterator_async_iteration_logs_completion assert hook_calls["post_call"] == 1 assert hook_calls["metadata"] == 1 + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_mock_responses_streaming_iterator_sync_iteration_logs_completion(monkeypatch): hook_calls = {"post_call": 0, "metadata": 0} @@ -2296,6 +2341,7 @@ def test_mock_responses_streaming_iterator_sync_iteration_logs_completion(monkey assert hook_calls["post_call"] == 1 assert hook_calls["metadata"] == 1 + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_cached_responses_stream_async_hit_triggers_success_callbacks( @@ -2355,6 +2401,7 @@ async def test_cached_responses_stream_async_hit_triggers_success_callbacks( litellm.cache.add_cache.assert_not_called() litellm.cache = original_cache + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_cached_responses_stream_sync_hit_triggers_success_callbacks(monkeypatch): hook_calls = {"post_call": 0, "metadata": 0} @@ -2453,7 +2500,9 @@ def test_faked_stream_bridged_to_chat_sends_tool_arguments_once(): tool_call_deltas: Final = tuple( tool_call for chunk in chunks - for tool_call in (chunk.choices[0].delta.tool_calls or ()) # comprehension-ok: flattening each chunk's tool calls + for tool_call in ( + chunk.choices[0].delta.tool_calls or () + ) # comprehension-ok: flattening each chunk's tool calls ) joined_arguments: Final = "".join(tool_call.function.arguments or "" for tool_call in tool_call_deltas) joined_text: Final = "".join(chunk.choices[0].delta.content or "" for chunk in chunks) @@ -2509,7 +2558,6 @@ def test_faked_stream_opens_items_in_progress_and_empty_like_a_real_stream(): assert [part.text for part in added_parts] == [""] - def _answer_with_reasoning_refusal_and_custom_tool_call() -> ResponsesAPIResponse: return ResponsesAPIResponse( id="resp_synthetic", @@ -2643,7 +2691,9 @@ async def test_shutdown_drain_waits_for_a_pending_logging_task(): @pytest.mark.asyncio async def test_shutdown_drain_gives_up_on_a_stuck_logging_task_at_its_timeout(): - stuck: Final = streaming_module.spawn_logging_task(asyncio.get_running_loop(), asyncio.Event().wait(), task_name="stuck") + stuck: Final = streaming_module.spawn_logging_task( + asyncio.get_running_loop(), asyncio.Event().wait(), task_name="stuck" + ) await asyncio.wait_for(streaming_module.drain_logging_tasks(timeout=0.05), timeout=1) From 83ae1bd63e24ee29dbf5bfa5dd3f1ba9991dce37 Mon Sep 17 00:00:00 2001 From: simonp9 <349053457+simonp9@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:46:36 -0700 Subject: [PATCH 04/10] fix(responses): drop params compact rejects and send Azure compaction to the v1 route (#48876) * fix(responses): drop metadata from compact requests to OpenAI and Azure Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): drop user and drop_params-rejected fields on compact and route Azure compaction to v1 * fix(responses): keep stream on compact and keep non-dated Azure api-versions on the v1 compact route * fix(responses): drop compact params only for providers verified to reject them --------- Co-authored-by: mateo Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/llms/azure/common_utils.py | 8 +- .../llms/azure/responses/transformation.py | 28 +++-- .../llms/openai/responses/transformation.py | 53 ++++++-- .../response/test_azure_transformation.py | 38 ++++++ .../llms/azure/test_azure_common_utils.py | 4 +- .../responses/test_transformation.py | 21 ++++ .../test_openai_responses_transformation.py | 116 ++++++++++++++++++ 7 files changed, 241 insertions(+), 27 deletions(-) diff --git a/litellm/llms/azure/common_utils.py b/litellm/llms/azure/common_utils.py index 4bd19889bcc..6bb6d584385 100644 --- a/litellm/llms/azure/common_utils.py +++ b/litellm/llms/azure/common_utils.py @@ -621,7 +621,7 @@ class BaseAzureLLM(BaseOpenAILLM): # For Azure v1 API, use standard OpenAI client instead of AzureOpenAI # See: https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#api-specs - if self._is_azure_v1_api_version(api_version): + if self.is_azure_v1_api_version(api_version): # Extract only params that OpenAI client accepts # Always use /openai/v1/ regardless of whether user passed "v1", "latest", or "preview" # The OpenAI client accepts a callable for `api_key` and re-invokes it @@ -922,7 +922,7 @@ class BaseAzureLLM(BaseOpenAILLM): else: new_url = api_base - if BaseAzureLLM._is_azure_v1_api_version(api_version): + if BaseAzureLLM.is_azure_v1_api_version(api_version): # ensure the request go to /openai/v1 and not just /openai if "/openai/v1" not in new_url: parsed_url: Final = httpx.URL(new_url) @@ -952,7 +952,7 @@ class BaseAzureLLM(BaseOpenAILLM): Returns None when ``api_version`` is a dated one, which still uses the deployment route. """ - if not BaseAzureLLM._is_azure_v1_api_version(api_version): + if not BaseAzureLLM.is_azure_v1_api_version(api_version): return None base_url: Final = httpx.URL(api_base) @@ -970,7 +970,7 @@ class BaseAzureLLM(BaseOpenAILLM): ) @staticmethod - def _is_azure_v1_api_version(api_version: str | None) -> bool: + def is_azure_v1_api_version(api_version: str | None) -> bool: if api_version is None: return False return api_version in {"preview", "latest", "v1"} diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index 647ad97e02c..cdeeef1d96c 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -54,6 +54,16 @@ def _resolve_responses_api_version(api_base: str | None, litellm_params: Mapping return api_version +def _v1_responses_url(api_base: str) -> httpx.URL: + parsed_url: Final = httpx.URL(api_base) + path: Final = parsed_url.path.rstrip("/") + root: Final = next( + (path.removesuffix(route) for route in ("/openai/v1/responses", "/openai/responses") if path.endswith(route)), + path, + ) + return parsed_url.copy_with(path=f"{root}/openai/v1/responses", query=None) + + def _with_responses_api_version(api_base: str | None, litellm_params: Mapping[str, object]) -> Mapping[str, object]: return MappingProxyType({**litellm_params, "api_version": _resolve_responses_api_version(api_base, litellm_params)}) @@ -247,20 +257,20 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): if api_base is None: raise ValueError("api_base is required for Azure WebSocket") - parsed_url: Final = httpx.URL(api_base) - path = parsed_url.path.rstrip("/") - # Strip existing /openai/responses path if the api_base already contains it - for suffix in ("/openai/v1/responses", "/openai/responses"): - if path.endswith(suffix): - path = path[: -len(suffix)] - break - scheme: Final = "wss" if parsed_url.scheme == "https" else "ws" - return str(parsed_url.copy_with(scheme=scheme, path=f"{path}/openai/v1/responses", query=None)) + v1_url: Final = _v1_responses_url(api_base) + return str(v1_url.copy_with(scheme="wss" if v1_url.scheme == "https" else "ws")) def model_in_websocket_url(self) -> bool: # Azure sends the model in the response.create body, not the URL return False + def compact_url(self, api_base: str) -> str: + params: Final = httpx.URL(api_base).params + v1_params: Final = ( + params if BaseAzureLLM.is_azure_v1_api_version(params.get("api-version")) else params.remove("api-version") + ) + return super().compact_url(str(_v1_responses_url(api_base).copy_with(params=v1_params))) + ######################################################### ########## DELETE RESPONSE API TRANSFORMATION ############## ######################################################### diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 9cb074a177e..033326e653f 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -66,6 +66,29 @@ _PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS: Final = frozenset({LlmProviders.AZURE, _PROVIDERS_REPLAYING_ONLY_THEIR_OWN_REASONING: Final = _PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS _ROLES_READING_PROMPT_CACHE_BREAKPOINTS: Final = frozenset({"system", "developer", "user"}) _PART_TYPES_READING_PROMPT_CACHE_BREAKPOINTS: Final = frozenset({"input_text", "input_image", "input_file"}) +_COMPACT_TRACKING_PARAMS: Final = frozenset({"metadata", "user"}) +_COMPACT_REJECTED_PARAMS: Final = frozenset( + { + "background", + "context_management", + "max_output_tokens", + "max_tool_calls", + "moderation", + "partial_images", + "prompt", + "safety_identifier", + "store", + "stream_options", + "temperature", + "tool_choice", + "top_logprobs", + "top_p", + "truncation", + } +) +_PROVIDERS_WITH_STRICT_COMPACT_PARAMS: Final = frozenset( + {LlmProviders.OPENAI, LlmProviders.AZURE, LlmProviders.AZURE_AI} +) _INPUT_ITEM: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) _CONTENT_PARTS: Final[TypeAdapter[Sequence[Mapping[str, object]]]] = TypeAdapter(Sequence[Mapping[str, object]]) @@ -1182,6 +1205,17 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): ######################################################### ########## COMPACT RESPONSE API TRANSFORMATION ########## ######################################################### + def compact_url(self, api_base: str) -> str: + parsed_url: Final = httpx.URL(api_base) + return str(parsed_url.copy_with(path=parsed_url.path.rstrip("/") + "/compact")) + + def compact_dropped_params(self, litellm_params: GenericLiteLLMParams) -> frozenset[str]: + if self.custom_llm_provider not in _PROVIDERS_WITH_STRICT_COMPACT_PARAMS: + return frozenset() + if _drops_params(litellm_params): + return _COMPACT_TRACKING_PARAMS | _COMPACT_REJECTED_PARAMS + return _COMPACT_TRACKING_PARAMS + def transform_compact_response_api_request( self, model: str, @@ -1197,10 +1231,8 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): OpenAI API expects the following request - POST /v1/responses/compact """ - # Preserve query params (e.g., api-version) while appending /compact. - parsed_url: Final = httpx.URL(api_base) - compact_path: Final = parsed_url.path.rstrip("/") + "/compact" - url: Final = str(parsed_url.copy_with(path=compact_path)) + url: Final = self.compact_url(api_base) + dropped_params: Final = self.compact_dropped_params(litellm_params) request_params: Final = cast( # cast-ok: the base signature leaves the params bag untyped "Mapping[str, object]", response_api_optional_request_params @@ -1214,14 +1246,11 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): ) if sanitized_tools is not None: response_api_optional_request_params["tools"] = sanitized_tools - data: Final = cast( # cast-ok: typed dict spread is inferred as Unknown by basedpyright - dict[str, object], - { - "model": model, - "input": replay_safe_input, - **response_api_optional_request_params, - }, - ) + data: Final = { + "model": model, + "input": replay_safe_input, + **{key: value for key, value in request_params.items() if key not in dropped_params}, + } return url, data diff --git a/tests/unit/llms/azure/response/test_azure_transformation.py b/tests/unit/llms/azure/response/test_azure_transformation.py index d46242fc5f7..29006901877 100644 --- a/tests/unit/llms/azure/response/test_azure_transformation.py +++ b/tests/unit/llms/azure/response/test_azure_transformation.py @@ -165,6 +165,44 @@ def test_explicit_v1_api_base_keeps_responses_api_version_selection( @pytest.mark.serial +@pytest.mark.parametrize( + ("api_base", "api_version", "expected_url"), + [ + ( + "https://x.openai.azure.com", + "2025-04-01-preview", + "https://x.openai.azure.com/openai/v1/responses/compact", + ), + ( + "https://x.openai.azure.com", + "preview", + "https://x.openai.azure.com/openai/v1/responses/compact?api-version=preview", + ), + ( + "https://gw.example.com/foundry/openai/openai/v1?tenant=example", + "v1", + "https://gw.example.com/foundry/openai/openai/v1/responses/compact?tenant=example&api-version=v1", + ), + ], +) +def test_compact_reaches_the_v1_route_and_drops_only_a_dated_api_version( + api_base: str, api_version: str, expected_url: str +) -> None: + config: Final = AzureOpenAIResponsesAPIConfig() + responses_url: Final = config.get_complete_url(api_base=api_base, litellm_params={"api_version": api_version}) + + url, _ = config.transform_compact_response_api_request( + model="gpt-5.4-mini", + input="hi", + response_api_optional_request_params={}, + api_base=responses_url, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert url == expected_url + + def test_response_id_path_requests_encode_response_id(): config = AzureOpenAIResponsesAPIConfig() api_base = ( diff --git a/tests/unit/llms/azure/test_azure_common_utils.py b/tests/unit/llms/azure/test_azure_common_utils.py index f7e2e73d426..8f083298555 100644 --- a/tests/unit/llms/azure/test_azure_common_utils.py +++ b/tests/unit/llms/azure/test_azure_common_utils.py @@ -1658,9 +1658,9 @@ def test_get_azure_ad_token_fallback_to_default_azure_credential( ) def test_is_azure_v1_api_version(api_version, expected): """ - Test that _is_azure_v1_api_version correctly identifies v1 API versions. + Test that is_azure_v1_api_version correctly identifies v1 API versions. """ - result = BaseAzureLLM._is_azure_v1_api_version(api_version=api_version) + result = BaseAzureLLM.is_azure_v1_api_version(api_version=api_version) assert result == expected diff --git a/tests/unit/llms/litellm_proxy/responses/test_transformation.py b/tests/unit/llms/litellm_proxy/responses/test_transformation.py index e710f4ac2c3..3d774b79eed 100644 --- a/tests/unit/llms/litellm_proxy/responses/test_transformation.py +++ b/tests/unit/llms/litellm_proxy/responses/test_transformation.py @@ -7,6 +7,7 @@ import respx import litellm from litellm.llms.litellm_proxy.responses.transformation import LiteLLMProxyResponsesAPIConfig +from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager from tests.unit.proxy.conftest import httpx_transport @@ -73,3 +74,23 @@ def test_get_complete_url_requires_api_base(monkeypatch: pytest.MonkeyPatch) -> monkeypatch.delenv("LITELLM_PROXY_API_BASE", raising=False) with pytest.raises(ValueError, match="api_base not set"): LiteLLMProxyResponsesAPIConfig().get_complete_url(api_base=None, litellm_params={}) + + +@pytest.mark.parametrize("drop_params", [False, True]) +def test_compact_request_forwards_metadata_and_user_to_the_downstream_proxy(drop_params: bool) -> None: + url, data = LiteLLMProxyResponsesAPIConfig().transform_compact_response_api_request( + model="gpt-5.5", + input="hi", + response_api_optional_request_params={"metadata": {"ticket": "qa-1"}, "user": "end-user-1", "temperature": 0.2}, + api_base="https://my-proxy.example.com/responses", + litellm_params=GenericLiteLLMParams(drop_params=drop_params), + headers={}, + ) + + assert url == "https://my-proxy.example.com/responses/compact" + assert {key: data[key] for key in data if key != "input"} == { + "model": "gpt-5.5", + "metadata": {"ticket": "qa-1"}, + "user": "end-user-1", + "temperature": 0.2, + } diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index 020f491dcfd..66e23115f3e 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -12,8 +12,11 @@ from pydantic import JsonValue, TypeAdapter import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig +from litellm.llms.azure_ai.responses.transformation import AzureAIResponsesAPIConfig from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig +from litellm.llms.hosted_vllm.responses.transformation import HostedVLLMResponsesAPIConfig from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.llms.xai.responses.transformation import XAIResponsesAPIConfig from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig from litellm.types.llms.openai import ( ImageGenerationPartialImageEvent, @@ -4277,3 +4280,116 @@ def test_dict_responses_input_filters_unset_reasoning_fields() -> None: function_call_item: Final = validated_input[2] assert function_call_item["type"] == "function_call" assert function_call_item["status"] == "completed" + + +_COMPACT_CONFIGS: Final = ( + pytest.param(OpenAIResponsesAPIConfig(), "https://api.openai.com/v1/responses", id="openai"), + pytest.param( + AzureOpenAIResponsesAPIConfig(), + "https://x.openai.azure.com/openai/responses?api-version=2025-04-01-preview", + id="azure", + ), + pytest.param(AzureAIResponsesAPIConfig(), "https://x.services.ai.azure.com/openai/v1/responses", id="azure_ai"), +) + + +@pytest.mark.parametrize(("config", "api_base"), _COMPACT_CONFIGS) +def test_compact_request_omits_metadata_and_user_and_keeps_the_rest_without_drop_params( + config: OpenAIResponsesAPIConfig, api_base: str +) -> None: + _, data = config.transform_compact_response_api_request( + model="gpt-5.4-mini", + input="hi", + response_api_optional_request_params={ + "metadata": {"ticket": "qa-1"}, + "user": "end-user-1", + "instructions": "be brief", + "previous_response_id": "resp_1", + "reasoning": {"effort": "low"}, + "temperature": 0.2, + }, + api_base=api_base, + litellm_params=GenericLiteLLMParams(drop_params=False), + headers={}, + ) + + assert {key: data[key] for key in data if key != "input"} == { + "model": "gpt-5.4-mini", + "instructions": "be brief", + "previous_response_id": "resp_1", + "reasoning": {"effort": "low"}, + "temperature": 0.2, + } + + +@pytest.mark.parametrize(("config", "api_base"), _COMPACT_CONFIGS) +def test_compact_request_drops_params_compact_rejects_under_drop_params( + config: OpenAIResponsesAPIConfig, api_base: str +) -> None: + _, data = config.transform_compact_response_api_request( + model="gpt-5.4-mini", + input="hi", + response_api_optional_request_params={ + "metadata": {"ticket": "qa-1"}, + "user": "end-user-1", + "temperature": 0.2, + "store": False, + "truncation": "auto", + "max_output_tokens": 100, + "tool_choice": "auto", + "reasoning": {"effort": "low"}, + "include": ["reasoning.encrypted_content"], + "text": {"format": {"type": "text"}}, + }, + api_base=api_base, + litellm_params=GenericLiteLLMParams(drop_params=True), + headers={}, + ) + + assert {key: data[key] for key in data if key != "input"} == { + "model": "gpt-5.4-mini", + "reasoning": {"effort": "low"}, + "include": ["reasoning.encrypted_content"], + "text": {"format": {"type": "text"}}, + } + + +@pytest.mark.parametrize( + ("config", "api_base"), + ( + pytest.param(XAIResponsesAPIConfig(), "https://api.x.ai/v1/responses", id="xai"), + pytest.param(HostedVLLMResponsesAPIConfig(), "http://vllm.example.com/v1/responses", id="hosted_vllm"), + ), +) +def test_compact_request_forwards_every_param_for_providers_not_known_to_reject_them( + config: OpenAIResponsesAPIConfig, api_base: str +) -> None: + _, data = config.transform_compact_response_api_request( + model="some-model", + input="hi", + response_api_optional_request_params={"metadata": {"ticket": "qa-1"}, "user": "end-user-1", "temperature": 0.2}, + api_base=api_base, + litellm_params=GenericLiteLLMParams(drop_params=True), + headers={}, + ) + + assert {key: data[key] for key in data if key != "input"} == { + "model": "some-model", + "metadata": {"ticket": "qa-1"}, + "user": "end-user-1", + "temperature": 0.2, + } + + +@pytest.mark.parametrize(("config", "api_base"), _COMPACT_CONFIGS) +def test_compact_request_keeps_stream_under_drop_params(config: OpenAIResponsesAPIConfig, api_base: str) -> None: + _, data = config.transform_compact_response_api_request( + model="gpt-5.4-mini", + input="hi", + response_api_optional_request_params={"stream": True, "stream_options": {"include_obfuscation": False}}, + api_base=api_base, + litellm_params=GenericLiteLLMParams(drop_params=True), + headers={}, + ) + + assert {key: data[key] for key in data if key != "input"} == {"model": "gpt-5.4-mini", "stream": True} From 4e5f585c2ad7f6f855f5dd0b1f232490949082bd Mon Sep 17 00:00:00 2001 From: mika-lt <349045450+mika-lt@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:01:07 -0700 Subject: [PATCH 05/10] fix(guardrails): scan Responses tool results in texts-mode guardrails (#48827) * fix(guardrails): scan Responses tool results in texts-mode guardrails * fix(guardrails): keep guardrail-added tools and non-text parts on Responses tool scans Tool result texts now come from the same conversion that builds the chat tool message, so items without a call_id, dict outputs, and outputs with no text parts stay aligned for Bedrock and PANW. A changed joined text replaces the first text part and keeps the other parts. scan_only_tool_results appends tools the guardrail returns with new names, like chat, and tool_result and computer_call_output items are scanned through the shared type list. * fix(guardrails): skip empty and screenshot tool outputs and keep text part types on Responses scans * fix(guardrails): keep Bedrock latest-turn masking aligned when tool outputs send no text * fix(guardrails): scan each Responses tool output text part on its own * fix(guardrails): split a Responses tool output's structured row when only textless outputs share its call_id * fix(guardrails): pair each Responses tool output with its own structured row --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../guardrail_translation/handler.py | 192 +++++- .../guardrail_translation/tool_merge.py | 16 + .../guardrail_hooks/bedrock_guardrails.py | 6 +- .../generic_guardrail_api/example_config.yaml | 27 +- .../panw_prisma_airs/panw_prisma_airs.py | 29 +- .../transformation.py | 110 ++- ...test_openai_responses_guardrail_handler.py | 640 ++++++++++++++++++ .../guardrail_hooks/test_panw_prisma_airs.py | 96 +++ ...test_function_call_output_normalization.py | 2 + 9 files changed, 1010 insertions(+), 108 deletions(-) diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index d37a12a91d6..559daf767c2 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -32,11 +32,11 @@ import copy import json import time import uuid -from collections.abc import Mapping, MutableSequence, Sequence +from collections.abc import Iterable, Mapping, MutableSequence, Sequence from dataclasses import dataclass -from itertools import accumulate, chain, repeat +from itertools import accumulate, chain, groupby, repeat from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, Union, cast +from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, TypeVar, Union, cast from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict @@ -55,7 +55,9 @@ from litellm.llms.base_llm.guardrail_translation.base_translation import ( ) from litellm.llms.base_llm.guardrail_translation.utils import ( blocked_responses_stream_usage, + effective_scan_only_tool_results_for_guardrail, effective_skip_system_message_for_guardrail, + effective_skip_tool_message_for_guardrail, list_field, merge_guardrailed_scoped_messages, role_out_of_guardrail_scope, @@ -65,7 +67,10 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( stream_item_items, unappliable_request_rewrite, ) -from litellm.llms.openai.responses.guardrail_translation.tool_merge import merge_guardrailed_tools +from litellm.llms.openai.responses.guardrail_translation.tool_merge import ( + append_returned_tools, + merge_guardrailed_tools, +) from litellm.responses.litellm_completion_transformation.compaction_items import ( compaction_summary, decode_compaction_item, @@ -74,7 +79,9 @@ from litellm.responses.litellm_completion_transformation.compaction_items import from litellm.responses.litellm_completion_transformation.custom_tools import without_malformed_tool_flags from litellm.responses.litellm_completion_transformation.tool_search import is_server_tool_search_tool from litellm.responses.litellm_completion_transformation.transformation import ( + TOOL_CALL_OUTPUT_ITEM_TYPES, LiteLLMCompletionResponsesConfig, + tool_output_to_chat_content, ) from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import ( @@ -270,6 +277,7 @@ _TOOL_CALL_PAYLOAD_EVENT_TYPES: Final = _TOOL_CALL_PAYLOAD_DELTA_EVENT_TYPES | f ) _OUTPUT_ITEM_EVENT_TYPES: Final = frozenset({"response.output_item.added", "response.output_item.done"}) _OUTPUT_TEXT_EVENT_TYPES: Final = frozenset({"response.output_text.delta", "response.output_text.done"}) +_TOOL_OUTPUT_TEXT_PART_TYPES: Final = frozenset({"input_text", "output_text", "text"}) _PATCHABLE_ITEM_FIELDS: Final[Mapping[str, str]] = MappingProxyType( {"function_call_output": "output", "custom_tool_call_output": "output", "message": "content"} ) @@ -519,10 +527,109 @@ def scannable_instructions(data: Mapping[str, object], *, skip_system: bool = Fa def _input_item_role(item: object) -> str: + if stream_item_field(item, "type") in TOOL_CALL_OUTPUT_ITEM_TYPES: + return "tool" role: Final = item.get("role") if isinstance(item, Mapping) else None return role.lower() if isinstance(role, str) else "" +def _scanned_text_field(item: Mapping[str, object]) -> str: + return "output" if item.get("type") in TOOL_CALL_OUTPUT_ITEM_TYPES else "content" + + +def _tool_output_part_text(part: object) -> str | None: + if not isinstance(part, dict): + return None + fields: Final = _item_fields_adapter.validate_python(part) + text: Final = fields.get("text") + return text if fields.get("type") in _TOOL_OUTPUT_TEXT_PART_TYPES and isinstance(text, str) else None + + +def tool_output_texts(item: object) -> tuple[tuple[int | None, str], ...]: + if not stream_item_field(item, "call_id") or stream_item_field(item, "type") == "computer_call_output": + return () + parts: Final = stream_item_items(item, "output") + part_texts: Final = tuple( + (index, text) for index, part in enumerate(parts) if (text := _tool_output_part_text(part)) + ) + if part_texts: + return part_texts + tool_content: Final = tool_output_to_chat_content(stream_item_field(item, "output")) + return ((None, tool_content),) if isinstance(tool_content, str) and tool_content else () + + +_Grouped: Final = TypeVar("_Grouped") + + +def _pair_call_id(pair: tuple[str, object]) -> str: + return pair[0] + + +def _grouped_by_call_id(pairs: Iterable[tuple[str, _Grouped]]) -> Mapping[str, tuple[_Grouped, ...]]: + ordered: Final = sorted(pairs, key=_pair_call_id) + return MappingProxyType( + {call_id: tuple(value for _, value in group) for call_id, group in groupby(ordered, key=_pair_call_id)} + ) + + +def _texts_by_tool_row( + messages: Sequence[Mapping[str, object]], input_data: "str | ResponseInputParam" +) -> Mapping[int, tuple[str, ...]]: + if isinstance(input_data, str): + return MappingProxyType({}) + output_texts_by_call_id: Final = _grouped_by_call_id( + (str(stream_item_field(item, "call_id")), tuple(text for _, text in tool_output_texts(item))) + for item in input_data + if stream_item_field(item, "type") in TOOL_CALL_OUTPUT_ITEM_TYPES + ) + row_indices_by_call_id: Final = _grouped_by_call_id( + (str(message.get("tool_call_id")), index) + for index, message in enumerate(messages) + if message.get("role") == "tool" + ) + return MappingProxyType( + dict( + chain.from_iterable( + zip(indices, output_texts_by_call_id[call_id], strict=True) + for call_id, indices in row_indices_by_call_id.items() + if len(indices) == len(output_texts_by_call_id.get(call_id, ())) + ) + ) + ) + + +def _is_string_tool_row(message: Mapping[str, object]) -> bool: + return message.get("role") == "tool" and isinstance(message.get("content"), str) + + +def _split_into_text_parts(message: AllMessageValues, texts: tuple[str, ...]) -> AllMessageValues: + if not _is_string_tool_row(message) or len(texts) < 2: + return message + return cast( # cast-ok: a tool row whose content is its text parts as chat text blocks + "AllMessageValues", {**message, "content": [{"type": "text", "text": text} for text in texts]} + ) + + +def _with_split_rows_restored( + rows: Sequence[AllMessageValues], split_rows: Sequence[tuple[AllMessageValues, AllMessageValues]] +) -> Sequence[AllMessageValues]: + if not isinstance(rows, list): + return rows + return [next((original for split, original in split_rows if row == split), row) for row in rows] + + +def _is_textless_tool_row(message: Mapping[str, object], texts: tuple[str, ...] | None) -> bool: + return _is_string_tool_row(message) and (not message.get("content") or texts == ()) + + +def _rewritten_whole_text(item: object, field: str, guardrail_response: str) -> object | None: + if isinstance(stream_item_field(item, field), str): + return guardrail_response + if tool_output_texts(item) == ((None, guardrail_response),): + return None + return guardrail_response + + def _patched_request_fields( raw_input: object, instructions: object, @@ -636,17 +743,28 @@ class OpenAIResponsesHandler(BaseTranslation): Handles both string input and list of message objects. """ - input_data: Final[str | ResponseInputParam | None] = data.get("input") - if not isinstance(input_data, (str, list)): + raw_input: Final[str | ResponseInputParam | None] = data.get("input") + if not isinstance(raw_input, (str, list)): return data + input_data: Final = cast("str | ResponseInputParam", raw_input) # cast-ok: fields read via stream_item_field skip_system: Final = effective_skip_system_message_for_guardrail(guardrail_to_apply) + skip_tool: Final = effective_skip_tool_message_for_guardrail(guardrail_to_apply) + scan_only: Final = effective_scan_only_tool_results_for_guardrail(guardrail_to_apply) structured_messages: Final = self.get_structured_messages(data) - scoped_indices: Final = scoped_structured_message_indices( - structured_messages or [], scan_only_tool_results=False, skip_system=skip_system, skip_tool=False + all_messages: Final = structured_messages or [] + texts_by_tool_row: Final = _texts_by_tool_row(all_messages, input_data) + scoped_indices: Final = tuple( + index + for index in scoped_structured_message_indices( + all_messages, scan_only_tool_results=scan_only, skip_system=skip_system, skip_tool=skip_tool + ) + if not _is_textless_tool_row(all_messages[index], texts_by_tool_row.get(index)) ) - scoped_structured_messages: Final = ( - [structured_messages[index] for index in scoped_indices] if structured_messages else None + split_rows: Final = tuple( + (_split_into_text_parts(all_messages[index], texts_by_tool_row.get(index, ())), all_messages[index]) + for index in scoped_indices ) + scoped_structured_messages: Final = [split for split, _ in split_rows] if structured_messages else None raw_tools: Final = data.get("tools") original_tools: Final[tuple[Mapping[str, object], ...]] = ( tuple(raw_tools) if isinstance(raw_tools, list) else () @@ -661,7 +779,12 @@ class OpenAIResponsesHandler(BaseTranslation): ) ) extracted: Final = self._extract_guardrail_inputs( - data, input_data, flattened_tool_groups, skip_system=skip_system + data, + input_data, + () if scan_only else flattened_tool_groups, + skip_system=skip_system, + skip_tool=skip_tool, + scan_only=scan_only, ) if not extracted.inputs.get("texts"): return data @@ -674,13 +797,14 @@ class OpenAIResponsesHandler(BaseTranslation): logging_obj=litellm_logging_obj, ) self._apply_guardrailed_tools_to_data( - data, original_tools, flattened_tool_groups, guardrailed_inputs.get("tools") + data, original_tools, flattened_tool_groups, guardrailed_inputs.get("tools"), scan_only=scan_only ) written_back: Final = self._written_back_request_fields( data, structured_messages or (), scoped_indices, scoped_structured_messages, + tuple((split, original) for split, original in split_rows if split is not original), guardrail_to_apply, guardrailed_inputs, ) @@ -730,8 +854,10 @@ class OpenAIResponsesHandler(BaseTranslation): flattened_tool_groups: Sequence[Sequence[Mapping[str, object]]], *, skip_system: bool = False, + skip_tool: bool = False, + scan_only: bool = False, ) -> _ExtractedInputs: - instructions: Final = scannable_instructions(data, skip_system=skip_system) + instructions: Final = scannable_instructions(data, skip_system=skip_system or scan_only) texts_to_check: Final[list[str]] = [] if instructions is None else [instructions] images_to_check: Final[list[str]] = [] task_mappings: Final[list[tuple[int, int | None]]] = [] @@ -745,11 +871,15 @@ class OpenAIResponsesHandler(BaseTranslation): ) ) if isinstance(input_data, str): - texts_to_check.append(input_data) + if not scan_only: + texts_to_check.append(input_data) else: for msg_idx, message in enumerate(input_data): if role_out_of_guardrail_scope( - _input_item_role(message), skip_system_message=skip_system, skip_tool_message=False + _input_item_role(message), + skip_system_message=skip_system, + skip_tool_message=skip_tool, + scan_only_tool_results=scan_only, ): continue self._extract_input_text_and_images( @@ -775,12 +905,14 @@ class OpenAIResponsesHandler(BaseTranslation): structured_messages: Sequence[AllMessageValues], scoped_indices: Sequence[int], scoped_structured_messages: Sequence[AllMessageValues] | None, + split_rows: Sequence[tuple[AllMessageValues, AllMessageValues]], guardrail_to_apply: "CustomGuardrail", guardrailed_inputs: GenericGuardrailAPIInputs, ) -> _RequestFields | None: - guardrailed: Final = guardrailed_inputs.get("structured_messages") - if guardrailed is None or guardrailed is scoped_structured_messages: + returned: Final = guardrailed_inputs.get("structured_messages") + if returned is None or returned is scoped_structured_messages: return None + guardrailed: Final = _with_split_rows_restored(returned, split_rows) covers_full_request: Final = len(scoped_indices) == len(structured_messages) or ( guardrail_to_apply.structured_messages_cover_full_request() and len(guardrailed) == len(structured_messages) ) @@ -814,11 +946,14 @@ class OpenAIResponsesHandler(BaseTranslation): original_tools: Sequence[Mapping[str, object]], flattened_tool_groups: Sequence[Sequence[Mapping[str, object]]], guardrailed_tools: Sequence[ChatCompletionToolParam] | None, + *, + scan_only: bool, ) -> None: if guardrailed_tools is None: return + merge: Final = append_returned_tools if scan_only else merge_guardrailed_tools data["tools"] = list( # rebind-ok: in-place request rewrite - merge_guardrailed_tools(original_tools, flattened_tool_groups, guardrailed_tools) + merge(original_tools, flattened_tool_groups, guardrailed_tools) ) def _extract_input_text_and_images( @@ -839,6 +974,11 @@ class OpenAIResponsesHandler(BaseTranslation): texts_to_check.append(summary) task_mappings.append((msg_idx, None)) return + if message.get("type") in TOOL_CALL_OUTPUT_ITEM_TYPES: + for content_idx, text in tool_output_texts(message): + texts_to_check.append(text) + task_mappings.append((msg_idx, content_idx)) + return content: Final = message.get("content", None) if content is None: return @@ -890,18 +1030,20 @@ class OpenAIResponsesHandler(BaseTranslation): ) continue - content = messages[msg_idx].get("content", None) + field = _scanned_text_field(messages[msg_idx]) + content = messages[msg_idx].get(field, None) if content is None: continue - if isinstance(content, str) and content_idx_optional is None: - # Replace string content with guardrail response - messages[msg_idx]["content"] = guardrail_response + if content_idx_optional is None: + rewritten = _rewritten_whole_text(messages[msg_idx], field, guardrail_response) + if rewritten is not None: + messages[msg_idx][field] = rewritten - elif isinstance(content, list) and content_idx_optional is not None: + elif isinstance(content, list): # Replace specific text item in list content - if isinstance(messages[msg_idx]["content"][content_idx_optional], dict): - messages[msg_idx]["content"][content_idx_optional]["text"] = guardrail_response + if isinstance(messages[msg_idx][field][content_idx_optional], dict): + messages[msg_idx][field][content_idx_optional]["text"] = guardrail_response async def process_output_response( self, diff --git a/litellm/llms/openai/responses/guardrail_translation/tool_merge.py b/litellm/llms/openai/responses/guardrail_translation/tool_merge.py index f295a864b71..e529cba4edf 100644 --- a/litellm/llms/openai/responses/guardrail_translation/tool_merge.py +++ b/litellm/llms/openai/responses/guardrail_translation/tool_merge.py @@ -6,6 +6,7 @@ from typing import Final, TypeAlias from pydantic import BaseModel, TypeAdapter, ValidationError from litellm._logging import verbose_logger +from litellm.llms.base_llm.guardrail_translation.utils import merge_returned_tools_into_request_tools, openai_tool_name from litellm.responses.litellm_completion_transformation.custom_tools import custom_tool_grammar_suffix from litellm.responses.litellm_completion_transformation.transformation import ( NAMESPACE_DESCRIPTION_SEPARATOR, @@ -183,3 +184,18 @@ def merge_guardrailed_tools( tuple(tool for key, tool in zip(guardrailed_keys, guardrailed) if key not in owned_keys) ) return tuple(chain(merged_originals, appended)) + + +def append_returned_tools( + original_tools: Sequence[Tool], + flattened_groups: Sequence[Sequence[Tool]], + returned_tools: Iterable[object], +) -> tuple[Tool, ...]: + request_chat_tools: Final = tuple(chain.from_iterable(flattened_groups)) + additions: Final = merge_returned_tools_into_request_tools( + request_tools=request_chat_tools, returned_tools=_validated_tools(returned_tools), tool_name=openai_tool_name + )[len(request_chat_tools) :] + return ( + *original_tools, + *LiteLLMCompletionResponsesConfig.transform_chat_completion_tool_params_to_responses_api_tools(additions), + ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index adda465ad95..9fa37e1827e 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -49,6 +49,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, bedrock_bearer_token, run_aws_signing from litellm.llms.bedrock.guardrail_attachments import find_request_attachments from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, get_async_httpx_client, httpxSpecialProvider, ) @@ -286,9 +287,12 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): streaming_sampling_rate: int | None = None, streaming_end_of_stream_only: bool | None = None, streaming_buffer_release_on_scan: bool | None = None, + async_handler: AsyncHTTPHandler | None = None, **kwargs, ): - self.async_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.GuardrailCallback) + self.async_handler = async_handler or get_async_httpx_client( + llm_provider=httpxSpecialProvider.GuardrailCallback + ) self._set_streaming_params( BedrockGuardrailStreamingParams.from_extras( MappingProxyType( diff --git a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/example_config.yaml b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/example_config.yaml index 9b5c4e557f3..665d8cde4cc 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/example_config.yaml +++ b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/example_config.yaml @@ -6,20 +6,19 @@ model_list: model: gpt-4 api_key: os.environ/OPENAI_API_KEY -litellm_settings: - guardrails: - - guardrail_name: "my-generic-guardrail" - litellm_params: - guardrail: generic_guardrail_api - mode: pre_call # Options: pre_call, post_call, during_call, [pre_call, post_call] - api_key: os.environ/GENERIC_GUARDRAIL_API_KEY # Optional if using Bearer auth - api_base: http://localhost:8080 # Required. Endpoint /beta/litellm_basic_guardrail_api is automatically appended - unreachable_fallback: fail_closed # Options: fail_closed (default, raise), fail_open (proceed if endpoint unreachable or upstream returns 502/503/504) - default_on: false # Set to true to apply to all requests by default - additional_provider_specific_params: - # Any additional parameters your guardrail API needs - api_version: "v1" - custom_param: "value" +guardrails: + - guardrail_name: "my-generic-guardrail" + litellm_params: + guardrail: generic_guardrail_api + mode: pre_call # Options: pre_call, post_call, during_call, [pre_call, post_call] + api_key: os.environ/GENERIC_GUARDRAIL_API_KEY # Optional if using Bearer auth + api_base: http://localhost:8080 # Required. Endpoint /beta/litellm_basic_guardrail_api is automatically appended + unreachable_fallback: fail_closed # Options: fail_closed (default, raise), fail_open (proceed if endpoint unreachable or upstream returns 502/503/504) + default_on: false # Set to true to apply to all requests by default + additional_provider_specific_params: + # Any additional parameters your guardrail API needs + api_version: "v1" + custom_param: "value" # Usage examples: diff --git a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py index d50001ba636..3940a320fb8 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py +++ b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py @@ -30,6 +30,7 @@ from litellm.integrations.custom_guardrail import ( from litellm.llms.base_llm.guardrail_translation.utils import ( effective_scan_only_tool_results_for_guardrail, effective_skip_system_message_for_guardrail, + effective_skip_tool_message_for_guardrail, role_out_of_guardrail_scope, ) from litellm.llms.custom_httpx.http_handler import ( @@ -37,12 +38,16 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, ) -from litellm.llms.openai.responses.guardrail_translation.handler import scannable_instructions +from litellm.llms.openai.responses.guardrail_translation.handler import ( + scannable_instructions, + tool_output_texts, +) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.callback_utils import ( add_guardrail_scan_id, add_guardrail_to_applied_guardrails_header, ) +from litellm.responses.litellm_completion_transformation.transformation import TOOL_CALL_OUTPUT_ITEM_TYPES from litellm.types.guardrails import GuardrailEventHooks from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import AllMessageValues @@ -112,12 +117,16 @@ class _ResponsesInputItem(LiteLLMBaseModel): type: str | None = None role: str | None = None content: str | tuple[_ResponsesContentPart, ...] | None = None + output: object = None + call_id: object = None - def text_count(self, *, skip_system: bool) -> int: - if role_out_of_guardrail_scope( - (self.role or "").lower(), skip_system_message=skip_system, skip_tool_message=False - ): + def text_count(self, *, skip_system: bool, skip_tool: bool) -> int: + is_tool_output: Final = self.type in TOOL_CALL_OUTPUT_ITEM_TYPES + role: Final = "tool" if is_tool_output else (self.role or "").lower() + if role_out_of_guardrail_scope(role, skip_system_message=skip_system, skip_tool_message=skip_tool): return 0 + if is_tool_output: + return len(tool_output_texts(self)) if isinstance(self.content, str): return 1 if self.content is None: @@ -1692,13 +1701,14 @@ class PanwPrismaAirsHandler(CustomGuardrail): request_data: Mapping[str, object], *, skip_system: bool, + skip_tool: bool, ) -> frozenset[int] | None: """Return the ``texts`` indices flattened from Responses ``reasoning`` input items. The Responses translation handler gives those model-authored items the default ``user`` role, so the latest-turn selection must not mistake one for a human turn. Empty for requests without a Responses ``input`` item list; None when the raw items - (after the leading ``instructions`` text, both minus whatever ``skip_system`` drops) + (after the leading ``instructions`` text, both minus whatever ``skip_system`` and ``skip_tool`` drop) do not account for every entry of ``texts``. """ try: @@ -1708,7 +1718,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): if not isinstance(raw_input, tuple): return frozenset() offset: Final = 0 if scannable_instructions(request_data, skip_system=skip_system) is None else 1 - counts: Final = tuple(item.text_count(skip_system=skip_system) for item in raw_input) + counts: Final = tuple(item.text_count(skip_system=skip_system, skip_tool=skip_tool) for item in raw_input) if offset + sum(counts) != len(texts): return None starts: Final = itertools.accumulate(counts, initial=offset) @@ -1738,7 +1748,10 @@ class PanwPrismaAirsHandler(CustomGuardrail): if sources is None: return None reasoning: Final = self._reasoning_item_text_indices( - texts, request_data, skip_system=effective_skip_system_message_for_guardrail(self) + texts, + request_data, + skip_system=effective_skip_system_message_for_guardrail(self), + skip_tool=effective_skip_tool_message_for_guardrail(self), ) if reasoning is None: return None diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index bc92bc9cb3a..a342d704a9d 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -321,7 +321,7 @@ def _responses_file_part(item: Mapping[str, object]) -> _ResponsesFilePart: ) -def _decode_responses_media(item: Mapping[str, object]) -> _ResponsesMediaPart | None: +def decode_responses_media(item: Mapping[str, object]) -> _ResponsesMediaPart | None: kind: Final = item.get("type") if kind in ("input_image", "image_url", "computer_screenshot") and any(key in item for key in ("image_url", "url")): image: Final = _responses_image_part(item) @@ -383,6 +383,52 @@ def _attribute_fields(value: object) -> dict[str, object]: return dict(cast("Iterable[tuple[str, object]]", value)) # cast-ok: dict() raises on non-pair values, as before +def tool_output_to_chat_content( + output: object, +) -> str | list[ToolMessageContentPart]: + if output is None: + return "" + if isinstance(output, str): + return output + + single_media: Final = decode_responses_media(output) if isinstance(output, Mapping) else None + if single_media is not None: + return [single_media] + if isinstance(output, list): + normalized_blocks: Final[list[ToolMessageContentPart]] = [] + text_acc: Final[list[str]] = [] + for part in output: + if not isinstance(part, dict): + continue + part_type = part.get("type") + if part_type in ("input_text", "output_text", "text"): + txt = part.get("text") + if isinstance(txt, str) and txt: + text_acc.append(txt) + normalized_blocks.append({"type": "text", "text": txt}) + elif (media := decode_responses_media(part)) is not None: + normalized_blocks.append(media) + + if any(b.get("type") != "text" for b in normalized_blocks): + return normalized_blocks + if text_acc: + return "".join(text_acc) + try: + return json.dumps(output) + except Exception: + return str(output) + + try: + return json.dumps(output) + except Exception: + return str(output) + + +TOOL_CALL_OUTPUT_ITEM_TYPES: Final = frozenset( + {"function_call_output", "custom_tool_call_output", "computer_call_output", "tool_result"} +) + + def _input_item_role(input_item: Mapping[str, object]) -> str: return cast(str, input_item.get("role") or "user") # cast-ok: client-supplied role forwarded verbatim, unvalidated @@ -2371,12 +2417,7 @@ class LiteLLMCompletionResponsesConfig: """ Check if the input item is a tool call output """ - return input_item.get("type") in [ - "function_call_output", - "custom_tool_call_output", - "computer_call_output", - "tool_result", # Anthropic/MCP format - ] + return input_item.get("type") in TOOL_CALL_OUTPUT_ITEM_TYPES @staticmethod def _is_input_item_function_call(input_item: Mapping[str, object]) -> bool: @@ -2400,58 +2441,7 @@ class LiteLLMCompletionResponsesConfig: if not call_id: return [] - def _normalize_function_call_output_to_tool_content( - output: object, - ) -> str | list[ToolMessageContentPart]: - """ - Preserve structured media for downstream chat adapters and token estimation. - Text-only outputs retain their existing string representation. - """ - if output is None: - return "" - if isinstance(output, str): - return output - - single_media: Final = _decode_responses_media(output) if isinstance(output, Mapping) else None - if single_media is not None: - return [single_media] - # Some adapters represent tool output as a list of "input_*" parts - if isinstance(output, list): - normalized_blocks: Final[list[ToolMessageContentPart]] = [] - text_acc: Final[list[str]] = [] - for part in output: - if not isinstance(part, dict): - continue - part_type = part.get("type") - if part_type in ("input_text", "output_text", "text"): - txt = part.get("text") - if isinstance(txt, str) and txt: - text_acc.append(txt) - normalized_blocks.append({"type": "text", "text": txt}) - elif (media := _decode_responses_media(part)) is not None: - normalized_blocks.append(media) - - if any(b.get("type") != "text" for b in normalized_blocks): - return normalized_blocks - if text_acc: - return "".join(text_acc) - try: - # last resort: keep something meaningful for providers that require a string - import json as _json - - return _json.dumps(output) - except Exception: - return str(output) - - # Fallback for dict/number/etc. - try: - import json as _json - - return _json.dumps(output) - except Exception: - return str(output) - - tool_content: Final = _normalize_function_call_output_to_tool_content(tool_call_output.get("output")) + tool_content: Final = tool_output_to_chat_content(tool_call_output.get("output")) output_cache_control: Final = _input_item_cache_control(tool_call_output) tool_output_message: Final = ( ChatCompletionToolMessage(role="tool", content=tool_content, tool_call_id=str(call_id)) @@ -2606,7 +2596,7 @@ class LiteLLMCompletionResponsesConfig: content_list.append(item) elif isinstance(item, dict): if ( - media := _decode_responses_media( + media := decode_responses_media( fields := _validate_responses_media(_STR_KEY_DICT_ADAPTER, item) ) ) is not None: diff --git a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py index 0d1b1aca9bc..673e880a1c0 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -7,12 +7,14 @@ with guardrail transformations. import copy import json +import uuid from collections.abc import Callable from typing import Any, Final, List, Literal, Optional, Tuple from unittest.mock import AsyncMock, MagicMock, patch import logging +import httpx import pytest @@ -33,6 +35,10 @@ from litellm.llms.openai.responses.guardrail_translation.handler import ( ) from litellm.llms.openai.responses.guardrail_translation.tool_merge import merge_guardrailed_tools from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api import GenericGuardrailAPI +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import BedrockGuardrail +from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ContentFilterGuardrail +from litellm.types.guardrails import ContentFilterAction, ContentFilterPattern from litellm.types.llms.openai import ChatCompletionToolCallChunk from litellm.responses.litellm_completion_transformation.compaction_items import compaction_summary from litellm.responses.litellm_completion_transformation.transformation import ( @@ -83,6 +89,7 @@ class RecordingMaskingGuardrail(MockGuardrail): super().__init__(guardrail_name=guardrail_name) self.seen_texts: list[list[str]] = [] self.seen_message_contents: list[list[object]] = [] + self.seen_tools: list[object] = [] async def apply_guardrail( self, @@ -93,6 +100,7 @@ class RecordingMaskingGuardrail(MockGuardrail): ) -> GenericGuardrailAPIInputs: self.seen_texts.append(list(inputs.get("texts", []))) self.seen_message_contents.append([m["content"] for m in inputs.get("structured_messages") or []]) + self.seen_tools.append(inputs.get("tools")) return await super().apply_guardrail(inputs, request_data, input_type, logging_obj) @@ -588,6 +596,638 @@ class TestSkipSystemMessageScopesInstructions: ] +_TOOL_TURNS: Final = ( + ( + {"type": "function_call", "call_id": "call_1", "name": "get_weather", "arguments": '{"city": "Paris"}'}, + "function_call_output", + ), + ( + {"type": "custom_tool_call", "call_id": "call_1", "name": "run_sql", "input": "select 1"}, + "custom_tool_call_output", + ), +) + + +def _tool_result_request(call_item: dict[str, str], output_type: str, output: object) -> dict[str, object]: + return { + "model": "gpt-5.4-mini", + "input": [ + {"role": "user", "content": "Weather?"}, + call_item, + {"type": output_type, "call_id": "call_1", "output": output}, + ], + } + + +def _bedrock_guardrail( + name: str, bedrock_api: Callable[[httpx.Request], httpx.Response], **settings: bool +) -> BedrockGuardrail: + return BedrockGuardrail( + guardrail_name=name, + guardrailIdentifier="test-guardrail", + guardrailVersion="DRAFT", + aws_access_key_id="test-access-key", + aws_secret_access_key="test-secret-key", + aws_region_name="us-east-1", + async_handler=AsyncHTTPHandler(transport=httpx.MockTransport(bedrock_api)), + **settings, + ) + + +def _bedrock_block_texts(request: httpx.Request) -> list[str]: + return [block["text"]["text"] for block in json.loads(request.content)["content"]] + + +class TestToolResultItemsReachTextsGuardrails: + """A Responses tool result is scanned and rewritten in its `output` field the way a chat `tool` + message's content is, and skip_tool_message_in_guardrail keeps it out of the scan the same way.""" + + @pytest.mark.asyncio + @pytest.mark.parametrize(("call_item", "output_type"), _TOOL_TURNS, ids=["function", "custom"]) + async def test_string_output_is_scanned_and_rewritten_in_place( + self, call_item: dict[str, str], output_type: str + ) -> None: + guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test") + + result: Final = await OpenAIResponsesHandler().process_input_messages( + _tool_result_request(call_item, output_type, "SECRET forecast"), guardrail + ) + + assert guardrail.seen_texts == [["Weather?", "SECRET forecast"]] + assert result["input"] == [ + {"role": "user", "content": "Weather? [GUARDRAILED]"}, + call_item, + {"type": output_type, "call_id": "call_1", "output": "SECRET forecast [GUARDRAILED]"}, + ] + + @pytest.mark.asyncio + async def test_content_part_output_is_scanned_and_rewritten_in_its_part(self) -> None: + guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test") + call_item, output_type = _TOOL_TURNS[0] + image_part: Final = {"type": "input_image", "image_url": "https://example.com/map.png"} + + result: Final = await OpenAIResponsesHandler().process_input_messages( + _tool_result_request( + call_item, output_type, [{"type": "input_text", "text": "SECRET forecast"}, image_part] + ), + guardrail, + ) + + assert guardrail.seen_texts == [["Weather?", "SECRET forecast"]] + assert result["input"][2]["output"] == [ + {"type": "input_text", "text": "SECRET forecast [GUARDRAILED]"}, + image_part, + ] + + @pytest.mark.asyncio + @pytest.mark.parametrize("part_type", ["input_text", "output_text"]) + async def test_text_only_parts_are_scanned_and_rewritten_part_by_part(self, part_type: str) -> None: + guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test") + call_item, output_type = _TOOL_TURNS[0] + parts: Final = [ + {"type": part_type, "text": "SECRET forecast: ", "annotations": []}, + {"type": part_type, "text": "sunny"}, + ] + + result: Final = await OpenAIResponsesHandler().process_input_messages( + _tool_result_request(call_item, output_type, parts), guardrail + ) + + assert guardrail.seen_texts == [["Weather?", "SECRET forecast: ", "sunny"]] + assert result["input"][2]["output"] == [ + {"type": part_type, "text": "SECRET forecast: [GUARDRAILED]", "annotations": []}, + {"type": part_type, "text": "sunny [GUARDRAILED]"}, + ] + + @pytest.mark.asyncio + async def test_a_pattern_ending_at_a_part_boundary_blocks_like_it_does_on_chat(self) -> None: + access_key: Final = "AKIA" + "SYNTHETIC0000000" + content_filter: Final = ContentFilterGuardrail( + guardrail_name="secrets", + patterns=[ + ContentFilterPattern( + pattern_type="prebuilt", pattern_name="aws_access_key", action=ContentFilterAction.BLOCK + ) + ], + ) + call_item, output_type = _TOOL_TURNS[0] + parts: Final = [ + {"type": "input_text", "text": f"key={access_key}"}, + {"type": "input_text", "text": "region=us-east-1"}, + ] + + with pytest.raises(HTTPException) as blocked: + await OpenAIResponsesHandler().process_input_messages( + _tool_result_request(call_item, output_type, parts), content_filter + ) + + assert blocked.value.status_code == 400 + + @pytest.mark.asyncio + @pytest.mark.parametrize("empty_output", ["", None], ids=["empty_string", "null"]) + async def test_empty_tool_output_sends_no_text_and_stays_as_sent(self, empty_output: str | None) -> None: + guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test") + call_item, output_type = _TOOL_TURNS[0] + + result: Final = await OpenAIResponsesHandler().process_input_messages( + _tool_result_request(call_item, output_type, empty_output), guardrail + ) + + assert guardrail.seen_texts == [["Weather?"]] + assert result["input"][2] == {"type": output_type, "call_id": "call_1", "output": empty_output} + + @pytest.mark.asyncio + @pytest.mark.parametrize("latest_user_only", [False, True], ids=["all_texts", "latest_user_only"]) + @pytest.mark.parametrize( + "tool_output", + [ + {"type": "function_call_output", "call_id": "call_1", "output": ""}, + {"type": "function_call_output", "call_id": "call_1", "output": None}, + { + "type": "computer_call_output", + "call_id": "call_1", + "output": {"type": "computer_screenshot", "file_id": "file-screen"}, + }, + ], + ids=["empty_string", "null", "file_id_screenshot"], + ) + async def test_bedrock_masking_applies_to_a_request_with_a_textless_tool_output( + self, tool_output: dict[str, object], latest_user_only: bool + ) -> None: + sent_texts: Final[list[list[str]]] = [] + + def bedrock_masking_emails(request: httpx.Request) -> httpx.Response: + texts: Final = _bedrock_block_texts(request) + sent_texts.append(texts) + outputs: Final = [{"text": text.replace("jane@example.com", "{EMAIL}")} for text in texts] + return httpx.Response(200, json={"action": "GUARDRAIL_INTERVENED", "outputs": outputs}) + + guardrail: Final = _bedrock_guardrail( + "bedrock-pre", bedrock_masking_emails, experimental_use_latest_role_message_only=latest_user_only + ) + + request: Final = { + "model": "gpt-5.4-mini", + "input": [ + {"role": "user", "content": "Look up the order."}, + {"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"}, + copy.deepcopy(tool_output), + {"role": "user", "content": "Email jane@example.com the result."}, + ], + } + + result: Final = await OpenAIResponsesHandler().process_input_messages(request, guardrail) + + expected_texts: Final = ( + [["Email jane@example.com the result."]] + if latest_user_only + else [["Look up the order.", "Email jane@example.com the result."]] + ) + assert sent_texts == expected_texts + assert result["input"][3]["content"] == "Email {EMAIL} the result." + assert result["input"][2] == tool_output + + @pytest.mark.asyncio + async def test_bedrock_latest_user_masking_applies_with_a_multi_part_tool_output(self) -> None: + sent_texts: Final[list[list[str]]] = [] + + def bedrock_masking_emails(request: httpx.Request) -> httpx.Response: + texts: Final = _bedrock_block_texts(request) + sent_texts.append(texts) + outputs: Final = [{"text": text.replace("jane@example.com", "{EMAIL}")} for text in texts] + return httpx.Response(200, json={"action": "GUARDRAIL_INTERVENED", "outputs": outputs}) + + guardrail: Final = _bedrock_guardrail( + "bedrock-pre", bedrock_masking_emails, experimental_use_latest_role_message_only=True + ) + tool_output: Final = { + "type": "function_call_output", + "call_id": "call_1", + "output": [{"type": "input_text", "text": "order 42"}, {"type": "input_text", "text": "shipped"}], + } + request: Final = { + "model": "gpt-5.4-mini", + "input": [ + {"role": "user", "content": "Look up the order."}, + {"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"}, + copy.deepcopy(tool_output), + {"role": "user", "content": "Email jane@example.com the result."}, + ], + } + + result: Final = await OpenAIResponsesHandler().process_input_messages(request, guardrail) + + assert sent_texts == [["Email jane@example.com the result."]] + assert result["input"][3]["content"] == "Email {EMAIL} the result." + assert result["input"][2] == tool_output + + @pytest.mark.asyncio + async def test_bedrock_only_scan_new_messages_scans_only_the_new_tool_result(self) -> None: + sent_texts: Final[list[list[str]]] = [] + + def bedrock_allowing(request: httpx.Request) -> httpx.Response: + sent_texts.append(_bedrock_block_texts(request)) + return httpx.Response(200, json={"action": "NONE", "outputs": []}) + + guardrail: Final = _bedrock_guardrail("bedrock-incremental", bedrock_allowing, only_scan_new_messages=True) + + first_turn: Final = [ + {"role": "user", "content": "Weather in Paris and Rome?"}, + {"type": "function_call", "call_id": "call_1", "name": "get_weather", "arguments": '{"city": "Paris"}'}, + {"type": "function_call_output", "call_id": "call_1", "output": "Paris: sunny"}, + ] + second_turn: Final = [ + *first_turn, + {"type": "function_call", "call_id": "call_2", "name": "get_weather", "arguments": '{"city": "Rome"}'}, + {"type": "function_call_output", "call_id": "call_2", "output": "Rome: rain"}, + ] + session: Final = {"litellm_session_id": f"session-{uuid.uuid4()}"} + + for turn in (first_turn, second_turn): + await OpenAIResponsesHandler().process_input_messages( + {"model": "gpt-5.4-mini", "input": copy.deepcopy(turn), **session}, guardrail + ) + + assert sent_texts == [["Weather in Paris and Rome?", "Paris: sunny"], ["Rome: rain"]] + + @pytest.mark.asyncio + async def test_file_id_screenshot_is_not_scanned_as_text_or_rewritten(self) -> None: + guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test") + screenshot_output: Final = { + "type": "computer_call_output", + "call_id": "cu_1", + "output": {"type": "computer_screenshot", "file_id": "file-SECRET-screen"}, + } + request: Final = { + "model": "gpt-5.4-mini", + "input": [ + {"role": "user", "content": "Open the page"}, + {"type": "computer_call", "call_id": "cu_1", "id": "cu_1", "action": {"type": "screenshot"}}, + copy.deepcopy(screenshot_output), + ], + } + + result: Final = await OpenAIResponsesHandler().process_input_messages(request, guardrail) + + assert guardrail.seen_texts == [["Open the page"]] + assert result["input"][2] == screenshot_output + + @pytest.mark.asyncio + async def test_text_parts_between_file_images_are_rewritten_in_place(self) -> None: + guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test") + call_item, output_type = _TOOL_TURNS[0] + logo_part: Final = {"type": "input_image", "file_id": "file-logo"} + image_part: Final = {"type": "input_image", "file_id": "file-map"} + parts: Final = [ + logo_part, + {"type": "input_text", "text": "SECRET forecast: "}, + image_part, + {"type": "input_text", "text": "sunny"}, + ] + + result: Final = await OpenAIResponsesHandler().process_input_messages( + _tool_result_request(call_item, output_type, parts), guardrail + ) + + assert guardrail.seen_texts == [["Weather?", "SECRET forecast: ", "sunny"]] + assert result["input"][2]["output"] == [ + logo_part, + {"type": "input_text", "text": "SECRET forecast: [GUARDRAILED]"}, + image_part, + {"type": "input_text", "text": "sunny [GUARDRAILED]"}, + ] + + @pytest.mark.asyncio + async def test_dict_output_is_rewritten_as_the_text_the_guardrail_returns(self) -> None: + guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test") + call_item, output_type = _TOOL_TURNS[0] + + result: Final = await OpenAIResponsesHandler().process_input_messages( + _tool_result_request(call_item, output_type, {"result": "SECRET forecast"}), guardrail + ) + + assert result["input"][2]["output"] == '{"result": "SECRET forecast"} [GUARDRAILED]' + + @pytest.mark.asyncio + async def test_dict_output_the_guardrail_leaves_unchanged_stays_a_dict(self) -> None: + guardrail: Final = TextsReplacingGuardrail(guardrail_name="test", texts=("[MASKED]", '{"result": "sunny"}')) + call_item, output_type = _TOOL_TURNS[0] + + result: Final = await OpenAIResponsesHandler().process_input_messages( + _tool_result_request(call_item, output_type, {"result": "sunny"}), guardrail + ) + + assert result["input"][0]["content"] == "[MASKED]" + assert result["input"][2]["output"] == {"result": "sunny"} + + @pytest.mark.asyncio + async def test_tool_result_without_call_id_is_left_out_like_its_structured_row(self) -> None: + guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test") + call_item, output_type = _TOOL_TURNS[0] + orphan: Final = {"type": output_type, "output": "SECRET orphan"} + request: Final = { + "model": "gpt-5.4-mini", + "input": [ + {"role": "user", "content": "Weather?"}, + call_item, + dict(orphan), + {"type": output_type, "call_id": "call_1", "output": "SECRET forecast"}, + ], + } + + result: Final = await OpenAIResponsesHandler().process_input_messages(request, guardrail) + + assert guardrail.seen_texts == [["Weather?", "SECRET forecast"]] + assert "SECRET orphan" not in guardrail.seen_message_contents[0] + assert result["input"][2] == orphan + assert result["input"][3]["output"] == "SECRET forecast [GUARDRAILED]" + + @pytest.mark.asyncio + async def test_a_textless_output_sharing_a_call_id_keeps_the_text_output_in_structured_messages(self) -> None: + guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test") + call_item, output_type = _TOOL_TURNS[0] + request: Final = _tool_result_request(call_item, output_type, "") + request["input"].append({"type": output_type, "call_id": "call_1", "output": "SECRET forecast"}) + + result: Final = await OpenAIResponsesHandler().process_input_messages(request, guardrail) + + assert guardrail.seen_texts == [["Weather?", "SECRET forecast"]] + assert "SECRET forecast" in guardrail.seen_message_contents[0] + assert "" not in guardrail.seen_message_contents[0] + assert result["input"][3]["output"] == "SECRET forecast [GUARDRAILED]" + + @pytest.mark.asyncio + async def test_multi_part_outputs_sharing_a_call_id_each_keep_their_own_text_in_structured_messages( + self, + ) -> None: + guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test") + call_item, output_type = _TOOL_TURNS[0] + request: Final = _tool_result_request( + call_item, + output_type, + [{"type": "input_text", "text": "first "}, {"type": "input_text", "text": "output"}], + ) + request["input"].append( + { + "type": output_type, + "call_id": "call_1", + "output": [{"type": "input_text", "text": "second "}, {"type": "input_text", "text": "output"}], + } + ) + + result: Final = await OpenAIResponsesHandler().process_input_messages(request, guardrail) + + assert guardrail.seen_texts == [["Weather?", "first ", "output", "second ", "output"]] + assert guardrail.seen_message_contents[0][2:] == [ + [{"type": "text", "text": "first "}, {"type": "text", "text": "output"}], + [{"type": "text", "text": "second "}, {"type": "text", "text": "output"}], + ] + assert [part["text"] for part in result["input"][3]["output"]] == [ + "second [GUARDRAILED]", + "output [GUARDRAILED]", + ] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "textless_output", + [ + {"type": "function_call_output", "call_id": "call_1", "output": ""}, + {"type": "function_call_output", "call_id": "call_1", "output": None}, + { + "type": "computer_call_output", + "call_id": "call_1", + "output": {"type": "computer_screenshot", "file_id": "file-screen"}, + }, + ], + ids=["empty_string", "null", "file_id_screenshot"], + ) + async def test_multi_part_output_sharing_a_call_id_with_a_textless_output_lists_its_parts_in_structured_messages( + self, textless_output: dict[str, object] + ) -> None: + guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test") + call_item, output_type = _TOOL_TURNS[0] + request: Final = _tool_result_request( + call_item, + output_type, + [{"type": "input_text", "text": "first "}, {"type": "input_text", "text": "output"}], + ) + request["input"].append(copy.deepcopy(textless_output)) + + await OpenAIResponsesHandler().process_input_messages(request, guardrail) + + assert guardrail.seen_texts == [["Weather?", "first ", "output"]] + assert guardrail.seen_message_contents[0][2:] == [ + [{"type": "text", "text": "first "}, {"type": "text", "text": "output"}], + ] + + @pytest.mark.asyncio + async def test_tool_result_item_is_scanned_like_function_call_output(self) -> None: + guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test") + call_item, _ = _TOOL_TURNS[0] + + result: Final = await OpenAIResponsesHandler().process_input_messages( + _tool_result_request(call_item, "tool_result", "SECRET forecast"), guardrail + ) + + assert guardrail.seen_texts == [["Weather?", "SECRET forecast"]] + assert guardrail.seen_message_contents[0][2] == "SECRET forecast" + assert result["input"][2]["output"] == "SECRET forecast [GUARDRAILED]" + + @pytest.mark.asyncio + async def test_text_only_parts_the_guardrail_leaves_unchanged_keep_their_parts(self) -> None: + call_item, output_type = _TOOL_TURNS[0] + parts: Final = [{"type": "input_text", "text": "SECRET forecast: "}, {"type": "input_text", "text": "sunny"}] + + result: Final = await OpenAIResponsesHandler().process_input_messages( + _tool_result_request(call_item, output_type, copy.deepcopy(parts)), CustomGuardrail(guardrail_name="test") + ) + + assert result["input"][2]["output"] == parts + + @pytest.mark.asyncio + async def test_text_parts_a_structured_guardrail_returns_unchanged_keep_their_parts(self) -> None: + class StructuredEchoGuardrail(CustomGuardrail): + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict[str, object], + input_type: Literal["request", "response"], + logging_obj: LiteLLMLoggingObj | None = None, + ) -> GenericGuardrailAPIInputs: + return {**inputs, "structured_messages": copy.deepcopy(inputs.get("structured_messages") or [])} + + call_item, output_type = _TOOL_TURNS[0] + parts: Final = [ + {"type": "output_text", "text": "SECRET forecast: ", "annotations": []}, + {"type": "output_text", "text": "sunny"}, + ] + + result: Final = await OpenAIResponsesHandler().process_input_messages( + _tool_result_request(call_item, output_type, copy.deepcopy(parts)), + StructuredEchoGuardrail(guardrail_name="t"), + ) + + assert result["input"][2]["output"] == parts + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("output", "expected_texts"), + [ + pytest.param("SECRET forecast", ["SECRET forecast"], id="string"), + pytest.param([{"type": "input_text", "text": "SECRET forecast"}], ["SECRET forecast"], id="one_part"), + pytest.param( + [{"type": "input_text", "text": "SECRET "}, {"type": "output_text", "text": "forecast"}], + ["SECRET ", "forecast"], + id="parts", + ), + pytest.param( + [ + {"type": "input_text", "text": "SECRET "}, + {"type": "input_image", "image_url": "https://example.com/map.png"}, + {"type": "input_text", "text": ""}, + {"type": "input_text", "text": "forecast"}, + ], + ["SECRET ", "forecast"], + id="parts_with_image", + ), + pytest.param( + [ + {"type": "input_text", "text": "SECRET forecast"}, + {"type": "summary_text", "text": "not tool output"}, + ], + ["SECRET forecast"], + id="non_text_part_with_text_field", + ), + pytest.param( + ["SECRET forecast", {"type": "input_text", "text": "visible"}], + ["visible"], + id="non_dict_part", + ), + pytest.param([], ["[]"], id="empty"), + pytest.param({"result": "SECRET forecast"}, ['{"result": "SECRET forecast"}'], id="dict"), + pytest.param( + [{"type": "input_image", "file_id": "file-map"}], + ['[{"type": "input_image", "file_id": "file-map"}]'], + id="no_text_parts", + ), + pytest.param({"type": "input_image", "image_url": "https://example.com/map.png"}, [], id="single_image"), + ], + ) + async def test_tool_result_texts_match_the_structured_tool_message( + self, output: object, expected_texts: list[str] + ) -> None: + guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test") + call_item, output_type = _TOOL_TURNS[0] + + await OpenAIResponsesHandler().process_input_messages( + _tool_result_request(call_item, output_type, output), guardrail + ) + + tool_content: Final = guardrail.seen_message_contents[0][2] + structured_texts: Final = ( + [tool_content] + if isinstance(tool_content, str) + else [part["text"] for part in tool_content if part.get("type") == "text"] + ) + tool_texts: Final = guardrail.seen_texts[0][1:] + assert tool_texts == expected_texts + assert tool_texts == structured_texts + + @pytest.mark.asyncio + @pytest.mark.parametrize(("call_item", "output_type"), _TOOL_TURNS, ids=["function", "custom"]) + async def test_skip_tool_message_keeps_the_tool_result_out_of_texts_and_rows( + self, call_item: dict[str, str], output_type: str + ) -> None: + guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test") + guardrail.skip_tool_message_in_guardrail = True + tool_result: Final = {"type": output_type, "call_id": "call_1", "output": "SECRET forecast"} + + result: Final = await OpenAIResponsesHandler().process_input_messages( + _tool_result_request(call_item, output_type, "SECRET forecast"), guardrail + ) + + assert guardrail.seen_texts == [["Weather?"]] + assert "SECRET forecast" not in guardrail.seen_message_contents[0] + assert result["input"][0] == {"role": "user", "content": "Weather? [GUARDRAILED]"} + assert result["input"][2] == tool_result + + @pytest.mark.asyncio + @pytest.mark.parametrize(("call_item", "output_type"), _TOOL_TURNS, ids=["function", "custom"]) + async def test_scan_only_tool_results_sends_only_the_tool_result( + self, call_item: dict[str, str], output_type: str + ) -> None: + guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test") + guardrail.scan_only_tool_results = True + tool: Final = {"type": "function", "name": "get_weather", "parameters": {"type": "object", "properties": {}}} + request: Final = { + **_tool_result_request(call_item, output_type, "SECRET forecast"), + "instructions": "Answer briefly", + "tools": [tool], + } + + result: Final = await OpenAIResponsesHandler().process_input_messages(request, guardrail) + + assert guardrail.seen_texts == [["SECRET forecast"]] + assert guardrail.seen_message_contents == [["SECRET forecast"]] + assert guardrail.seen_tools == [None] + assert result["instructions"] == "Answer briefly" + assert result["tools"] == [tool] + assert result["input"] == [ + {"role": "user", "content": "Weather?"}, + call_item, + {"type": output_type, "call_id": "call_1", "output": "SECRET forecast [GUARDRAILED]"}, + ] + + @pytest.mark.asyncio + async def test_scan_only_tool_results_appends_tools_the_guardrail_adds(self) -> None: + added_tool: Final = { + "type": "function", + "function": {"name": "retrieve", "description": "Fetch the full text", "parameters": {"type": "object"}}, + } + renamed_original: Final = { + "type": "function", + "function": {"name": "get_weather", "description": "rewritten", "parameters": {"type": "object"}}, + } + + class ToolAddingGuardrail(RecordingMaskingGuardrail): + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict[str, object], + input_type: Literal["request", "response"], + logging_obj: LiteLLMLoggingObj | None = None, + ) -> GenericGuardrailAPIInputs: + masked: Final = await super().apply_guardrail(inputs, request_data, input_type, logging_obj) + return {**masked, "tools": [renamed_original, added_tool]} + + guardrail: Final = ToolAddingGuardrail(guardrail_name="test") + guardrail.scan_only_tool_results = True + call_item, output_type = _TOOL_TURNS[0] + tool: Final = {"type": "function", "name": "get_weather", "parameters": {"type": "object", "properties": {}}} + request: Final = {**_tool_result_request(call_item, output_type, "SECRET forecast"), "tools": [tool]} + + result: Final = await OpenAIResponsesHandler().process_input_messages(request, guardrail) + + assert guardrail.seen_tools == [None] + assert result["tools"][0] == tool + assert [(t["type"], t["name"]) for t in result["tools"]] == [ + ("function", "get_weather"), + ("function", "retrieve"), + ] + assert result["tools"][1]["description"] == "Fetch the full text" + + @pytest.mark.asyncio + async def test_scan_only_tool_results_leaves_a_request_without_tool_results_unscanned(self) -> None: + guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test") + guardrail.scan_only_tool_results = True + request: Final = {"model": "gpt-5.4-mini", "instructions": "Answer briefly", "input": "Weather?"} + + result: Final = await OpenAIResponsesHandler().process_input_messages(dict(request), guardrail) + + assert guardrail.seen_texts == [] + assert result == request + + class TestOpenAIResponsesHandlerOutputProcessing: """Test output processing functionality""" diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py index 83301db5d86..58055daea4c 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py @@ -4834,6 +4834,83 @@ class TestPanwAirsLatestRoleMessageOnlyEveryRequestShape: None, id="function_call_output", ), + pytest.param( + ( + {"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"}, + { + "type": "function_call_output", + "call_id": "call_1", + "output": [{"type": "input_text", "text": "tool "}, {"type": "input_text", "text": "result"}], + }, + ), + None, + id="function_call_output_text_parts", + ), + pytest.param( + ( + {"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"}, + { + "type": "function_call_output", + "call_id": "call_1", + "output": [ + {"type": "input_text", "text": "tool "}, + {"type": "input_image", "image_url": "https://example.com/chart.png"}, + {"type": "input_text", "text": "result"}, + ], + }, + ), + None, + id="function_call_output_text_and_image_parts", + ), + pytest.param( + ( + {"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"}, + {"type": "function_call_output", "output": "orphan tool result"}, + {"type": "function_call_output", "call_id": "call_1", "output": "tool result"}, + ), + None, + id="function_call_output_without_call_id", + ), + pytest.param( + ( + {"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"}, + {"type": "tool_result", "call_id": "call_1", "output": "tool result"}, + ), + None, + id="tool_result", + ), + pytest.param( + ( + {"type": "computer_call", "call_id": "cu_1", "id": "cu_1", "action": {"type": "screenshot"}}, + { + "type": "computer_call_output", + "call_id": "cu_1", + "output": {"type": "computer_screenshot", "image_url": "https://example.com/screen.png"}, + }, + ), + None, + id="computer_call_output", + ), + pytest.param( + ( + {"type": "computer_call", "call_id": "cu_1", "id": "cu_1", "action": {"type": "screenshot"}}, + { + "type": "computer_call_output", + "call_id": "cu_1", + "output": {"type": "computer_screenshot", "file_id": "file-screen"}, + }, + ), + None, + id="computer_call_output_file_id", + ), + pytest.param( + ( + {"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "call_1", "output": ""}, + ), + None, + id="function_call_output_empty", + ), pytest.param( ({"type": "reasoning", "id": "rs_1", "summary": [{"type": "summary_text", "text": "thinking"}]},), None, @@ -4898,6 +4975,25 @@ class TestPanwAirsLatestRoleMessageOnlyEveryRequestShape: assert [call.kwargs["content"] for call in mock_api.call_args_list] == [self.LATEST] + @pytest.mark.asyncio + async def test_flag_true_with_skip_tool_still_scans_only_the_latest_turn_on_responses(self) -> None: + from litellm.llms.openai.responses.guardrail_translation.handler import ( + OpenAIResponsesHandler, + ) + + handler = make_handler(experimental_use_latest_role_message_only=True) + handler.skip_tool_message_in_guardrail = True + request_data = self._responses_request( + {"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "call_1", "output": "tool result"}, + {"role": "user", "content": self.LATEST}, + ) + patcher, mock_api = self._scan(handler) + with patcher: + await OpenAIResponsesHandler().process_input_messages(data=request_data, guardrail_to_apply=handler) + + assert [call.kwargs["content"] for call in mock_api.call_args_list] == [self.LATEST] + @pytest.mark.asyncio async def test_flag_false_responses_scans_instructions_and_full_history(self) -> None: from litellm.llms.openai.responses.guardrail_translation.handler import ( diff --git a/tests/unit/responses/litellm_completion_transformation/test_function_call_output_normalization.py b/tests/unit/responses/litellm_completion_transformation/test_function_call_output_normalization.py index 326583d94f5..65800e42124 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_function_call_output_normalization.py +++ b/tests/unit/responses/litellm_completion_transformation/test_function_call_output_normalization.py @@ -111,6 +111,8 @@ def test_media_normalization_preserves_formats_references_and_metadata(kind: str "keep", ), ([{"type": "input_image", "file_id": "file-only"}], '[{"type": "input_image", "file_id": "file-only"}]'), + ({"ids": {7}}, "{'ids': {7}}"), + ([{"type": "unknown", "ids": {7}}], "[{'type': 'unknown', 'ids': {7}}]"), ], ) def test_tool_output_without_usable_media_keeps_its_existing_fallback(output: object, expected: str) -> None: From 0f68b9919ca31df448bcc89e3dfe344b6ff039b9 Mon Sep 17 00:00:00 2001 From: simonp9 <349053457+simonp9@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:05:03 -0700 Subject: [PATCH 06/10] fix(guardrails): skip embeddings input in google_text_moderation (#48934) * fix(guardrails): skip embeddings input in google_text_moderation and openai_moderations Since v1.103.0 the proxy dispatches these legacy moderation callbacks on every route, and they read the embeddings input field as text, so benign embedding inputs were blocked. Return early for embedding call types, matching the Aim and Cato guardrails. * test(guardrails): type the embeddings moderation test parameters * fix(guardrails): keep openai_moderations moderating embeddings, skip only google_text_moderation Google's moderation scores topic categories like Finance and Health, so it blocks documents being indexed by subject. OpenAI's moderation scores harm categories only, so it keeps moderating embedding input as it has since v1.103.0 * fix(guardrails): skip Gigachat passthrough embeddings in google_text_moderation Treat an allm_passthrough_route call whose endpoint ends in embeddings as non-conversational, log the skip, return None from the early exit, and type the hook's data so the basedpyright gate does not grow * fix(guardrails): match only plain embeddings passthrough paths in google_text_moderation --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../google_text_moderation.py | 10 ++- litellm/proxy/guardrails/_content_utils.py | 14 ++++ .../proxy/guardrails/test_content_utils.py | 25 ++++++ .../guardrails/test_guardrail_coverage.py | 79 ++++++++++++++++++- 4 files changed, 125 insertions(+), 3 deletions(-) diff --git a/enterprise/enterprise_hooks/google_text_moderation.py b/enterprise/enterprise_hooks/google_text_moderation.py index 7dbbc9ed6ce..ddf092a143e 100644 --- a/enterprise/enterprise_hooks/google_text_moderation.py +++ b/enterprise/enterprise_hooks/google_text_moderation.py @@ -15,7 +15,10 @@ import litellm from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import UserAPIKeyAuth -from litellm.proxy.guardrails._content_utils import iter_message_text +from litellm.proxy.guardrails._content_utils import ( + is_non_conversational_request, + iter_message_text, +) from litellm.types.utils import CallTypesLiteral _THRESHOLD_VALUE: Final = TypeAdapter(object) @@ -94,7 +97,7 @@ class ENTERPRISE_GoogleTextModeration(CustomLogger): async def async_moderation_hook( self, - data: dict, + data: dict[str, object], user_api_key_dict: UserAPIKeyAuth, call_type: CallTypesLiteral, ): @@ -102,6 +105,9 @@ class ENTERPRISE_GoogleTextModeration(CustomLogger): - Calls Google's Text Moderation API - Rejects request if it fails safety check """ + if is_non_conversational_request(call_type, data): + verbose_proxy_logger.debug("Google text moderation: skipping non-conversational call type %s", call_type) + return # Covers multimodal list content + Responses-API input. text = "".join(iter_message_text(data)) if text: diff --git a/litellm/proxy/guardrails/_content_utils.py b/litellm/proxy/guardrails/_content_utils.py index 9bea22acd71..51934c99db6 100644 --- a/litellm/proxy/guardrails/_content_utils.py +++ b/litellm/proxy/guardrails/_content_utils.py @@ -62,6 +62,20 @@ def is_non_conversational_call_type(call_type: str) -> bool: return call_type in NON_CONVERSATIONAL_CALL_TYPES +_PASSTHROUGH_EMBEDDINGS_ENDPOINT: Final = re.compile(r"/?(?:[\w-]+/)*embeddings/?") + + +def is_non_conversational_request(call_type: str, data: Mapping[str, object]) -> bool: + if is_non_conversational_call_type(call_type): + return True + endpoint: Final = data.get("endpoint") + return ( + call_type == "allm_passthrough_route" + and isinstance(endpoint, str) + and _PASSTHROUGH_EMBEDDINGS_ENDPOINT.fullmatch(endpoint) is not None + ) + + TEXT_PART_TYPES: Final[frozenset[str]] = frozenset( {"text", "input_text", "output_text", "summary_text", "reasoning_text"} ) diff --git a/tests/unit/proxy/guardrails/test_content_utils.py b/tests/unit/proxy/guardrails/test_content_utils.py index 21d96a3ea97..c50646d233d 100644 --- a/tests/unit/proxy/guardrails/test_content_utils.py +++ b/tests/unit/proxy/guardrails/test_content_utils.py @@ -12,6 +12,7 @@ from litellm.proxy.guardrails._content_utils import ( build_inspection_messages, has_non_string_content, is_non_conversational_call_type, + is_non_conversational_request, is_string_batch_input, iter_message_text, iter_tool_json_text, @@ -819,6 +820,30 @@ def test_is_non_conversational_call_type_defaults_to_inspecting_unknown_call_typ assert is_non_conversational_call_type("some_future_call_type") is False +@pytest.mark.parametrize( + ("call_type", "endpoint", "expected"), + [ + ("aembedding", None, True), + ("allm_passthrough_route", "embeddings", True), + ("allm_passthrough_route", "/v1/embeddings/", True), + ("allm_passthrough_route", "chat/completions", False), + ("allm_passthrough_route", "model/amazon.titan-embed-text-v2:0/invoke", False), + ("allm_passthrough_route", "fine_tuned_embeddings_jobs", False), + ("allm_passthrough_route", "chat/completions#embeddings", False), + ("allm_passthrough_route", "chat/completions?x=embeddings", False), + ("allm_passthrough_route", "chat/completions;embeddings", False), + ("allm_passthrough_route", None, False), + ("acompletion", "embeddings", False), + ], +) +def test_is_non_conversational_request_skips_only_embeddings_requests( + call_type: str, endpoint: str | None, expected: bool +): + data: Final[dict[str, object]] = {"endpoint": endpoint} if endpoint is not None else {} + + assert is_non_conversational_request(call_type, data) is expected + + @pytest.mark.parametrize( "tool_input", ( diff --git a/tests/unit/proxy/guardrails/test_guardrail_coverage.py b/tests/unit/proxy/guardrails/test_guardrail_coverage.py index 1feae213eba..493eac430d0 100644 --- a/tests/unit/proxy/guardrails/test_guardrail_coverage.py +++ b/tests/unit/proxy/guardrails/test_guardrail_coverage.py @@ -12,17 +12,19 @@ inspection by the relevant guardrail hook: not just ``choices[0]``. """ +from types import SimpleNamespace from typing import Any, Dict, Final from unittest.mock import AsyncMock, MagicMock, patch import pytest +from fastapi import HTTPException from httpx import Request, Response import litellm from litellm import DualCache from litellm.constants import DEFAULT_OPENAI_MODERATIONS_MODEL from litellm.proxy._types import UserAPIKeyAuth -from litellm.types.utils import Choices, Message, ModelResponse +from litellm.types.utils import CallTypesLiteral, Choices, Message, ModelResponse @pytest.fixture @@ -905,3 +907,78 @@ async def test_google_text_moderation_inspects_multimodal_content(user_api_key): ) assert seen_documents == ["hello world"] + + +class _FinanceFlaggingLanguageClient: + def __init__(self) -> None: + self.moderated_texts: Final[list[str]] = [] # mutable-ok: records what reached Google + + def moderate_text(self, request: SimpleNamespace) -> SimpleNamespace: + self.moderated_texts.append(request.document.content) + return SimpleNamespace(moderation_categories=[SimpleNamespace(name="Finance", confidence=0.885)]) + + +def _google_text_moderation_flagging_finance(): + from enterprise.enterprise_hooks.google_text_moderation import ( + ENTERPRISE_GoogleTextModeration, + ) + + guard: Final = ENTERPRISE_GoogleTextModeration.__new__(ENTERPRISE_GoogleTextModeration) + guard.client = _FinanceFlaggingLanguageClient() + guard.language_document = lambda content, type_: SimpleNamespace(content=content) + guard.moderate_text_request = lambda document: SimpleNamespace(document=document) + guard.document_type = "PLAIN_TEXT" + guard.finance_confidence_threshold = 0.8 + return guard + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("call_type", "data"), + [ + ("aembedding", {"model": "text-embedding-ada-002", "input": "stocks"}), + ("embedding", {"model": "text-embedding-ada-002", "input": ["stocks", "mortgage"]}), + ("allm_passthrough_route", {"model": "gigachat-embeddings", "endpoint": "embeddings", "input": ["stocks"]}), + ], +) +async def test_google_text_moderation_leaves_embedding_input_unmoderated( + user_api_key: UserAPIKeyAuth, call_type: CallTypesLiteral, data: dict[str, object] +): + guard: Final = _google_text_moderation_flagging_finance() + + await guard.async_moderation_hook(data=data, user_api_key_dict=user_api_key, call_type=call_type) + + assert guard.client.moderated_texts == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("call_type", "data"), + [ + ("acompletion", {"model": "gpt-4o", "messages": [{"role": "user", "content": "stocks"}]}), + ("anthropic_messages", {"model": "claude", "messages": [{"role": "user", "content": "stocks"}]}), + ("aresponses", {"model": "gpt-4o", "input": "stocks"}), + ( + "allm_passthrough_route", + {"model": "gigachat", "endpoint": "chat/completions", "messages": [{"role": "user", "content": "stocks"}]}, + ), + ( + "allm_passthrough_route", + { + "model": "gigachat", + "endpoint": "chat/completions#embeddings", + "messages": [{"role": "user", "content": "stocks"}], + }, + ), + ], +) +async def test_google_text_moderation_still_blocks_conversations( + user_api_key: UserAPIKeyAuth, call_type: CallTypesLiteral, data: dict[str, object] +): + guard: Final = _google_text_moderation_flagging_finance() + + with pytest.raises(HTTPException) as exc_info: + await guard.async_moderation_hook(data=data, user_api_key_dict=user_api_key, call_type=call_type) + + assert exc_info.value.status_code == 400 + assert guard.client.moderated_texts == ["stocks"] From 3a350cacc93a2edc3aee3d812c05868e056cada6 Mon Sep 17 00:00:00 2001 From: kailian zhou <101053705+ZHOUKAILIAN@users.noreply.github.com> Date: Thu, 29 Oct 2026 11:19:06 +0800 Subject: [PATCH 07/10] feat!: add StreamLake provider metadata (#41006) * feat: add StreamLake provider metadata * fix(streamlake): complete provider discovery and test request flows * test(streamlake): assert cost and tool-param invariants instead of pinning vendor prices * fix(streamlake): accept tools and reasoning on endpoint ids and match the current price page Endpoint ids have no cost-map row, so tools and reasoning_effort were rejected before reaching StreamLake. The provider now declares both capabilities, and named non-reasoning rows opt out explicitly. The rows now follow StreamLake's 2026-09-17 price page: drop the KAT-Coder and Qwen3 rows it no longer serves, add the served models that had no price, drop account-level rpm and tpm, and fill missing context windows. StreamLake is also registered for streamlake/* wildcards and its messages and responses support. * fix(streamlake): regenerate the endpoint matrix and type the new tests The backup endpoint matrix advertised chat only while the root map listed Messages and Responses, so /public/endpoints hid both. Regenerating with scripts/generate_provider_endpoints_support.py makes the two files match. Types the StreamLake tests and sets the Add Model test's API key with fireEvent.change, per the dashboard test guide. * fix(streamlake): drop the retired max_tokens field from the StreamLake cost-map rows * fix(streamlake): price the four served models StreamLake publishes rates for and add the Qwen3.8 cache prices * fix(streamlake): send token caps as max_completion_tokens and mark image models StreamLake ignores max_tokens on kimi-k2.6, kimi-k2.7-code, qwen3.6-27b and qwen3.6-35b-a3b, while every served model honors max_completion_tokens, so the JSON provider now maps max_tokens to max_completion_tokens instead of the reverse. Rows whose models read an image in a live probe set supports_vision, and the Qwen3.8 rows drop the explicit cache write price, which this route never uses since cache_control is dropped * fix(streamlake): keep cache_control marks, price explicit cache reads, and use API input limits StreamLake reads cache_control, so the provider keeps the marks through a preserve_cache_control flag. Qwen3.8 explicit cache hits now use their own read price, rows with a cache read price declare prompt caching, three models StreamLake serves without reasoning drop reasoning_effort, and max_input_tokens follows the limits the API reports. * fix(utils): warn when a custom_provider_map name matches a built-in provider --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/__init__.py | 5 + litellm/constants.py | 2 + litellm/llms/openai_like/dynamic_config.py | 11 +- litellm/llms/openai_like/json_loader.py | 10 +- litellm/llms/openai_like/providers.json | 16 + ...odel_prices_and_context_window_backup.json | 353 +++++++++++++++ .../provider_endpoints_support_backup.json | 26 ++ .../provider_create_fields.json | 28 ++ litellm/types/utils.py | 1 + litellm/utils.py | 14 + model_prices_and_context_window.json | 353 +++++++++++++++ provider_endpoints_support.json | 26 ++ .../openai_like/test_streamlake_provider.py | 402 ++++++++++++++++++ .../public_endpoints/test_public_endpoints.py | 19 + tests/unit/test_utils.py | 20 + .../panels/AddModelPanel.integration.test.tsx | 31 +- .../components/provider_info_helpers.test.tsx | 14 + .../src/components/provider_info_helpers.tsx | 3 + 18 files changed, 1329 insertions(+), 5 deletions(-) create mode 100644 tests/unit/llms/openai_like/test_streamlake_provider.py diff --git a/litellm/__init__.py b/litellm/__init__.py index c649a48211a..fe11884a484 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -681,6 +681,7 @@ moonshot_models: Set = set() publicai_models: Set = set() darkbloom_models: Set = set() openinfer_models: Final[Set[str]] = set() +streamlake_models: Final[Set[str]] = set() zerogpu_models: Final[Set[str]] = set() v0_models: Set = set() morph_models: Set = set() @@ -970,6 +971,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje darkbloom_models.add(key) elif value.get("litellm_provider") == "openinfer": openinfer_models.add(key) + elif value.get("litellm_provider") == "streamlake": + streamlake_models.add(key) elif value.get("litellm_provider") == "zerogpu": zerogpu_models.add(key) elif value.get("litellm_provider") == "v0": @@ -1145,6 +1148,7 @@ model_list = list( | publicai_models | darkbloom_models | openinfer_models + | streamlake_models | zerogpu_models | v0_models | morph_models @@ -1265,6 +1269,7 @@ def _build_models_by_provider() -> dict: "publicai": publicai_models, "darkbloom": darkbloom_models, "openinfer": openinfer_models, + "streamlake": streamlake_models, "zerogpu": zerogpu_models, "v0": v0_models, "morph": morph_models, diff --git a/litellm/constants.py b/litellm/constants.py index 25770d960bb..42ad624026c 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1064,6 +1064,7 @@ openai_compatible_endpoints: Final[list[str]] = [ "https://api.reka.ai/v1", "https://gigachat.devices.sberbank.ru/api/v1", "https://api.openinfer.ai/v1", + "https://vanchin.streamlake.ai/api/gateway/v1/endpoints", ] @@ -1145,6 +1146,7 @@ openai_compatible_providers: Final[list[str]] = [ "prism", "sail", "reka", + "streamlake", ] OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS: Final = frozenset({"openai"} | frozenset(openai_compatible_providers)) diff --git a/litellm/llms/openai_like/dynamic_config.py b/litellm/llms/openai_like/dynamic_config.py index 685561f55a5..d7aafe9f06e 100644 --- a/litellm/llms/openai_like/dynamic_config.py +++ b/litellm/llms/openai_like/dynamic_config.py @@ -55,6 +55,15 @@ def create_config_class(provider: SimpleProviderConfig): else: return super()._transform_messages(messages=messages, model=model, is_async=False) + def _should_preserve_cache_control_for_endpoint( + self, + custom_llm_provider: str | None, + api_base: str | None, + ) -> bool: + return provider.special_handling.get( + "preserve_cache_control", False + ) or super()._should_preserve_cache_control_for_endpoint(custom_llm_provider, api_base) + def _get_openai_compatible_provider_info( self, api_base: str | None, api_key: str | None ) -> tuple[str | None, str | None]: @@ -177,7 +186,7 @@ def create_config_class(provider: SimpleProviderConfig): return JSONProviderConfig -_responses_config_cache: Final[dict] = {} +_responses_config_cache: Final[dict[str, "type[OpenAILikeResponsesConfig]"]] = {} def create_responses_config_class(provider: SimpleProviderConfig) -> "type[OpenAILikeResponsesConfig]": diff --git a/litellm/llms/openai_like/json_loader.py b/litellm/llms/openai_like/json_loader.py index 23b63177137..6e99b9e7fec 100644 --- a/litellm/llms/openai_like/json_loader.py +++ b/litellm/llms/openai_like/json_loader.py @@ -3,6 +3,7 @@ JSON-based provider configuration loader for OpenAI-compatible providers. """ import json +from collections.abc import Mapping from pathlib import Path from typing import Final @@ -12,6 +13,7 @@ from litellm._logging import verbose_logger from litellm.types.utils import ProviderSpecificModelInfo _PROVIDER_INFO_ADAPTER: Final = TypeAdapter(ProviderSpecificModelInfo) +_SPECIAL_HANDLING_ADAPTER: Final = TypeAdapter(Mapping[str, bool]) class SimpleProviderConfig: @@ -20,12 +22,14 @@ class SimpleProviderConfig: def __init__(self, slug: str, data: dict): self.slug = slug self.base_url = data["base_url"] - self.api_key_env = data["api_key_env"] - self.api_base_env = data.get("api_base_env") + self.api_key_env: str = data["api_key_env"] + self.api_base_env: str | None = data.get("api_base_env") self.base_class = data.get("base_class", "openai_gpt") self.param_mappings = data.get("param_mappings", {}) self.constraints = data.get("constraints", {}) - self.special_handling = data.get("special_handling", {}) + self.special_handling: Mapping[str, bool] = _SPECIAL_HANDLING_ADAPTER.validate_python( + data.get("special_handling", {}) + ) self.supported_endpoints = data.get("supported_endpoints", []) self.provider_info: ProviderSpecificModelInfo = ( _PROVIDER_INFO_ADAPTER.validate_python(data["provider_info"]) if "provider_info" in data else {} diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index 2b71707239b..fe88ac03e04 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -269,5 +269,21 @@ "/v1/chat/completions", "/v1/responses" ] + }, + "streamlake": { + "base_url": "https://vanchin.streamlake.ai/api/gateway/v1/endpoints", + "api_key_env": "STREAMLAKE_API_KEY", + "api_base_env": "STREAMLAKE_API_BASE", + "param_mappings": { + "max_tokens": "max_completion_tokens" + }, + "provider_info": { + "supports_function_calling": true, + "supports_reasoning": true + }, + "special_handling": { + "preserve_cache_control": true + }, + "supported_endpoints": ["/v1/chat/completions"] } } diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3ab297425f8..dccf24fd63a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -61389,6 +61389,359 @@ "supports_tool_choice": true, "supports_vision": true }, + "streamlake/deepseek-v4-flash-0731": { + "cache_read_input_token_cost": 1.4e-08, + "input_cost_per_token": 4.4e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 1.32e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/DeepSeek-V4-Pro-0813": { + "cache_read_input_token_cost": 4.4e-08, + "input_cost_per_token": 1.32e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 1048566, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 3.96e-06, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/DeepSeek-V3": { + "cache_read_input_token_cost": 1.14e-07, + "input_cost_per_token": 2.86e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 56000, + "max_output_tokens": 8000, + "mode": "chat", + "output_cost_per_token": 1.143e-06, + "supports_function_calling": true, + "supports_reasoning": false, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/GLM-5.3": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_assistant_prefill": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/GLM-5.3-Flash": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_assistant_prefill": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/Kimi-K2-Instruct-0905": { + "cache_read_input_token_cost": 1.43e-07, + "input_cost_per_token": 5.71e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 224000, + "max_output_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 2.286e-06, + "supports_function_calling": true, + "supports_reasoning": false, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-r1": { + "cache_read_input_token_cost": 2.29e-07, + "input_cost_per_token": 5.71e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 98304, + "max_output_tokens": 16000, + "mode": "chat", + "output_cost_per_token": 2.286e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-v3.1": { + "cache_read_input_token_cost": 2.29e-07, + "input_cost_per_token": 5.71e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 98304, + "max_output_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 1.714e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-v4-flash": { + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 1.4e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 2.8e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-v4-pro": { + "cache_read_input_token_cost": 1.45e-07, + "input_cost_per_token": 1.74e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 3.48e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-v4.1-flash-1017": { + "cache_read_input_token_cost": 6e-09, + "input_cost_per_token": 3e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-v4.1-flash-260910": { + "cache_read_input_token_cost": 6e-09, + "input_cost_per_token": 3e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 1048568, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/glm-5": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 202752, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3.2e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/glm-5.1": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 202752, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/glm-5.2": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 1048566, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/kimi-k2.6": { + "cache_read_input_token_cost": 1.6e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 262144, + "max_output_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 4e-06, + "supports_function_calling": true, + "supports_vision": true, + "supports_reasoning": false, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/kimi-k2.7-code": { + "cache_read_input_token_cost": 1.9e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 262144, + "max_output_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 4e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/mimo-v2.6-flash": { + "cache_read_input_token_cost": 2.8e-09, + "input_cost_per_token": 1.4e-07, + "litellm_provider": "streamlake", + "mode": "chat", + "output_cost_per_token": 2.8e-07, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/mimo-v2.6-pro": { + "cache_read_input_token_cost": 3.6e-09, + "input_cost_per_token": 4.35e-07, + "litellm_provider": "streamlake", + "mode": "chat", + "output_cost_per_token": 8.7e-07, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/minimax-m2.5": { + "cache_creation_input_token_cost": 3.75e-07, + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 196608, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/minimax-m3": { + "cache_read_input_token_cost": 1.2e-07, + "cache_read_input_token_cost_above_512k_tokens": 2.4e-07, + "input_cost_per_token": 6e-07, + "input_cost_per_token_above_512k_tokens": 1.2e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 1000000, + "max_output_tokens": 512000, + "mode": "chat", + "output_cost_per_token": 2.4e-06, + "output_cost_per_token_above_512k_tokens": 4.8e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/qwen3.5-397b-a17b": { + "cache_read_input_token_cost": 1.2e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 258048, + "max_output_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/qwen3.6-27b": { + "input_cost_per_token": 6e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 229376, + "max_output_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "supports_function_calling": true, + "supports_vision": true, + "supports_reasoning": false, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/qwen3.6-35b-a3b": { + "input_cost_per_token": 3.75e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 229376, + "max_output_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.25e-06, + "supports_function_calling": true, + "supports_vision": true, + "supports_reasoning": false, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/qwen3.8-2.4t-a95b": { + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 983616, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 6e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "cache_creation_input_token_cost": 2.5e-06, + "explicit_cache_read_input_token_cost": 1.7e-07, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/qwen3.8-27b": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 5e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 983616, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "cache_creation_input_token_cost": 6.25e-07, + "explicit_cache_read_input_token_cost": 5e-08, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, "deepseek-v4-flash": { "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 6e-09, diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index 0a6c2bcb2a9..e40cd134759 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -3785,6 +3785,32 @@ "generateContent": true } }, + "streamlake": { + "display_name": "StreamLake (`streamlake`)", + "url": "https://docs.litellm.ai/docs/providers/openai_compatible", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, "tencent": { "display_name": "Tencent TokenHub (`tencent`)", "url": "https://docs.litellm.ai/docs/providers/tencent", diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index f2d88702470..2e72cf3d984 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -3519,6 +3519,34 @@ ], "default_model_placeholder": "inferenceapis/deepseek-ai/DeepSeek-V4-Flash" }, + { + "provider": "StreamLake", + "provider_display_name": "StreamLake", + "litellm_provider": "streamlake", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://vanchin.streamlake.ai/api/gateway/v1/endpoints", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "streamlake/" + }, { "provider": "Snowflake", "provider_display_name": "Snowflake", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 4855a16a1cf..70ca05904ef 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4593,6 +4593,7 @@ class LlmProviders(str, Enum): META = "meta" SAIL = "sail" REKA = "reka" + STREAMLAKE = "streamlake" LITELLM_AGENT = "litellm_agent" CURSOR = "cursor" BEDROCK_MANTLE = "bedrock_mantle" diff --git a/litellm/utils.py b/litellm/utils.py index f790966d167..63fa848d453 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -606,9 +606,23 @@ def custom_llm_setup(): litellm.provider_list.append(custom_llm["provider"]) if custom_llm["provider"] not in litellm._custom_providers: + _warn_if_built_in_provider_shadows_custom_handler(custom_llm["provider"]) litellm._custom_providers.append(custom_llm["provider"]) +def _warn_if_built_in_provider_shadows_custom_handler(provider: str) -> None: + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + if provider not in LlmProvidersSet and not JSONProviderRegistry.exists(provider): + return + verbose_logger.warning( + "custom_provider_map provider '%s' has the same name as a built-in LiteLLM provider, which handles " + "'%s/' models instead of your custom_handler. Rename the provider and its models to use your handler", + provider, + provider, + ) + + def add_custom_logger_callback_to_specific_event(callback: str, logging_event: Literal["success", "failure"]) -> None: """ Add a custom logger callback to the specific event diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3ab297425f8..dccf24fd63a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -61389,6 +61389,359 @@ "supports_tool_choice": true, "supports_vision": true }, + "streamlake/deepseek-v4-flash-0731": { + "cache_read_input_token_cost": 1.4e-08, + "input_cost_per_token": 4.4e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 1.32e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/DeepSeek-V4-Pro-0813": { + "cache_read_input_token_cost": 4.4e-08, + "input_cost_per_token": 1.32e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 1048566, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 3.96e-06, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/DeepSeek-V3": { + "cache_read_input_token_cost": 1.14e-07, + "input_cost_per_token": 2.86e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 56000, + "max_output_tokens": 8000, + "mode": "chat", + "output_cost_per_token": 1.143e-06, + "supports_function_calling": true, + "supports_reasoning": false, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/GLM-5.3": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_assistant_prefill": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/GLM-5.3-Flash": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_assistant_prefill": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/Kimi-K2-Instruct-0905": { + "cache_read_input_token_cost": 1.43e-07, + "input_cost_per_token": 5.71e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 224000, + "max_output_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 2.286e-06, + "supports_function_calling": true, + "supports_reasoning": false, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-r1": { + "cache_read_input_token_cost": 2.29e-07, + "input_cost_per_token": 5.71e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 98304, + "max_output_tokens": 16000, + "mode": "chat", + "output_cost_per_token": 2.286e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-v3.1": { + "cache_read_input_token_cost": 2.29e-07, + "input_cost_per_token": 5.71e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 98304, + "max_output_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 1.714e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-v4-flash": { + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 1.4e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 2.8e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-v4-pro": { + "cache_read_input_token_cost": 1.45e-07, + "input_cost_per_token": 1.74e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 3.48e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-v4.1-flash-1017": { + "cache_read_input_token_cost": 6e-09, + "input_cost_per_token": 3e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-v4.1-flash-260910": { + "cache_read_input_token_cost": 6e-09, + "input_cost_per_token": 3e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 1048568, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/glm-5": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 202752, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3.2e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/glm-5.1": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 202752, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/glm-5.2": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 1048566, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/kimi-k2.6": { + "cache_read_input_token_cost": 1.6e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 262144, + "max_output_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 4e-06, + "supports_function_calling": true, + "supports_vision": true, + "supports_reasoning": false, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/kimi-k2.7-code": { + "cache_read_input_token_cost": 1.9e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 262144, + "max_output_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 4e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/mimo-v2.6-flash": { + "cache_read_input_token_cost": 2.8e-09, + "input_cost_per_token": 1.4e-07, + "litellm_provider": "streamlake", + "mode": "chat", + "output_cost_per_token": 2.8e-07, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/mimo-v2.6-pro": { + "cache_read_input_token_cost": 3.6e-09, + "input_cost_per_token": 4.35e-07, + "litellm_provider": "streamlake", + "mode": "chat", + "output_cost_per_token": 8.7e-07, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/minimax-m2.5": { + "cache_creation_input_token_cost": 3.75e-07, + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 196608, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/minimax-m3": { + "cache_read_input_token_cost": 1.2e-07, + "cache_read_input_token_cost_above_512k_tokens": 2.4e-07, + "input_cost_per_token": 6e-07, + "input_cost_per_token_above_512k_tokens": 1.2e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 1000000, + "max_output_tokens": 512000, + "mode": "chat", + "output_cost_per_token": 2.4e-06, + "output_cost_per_token_above_512k_tokens": 4.8e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/qwen3.5-397b-a17b": { + "cache_read_input_token_cost": 1.2e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 258048, + "max_output_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/qwen3.6-27b": { + "input_cost_per_token": 6e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 229376, + "max_output_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "supports_function_calling": true, + "supports_vision": true, + "supports_reasoning": false, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/qwen3.6-35b-a3b": { + "input_cost_per_token": 3.75e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 229376, + "max_output_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.25e-06, + "supports_function_calling": true, + "supports_vision": true, + "supports_reasoning": false, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/qwen3.8-2.4t-a95b": { + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 983616, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 6e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "cache_creation_input_token_cost": 2.5e-06, + "explicit_cache_read_input_token_cost": 1.7e-07, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/qwen3.8-27b": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 5e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 983616, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "cache_creation_input_token_cost": 6.25e-07, + "explicit_cache_read_input_token_cost": 5e-08, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, "deepseek-v4-flash": { "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 6e-09, diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 0a6c2bcb2a9..e40cd134759 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -3785,6 +3785,32 @@ "generateContent": true } }, + "streamlake": { + "display_name": "StreamLake (`streamlake`)", + "url": "https://docs.litellm.ai/docs/providers/openai_compatible", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, "tencent": { "display_name": "Tencent TokenHub (`tencent`)", "url": "https://docs.litellm.ai/docs/providers/tencent", diff --git a/tests/unit/llms/openai_like/test_streamlake_provider.py b/tests/unit/llms/openai_like/test_streamlake_provider.py new file mode 100644 index 00000000000..f1a7b9edba5 --- /dev/null +++ b/tests/unit/llms/openai_like/test_streamlake_provider.py @@ -0,0 +1,402 @@ +import json +from pathlib import Path +from typing import Final + +import httpx +import pytest +import respx + +import litellm +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.llms.openai_like.dynamic_config import create_config_class +from litellm.llms.openai_like.json_loader import JSONProviderRegistry +from litellm.proxy.auth.model_checks import get_known_models_from_wildcard +from litellm.types.utils import ModelResponse, Usage + +STREAMLAKE_BASE_URL: Final = "https://vanchin.streamlake.ai/api/gateway/v1/endpoints" +STREAMLAKE_CHAT_URL: Final = f"{STREAMLAKE_BASE_URL}/chat/completions" + +REPO_ROOT: Final = Path(__file__).parents[4] + + +class TestStreamlakeProviderConfig: + def test_streamlake_provider_resolution(self) -> None: + model, provider, api_key, api_base = get_llm_provider( + model="streamlake/ep-test-endpoint", + custom_llm_provider=None, + api_base=None, + api_key="sk-test", + ) + + assert model == "ep-test-endpoint" + assert provider == "streamlake" + assert api_base == STREAMLAKE_BASE_URL + assert api_key == "sk-test" + + def test_streamlake_api_base_override(self) -> None: + model, provider, api_key, api_base = get_llm_provider( + model="streamlake/ep-test-endpoint", + custom_llm_provider=None, + api_base="https://custom.streamlake.example/v1", + api_key="sk-test", + ) + + assert model == "ep-test-endpoint" + assert provider == "streamlake" + assert api_base == "https://custom.streamlake.example/v1" + assert api_key == "sk-test" + + def test_streamlake_url_autodetection(self) -> None: + _, provider, _, api_base = get_llm_provider( + model="some-public-model", + custom_llm_provider=None, + api_base=STREAMLAKE_BASE_URL, + api_key=None, + ) + assert provider == "streamlake" + assert api_base == STREAMLAKE_BASE_URL + + def test_streamlake_api_key_resolved_from_env(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("STREAMLAKE_API_KEY", "sk-env-key") + + _, provider, api_key, _ = get_llm_provider( + model="streamlake/ep-test-endpoint", + custom_llm_provider=None, + api_base=None, + api_key=None, + ) + assert provider == "streamlake" + assert api_key == "sk-env-key" + + def test_streamlake_validate_environment_sets_bearer(self) -> None: + provider: Final = JSONProviderRegistry.get("streamlake") + assert provider is not None + config: Final = create_config_class(provider)() + + headers: Final = config.validate_environment( + headers={}, + model="ep-test-endpoint", + messages=[{"role": "user", "content": "hi"}], + optional_params={}, + litellm_params={}, + api_key="sk-resolved-key", + api_base=None, + ) + assert headers["Authorization"] == "Bearer sk-resolved-key" + + def test_streamlake_complete_url_appends_chat_completions(self) -> None: + provider: Final = JSONProviderRegistry.get("streamlake") + assert provider is not None + config: Final = create_config_class(provider)() + + url: Final = config.get_complete_url( + api_base=None, + api_key="sk-test", + model="ep-test-endpoint", + optional_params={}, + litellm_params={}, + ) + assert url == STREAMLAKE_CHAT_URL + + @pytest.mark.parametrize("cap_param", ("max_tokens", "max_completion_tokens")) + def test_streamlake_token_cap_sent_as_max_completion_tokens(self, cap_param: str) -> None: + provider: Final = JSONProviderRegistry.get("streamlake") + assert provider is not None + config: Final = create_config_class(provider)() + + optional_params: Final = config.map_openai_params( + non_default_params={cap_param: 1024}, + optional_params={}, + model="ep-test-endpoint", + drop_params=False, + ) + assert optional_params == {"max_completion_tokens": 1024} + + def test_streamlake_chat_params_pass_through(self) -> None: + provider: Final = JSONProviderRegistry.get("streamlake") + assert provider is not None + config: Final = create_config_class(provider)() + + optional_params: Final = config.map_openai_params( + non_default_params={"temperature": 0.2, "top_p": 0.9, "stream": True}, + optional_params={}, + model="ep-test-endpoint", + drop_params=False, + ) + assert optional_params == {"temperature": 0.2, "top_p": 0.9, "stream": True} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "is_async, api_base", + ( + (False, None), + (False, "https://custom.streamlake.example/v1"), + (False, "https://custom.streamlake.example/v1/"), + (True, None), + (True, "https://custom.streamlake.example/v1"), + (True, "https://custom.streamlake.example/v1/"), + ), + ids=( + "sync-default", + "sync-override", + "sync-override-trailing-slash", + "async-default", + "async-override", + "async-override-trailing-slash", + ), +) +async def test_streamlake_completion_uses_openai_compatible_http( + is_async: bool, api_base: str | None, monkeypatch: pytest.MonkeyPatch +) -> None: + response_body: Final[dict[str, object]] = { + "id": "chatcmpl-streamlake-test", + "object": "chat.completion", + "created": 1710000000, + "model": "ep-test-endpoint", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "StreamLake response"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}, + } + expected_url: Final[str] = f"{(api_base or STREAMLAKE_BASE_URL).rstrip('/')}/chat/completions" + expected_request_body: Final[dict[str, object]] = { + "messages": [{"role": "user", "content": "hello"}], + "model": "ep-test-endpoint", + "max_completion_tokens": 512, + } + + def handler(request: httpx.Request) -> httpx.Response: + assert request.method == "POST" + assert str(request.url) == expected_url + assert request.headers["authorization"] == "Bearer sk-streamlake-test" + assert request.headers["content-type"] == "application/json" + assert json.loads(request.content) == expected_request_body + return httpx.Response(200, json=response_body, request=request) + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post(expected_url).mock(side_effect=handler) + if is_async: + response: Final = await litellm.acompletion( + model="streamlake/ep-test-endpoint", + messages=[{"role": "user", "content": "hello"}], + api_key="sk-streamlake-test", + api_base=api_base, + max_tokens=512, + ) + else: + response: Final = litellm.completion( + model="streamlake/ep-test-endpoint", + messages=[{"role": "user", "content": "hello"}], + api_key="sk-streamlake-test", + api_base=api_base, + max_tokens=512, + ) + + assert route.call_count == 1 + assert len(response.choices) == 1 + assert response.choices[0].message.role == "assistant" + assert response.choices[0].message.content == "StreamLake response" + assert response.usage is not None + assert response.usage.prompt_tokens == 3 + assert response.usage.completion_tokens == 2 + assert response.usage.total_tokens == 5 + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_endpoint_id_call_forwards_tools_and_reasoning_effort_to_streamlake(monkeypatch: pytest.MonkeyPatch) -> None: + tools: Final = [ + { + "type": "function", + "function": { + "name": "get_weather", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, + }, + } + ] + response_body: Final[dict[str, object]] = { + "id": "chatcmpl-streamlake-tools", + "object": "chat.completion", + "created": 1710000000, + "model": "glm-5.3", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}, + } + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post(STREAMLAKE_CHAT_URL).mock(return_value=httpx.Response(200, json=response_body)) + litellm.completion( + model="streamlake/ep-test-endpoint", + messages=[{"role": "user", "content": "weather in Paris?"}], + api_key="sk-streamlake-test", + tools=tools, + tool_choice="auto", + reasoning_effort="low", + ) + + sent: Final = json.loads(route.calls.last.request.content) + assert sent["tools"] == tools + assert sent["tool_choice"] == "auto" + assert sent["reasoning_effort"] == "low" + + +def _streamlake_keys(cost_map: dict[str, object]) -> tuple[str, ...]: + return tuple(key for key in cost_map if key.startswith("streamlake/")) + + +def test_root_and_backup_cost_maps_carry_the_same_streamlake_rows() -> None: + root: Final = json.loads((REPO_ROOT / "model_prices_and_context_window.json").read_text()) + backup: Final = json.loads((REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json").read_text()) + + assert _streamlake_keys(root) + assert {key: root[key] for key in _streamlake_keys(root)} == {key: backup[key] for key in _streamlake_keys(backup)} + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_endpoint_id_call_is_priced_from_the_lowercase_model_streamlake_returns() -> None: + keys: Final = _streamlake_keys(litellm.model_cost) + assert keys + + for key in keys: + info: Final = litellm.get_model_info(model=key) + response: Final = ModelResponse( + model=key.split("/", 1)[1].lower(), + usage=Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500), + ) + cost: Final = litellm.completion_cost( + completion_response=response, + model="streamlake/ep-test-endpoint", + custom_llm_provider="streamlake", + ) + expected: Final = 1000 * info["input_cost_per_token"] + 500 * info["output_cost_per_token"] + assert cost == pytest.approx(expected), key + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_named_model_params_follow_the_row_and_fall_back_to_the_provider_default() -> None: + provider: Final = JSONProviderRegistry.get("streamlake") + assert provider is not None + config: Final = create_config_class(provider)() + keys: Final = _streamlake_keys(litellm.model_cost) + assert keys + + for key in keys: + row: Final = litellm.model_cost[key] + supported: Final = config.get_supported_openai_params(model=key.split("/", 1)[1].lower()) + assert ("tools" in supported) is (row.get("supports_function_calling") is not False), key + assert ("reasoning_effort" in supported) is (row.get("supports_reasoning") is not False), key + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_streamlake_wildcard_expands_to_every_streamlake_row() -> None: + expanded: Final = get_known_models_from_wildcard("streamlake/*") + assert set(expanded) == set(_streamlake_keys(litellm.model_cost)) + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_cache_control_marks_reach_streamlake(monkeypatch: pytest.MonkeyPatch) -> None: + system_block: Final = {"type": "text", "text": "You are terse.", "cache_control": {"type": "ephemeral"}} + tool: Final = { + "type": "function", + "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}}, + "cache_control": {"type": "ephemeral"}, + } + response_body: Final[dict[str, object]] = { + "id": "chatcmpl-streamlake-cache", + "object": "chat.completion", + "created": 1710000000, + "model": "qwen3.8-27b", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}, + } + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post(STREAMLAKE_CHAT_URL).mock(return_value=httpx.Response(200, json=response_body)) + litellm.completion( + model="streamlake/qwen3.8-27b", + messages=[{"role": "system", "content": [system_block]}, {"role": "user", "content": "hi"}], + api_key="sk-streamlake-test", + tools=[tool], + ) + + sent: Final = json.loads(route.calls.last.request.content) + assert sent["messages"][0]["content"] == [system_block] + assert sent["tools"][0]["cache_control"] == {"type": "ephemeral"} + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize("model", ("kimi-k2.6", "qwen3.6-27b", "qwen3.6-35b-a3b")) +def test_reasoning_effort_is_dropped_for_models_streamlake_lists_without_reasoning( + model: str, monkeypatch: pytest.MonkeyPatch +) -> None: + # StreamLake's Models Overview (https://www.streamlake.ai/document/DOC/mh1gbfvrdn6hpbzxixv, read 2026-10-28) + # gives these models no Reasoning tag, and kimi-k2.6 answers 400 to reasoning_effort under an 8193 token cap + response_body: Final[dict[str, object]] = { + "id": "chatcmpl-streamlake-reasoning", + "object": "chat.completion", + "created": 1710000000, + "model": model, + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}, + } + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post(STREAMLAKE_CHAT_URL).mock(return_value=httpx.Response(200, json=response_body)) + litellm.completion( + model=f"streamlake/{model}", + messages=[{"role": "user", "content": "hi"}], + api_key="sk-streamlake-test", + reasoning_effort="low", + drop_params=True, + ) + + assert "reasoning_effort" not in json.loads(route.calls.last.request.content) + with pytest.raises(litellm.UnsupportedParamsError): + litellm.completion( + model=f"streamlake/{model}", + messages=[{"role": "user", "content": "hi"}], + api_key="sk-streamlake-test", + reasoning_effort="low", + ) + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_streamlake_rows_with_a_cache_read_price_declare_prompt_caching() -> None: + cached_rows: Final = tuple( + key for key in _streamlake_keys(litellm.model_cost) if "cache_read_input_token_cost" in litellm.model_cost[key] + ) + assert cached_rows + assert [key for key in cached_rows if litellm.model_cost[key].get("supports_prompt_caching") is not True] == [] + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize("model", ("qwen3.8-27b", "qwen3.8-2.4t-a95b")) +@pytest.mark.parametrize("cache_type", ("ephemeral", None)) +def test_qwen3_8_explicit_cache_hits_and_prefix_hits_use_their_own_read_price( + model: str, cache_type: str | None +) -> None: + # StreamLake's price page (https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce, read 2026-10-28) lists + # separate PrefixCache and Cache Read prices for Qwen3.8, and its usage tags only explicit hits with cache_type + rate_key: Final = "explicit_cache_read_input_token_cost" if cache_type else "cache_read_input_token_cost" + row: Final = litellm.model_cost[f"streamlake/{model}"] + details: Final = {"cached_tokens": 4000} | ({"cache_type": cache_type} if cache_type else {}) + response: Final = ModelResponse( + model=model, + usage=Usage(prompt_tokens=5000, completion_tokens=100, total_tokens=5100, prompt_tokens_details=details), + ) + + cost: Final = litellm.completion_cost( + completion_response=response, model=f"streamlake/{model}", custom_llm_provider="streamlake" + ) + + expected: Final = 4000 * row[rate_key] + 1000 * row["input_cost_per_token"] + 100 * row["output_cost_per_token"] + assert cost == pytest.approx(expected) diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index d55ce25d878..013d3e59125 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -1141,6 +1141,25 @@ def test_get_supported_endpoints_chat_completions_present(reset_endpoints_cache) assert len(chat["providers"]) > 0 +def test_get_supported_endpoints_lists_streamlake_on_chat_messages_and_responses( + reset_endpoints_cache: None, +) -> None: + app: Final[FastAPI] = FastAPI() + app.include_router(router) + client: Final[TestClient] = TestClient(app) + with client: + response: Final = client.get("/public/endpoints") + assert response.status_code == 200, response.text + + streamlake_endpoints: Final = { + endpoint["key"] + for endpoint in response.json()["endpoints"] + if any(provider["slug"] == "streamlake" for provider in endpoint["providers"]) + } + assert {"chat_completions", "messages", "responses"} <= streamlake_endpoints + assert "embeddings" not in streamlake_endpoints + + def test_get_supported_endpoints_display_names_have_no_slug_suffix( reset_endpoints_cache, ): diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 146e2df6413..665a540e462 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -10882,3 +10882,23 @@ def test_get_optional_params_forwards_a_tool_config_cache_point_to_a_downstream_ ) assert optional_params["extra_body"] == {"kept": True, "cache_control_injection_points": [point]} + + +@pytest.mark.parametrize( + ("provider", "expected_warnings"), + (("streamlake", 1), ("veniceai", 1), ("my-own-llm", 0)), +) +def test_custom_provider_named_like_a_built_in_warns_once( + provider: str, expected_warnings: int, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + monkeypatch.setattr(litellm, "custom_provider_map", [{"provider": provider, "custom_handler": litellm.CustomLLM()}]) + monkeypatch.setattr(litellm, "provider_list", list(litellm.provider_list)) + monkeypatch.setattr(litellm, "_custom_providers", []) + + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + litellm.utils.custom_llm_setup() + litellm.utils.custom_llm_setup() + + warnings: Final = [r for r in caplog.records if f"provider '{provider}'" in r.getMessage()] + assert len(warnings) == expected_warnings + assert provider in litellm._custom_providers diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx index 43b9a5541db..c9b3defcd28 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx @@ -1,5 +1,5 @@ import { useSyncExternalStore } from "react"; -import { act, renderWithProviders, screen, waitFor, within } from "../../../../../tests/test-utils"; +import { act, fireEvent, renderWithProviders, screen, waitFor, within } from "../../../../../tests/test-utils"; import type { OnUrlUpdateFunction } from "nuqs/adapters/testing"; import userEvent, { PointerEventsCheckLevel } from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi } from "vitest"; @@ -80,6 +80,13 @@ vi.mock("@/app/(dashboard)/hooks/providers/useProviderFields", () => ({ default_model_placeholder: "jev-latest", credential_fields: [{ key: "api_key", label: "API Key", field_type: "password", required: false }], }, + { + provider: "StreamLake", + provider_display_name: "StreamLake", + litellm_provider: "streamlake", + default_model_placeholder: "streamlake/", + credential_fields: [{ key: "api_key", label: "API Key", field_type: "password", required: false }], + }, ], isLoading: false, error: null, @@ -278,6 +285,28 @@ describe("AddModelPanel submit payload contract", () => { }); }); + it("creates a StreamLake model from a public model selection and API key", async () => { + mockUseModelCostMap.mockReturnValue({ data: { "streamlake/GLM-5.3": { litellm_provider: "streamlake" } } }); + const { user, submit } = await setup(); + + await user.click(screen.getByRole("combobox", { name: /provider/i })); + await user.click(await screen.findByText("StreamLake")); + await user.click(screen.getByRole("combobox", { name: "Select models" })); + await user.click(await screen.findByText("streamlake/GLM-5.3")); + fireEvent.change(screen.getByLabelText("API Key"), { target: { value: "sk-streamlake-test" } }); + await submit(); + + expect(lastCreatedModel()).toStrictEqual({ + model_name: "streamlake/GLM-5.3", + litellm_params: { + model: "streamlake/GLM-5.3", + custom_llm_provider: "streamlake", + api_key: "sk-streamlake-test", + }, + model_info: { ...baseModelInfo }, + }); + }); + it("registers four more keys as undefined once Advanced Settings opens", async () => { const { openAdvanced, fillRequired, submit } = await setup(); await fillRequired(); diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx index 83e80911cff..53e54e2b032 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx @@ -222,6 +222,7 @@ describe("provider_info_helpers", () => { Providers.PG_VECTOR, Providers.PREDIBASE, Providers.Sail, + Providers.StreamLake, Providers.WANDB, Providers.ZAI, Providers.ZeroGPU, @@ -462,6 +463,19 @@ describe("provider_info_helpers", () => { expect(result).toEqual(["gpt-3.5-turbo", "gpt-4"]); }); + it("should return StreamLake models and exclude models from other providers", () => { + const modelMap = { + "streamlake/GLM-5.3": { litellm_provider: "streamlake" }, + "streamlake/DeepSeek-V3": { litellm_provider: "streamlake" }, + "gpt-4o": { litellm_provider: "openai" }, + }; + const result = getProviderModels(Providers.StreamLake, modelMap); + + expect(provider_map.StreamLake).toBe("streamlake"); + expect(result).toEqual(["streamlake/GLM-5.3", "streamlake/DeepSeek-V3"]); + expect(result).not.toContain("gpt-4o"); + }); + it("should return models whose litellm_provider is a prefix-anchored variant of the provider", () => { const modelMap = { "anthropic-text-model": { litellm_provider: "anthropic_text" }, diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index bfb578b218a..1e1aafde1b5 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -177,6 +177,7 @@ export enum Providers { Snowflake = "Snowflake", Soniox = "Soniox", StrandsDecider = "Strands Decider", + StreamLake = "StreamLake", TEXT_COMPLETION_CODESTRAL = "Text-Completion-Codestral", Tencent = "Tencent", TogetherAI = "TogetherAI", @@ -306,6 +307,7 @@ export const provider_map: Record = { Snowflake: "snowflake", Soniox: "soniox", StrandsDecider: "strands_decider", + StreamLake: "streamlake", TEXT_COMPLETION_CODESTRAL: "text-completion-codestral", Tencent: "tencent", TogetherAI: "together_ai", @@ -494,6 +496,7 @@ const providerPlaceholderMap: Partial> = { [Providers.Tencent]: "tencent/deepseek-v4-pro", [Providers.TypeSafe]: "typesafe/jev-latest", [Providers.Nace]: "nace/drex-latest", + [Providers.StreamLake]: "streamlake/", [Providers.Vertex_AI]: "gemini-pro", [Providers.VolcEngine]: "volcengine/", [Providers.Voyage]: "voyage/", From f46adf80e45b21d4444a300db4ee1db93388552d Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:19:58 -0700 Subject: [PATCH 08/10] fix(proxy): let typesafe/* allowlists cover /typesafe Jev calls and list Jev in the model picker (#47697) * fix(proxy): list TypeSafe Jev models in the admin model picker * fix(proxy): let typesafe/* allowlists cover bare /typesafe Jev ids Adds a typesafe-jev-ids routing rule to the cost map so a bare jev-* id resolves to TypeSafe, which lets typesafe/* on a key, team or project allowlist match /typesafe calls. The picker gate now only checks that a TypeSafe key is set, so an unresolvable OIDC reference cannot fail the listing, and the /v1/models route tests mock deployments with litellm_params like real ones. * fix(typesafe): match only published Jev ids and keep picker loads off the secret manager Narrows the typesafe-jev-ids rule to jev-latest, jev-preview and versioned ids so a model group such as jev-internal-router never resolves to TypeSafe, checks TYPESAFE_API_KEY in the environment instead of through the secret manager on each picker load, and moves the picker tests into the mapped test file. * test(typesafe): type the picker test fixture and tests * fix(typesafe): match only published Jev versions and cover non-version suffixes in tests * fix(typesafe): match only priced Jev ids and offer TypeSafe in the no-team key picker * fix(proxy): offer TypeSafe allowlist names only to the Admin UI allowlist pickers The dashboard sends include_model_access_groups on every /models call, so gate the TypeSafe names on wildcard routes too and skip access-group-only listings. Also say the typesafe-jev-ids rule matches case-insensitively. * fix(ui): offer only the TypeSafe names the cost-map rule routes, and keep /v1/models listing as on main A typesafe/* router deployment registers typesafe/* in the cost map, so the allowlist pickers offered a bare *, which allows every model. The pickers now list only priced names the rule sends to TypeSafe. The non-expand list also stops de-duplicating every entry and only skips TypeSafe names already listed. --- ...odel_prices_and_context_window_backup.json | 8 + .../llm_passthrough_endpoints.py | 15 ++ .../passthrough_endpoint_router.py | 9 + litellm/proxy/proxy_server.py | 18 +- model_prices_and_context_window.json | 8 + .../test_get_llm_provider_logic.py | 42 ++++ tests/unit/proxy/auth/test_auth_checks.py | 47 +++- tests/unit/proxy/proxy_server/conftest.py | 27 ++- .../proxy/proxy_server/test_routes_models.py | 216 +++++++++++++++++- 9 files changed, 385 insertions(+), 5 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index dccf24fd63a..23c6ccf9c0a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -63057,6 +63057,14 @@ "litellm_provider": "anthropic" } }, + { + "name": "typesafe-jev-ids", + "pattern": "^jev-(?:latest|preview|1\\.13\\.0|1\\.14\\.0)$", + "description": "A bare Jev id that has a typesafe/ price entry (jev-latest, jev-preview, jev-1.13.0, jev-1.14.0), the name the /typesafe pass-through forwards to TypeSafe. Listed by name, matched case-insensitively like every rule here, so foo/jev-latest or a model group named like a Jev id TypeSafe does not serve, such as jev-1.0 or jev-internal-router, never resolves to typesafe. A new Jev version goes here together with its price entry. Provider inference feeds the proxy's provider/* wildcard access checks, so this rule is what lets typesafe/* on a key, team or project allowlist cover a jev-latest request.", + "model_info": { + "litellm_provider": "typesafe" + } + }, { "name": "claude-family-baseline", "pattern": "claude-[a-z]+-\\d+(?:[-.]\\d+)?(?:-\\d{8})?", diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index e478096f7ad..972ec18d487 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -50,6 +50,7 @@ from litellm.constants import ( from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException from litellm.integrations.custom_guardrail import guardrail_request_data_with_streaming, is_guardrail_intervention from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix +from litellm.litellm_core_utils.fallback_generalizations import match_routing_generalization from litellm.llms.anthropic.common_utils import AnthropicModelInfo, merge_anthropic_beta_headers from litellm.llms.azure.passthrough.transformation import ( foreign_azure_deployment, @@ -698,6 +699,20 @@ async def typesafe_proxy_route( return await endpoint_func(request, fastapi_response, user_api_key_dict) +_COST_MAP_ENTRIES: Final = TypeAdapter(Mapping[str, object]) + + +def typesafe_allowlist_model_names() -> tuple[str, ...]: + if not passthrough_endpoint_router.has_credentials(custom_llm_provider="typesafe"): + return () + cost_map: Final = _COST_MAP_ENTRIES.validate_python(litellm.model_cost) # pyright: ignore[reportUnknownMemberType] # legacy registry is validated at this boundary + priced_names: Final = (name.removeprefix("typesafe/") for name in cost_map if name.startswith("typesafe/")) + jev_models_typesafe_serves: Final = sorted( + name for name in priced_names if match_routing_generalization(name) == "typesafe" + ) + return ("typesafe/*", *jev_models_typesafe_serves) + + @router.post( "/laya/v1/systemone", tags=["Laya Pass-through", "pass-through"], diff --git a/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py b/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py index 270a1cc2f3b..1d0b3d00434 100644 --- a/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py +++ b/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py @@ -1,4 +1,5 @@ import json +import os from collections.abc import Callable from typing import TYPE_CHECKING, Final @@ -88,6 +89,14 @@ class PassthroughEndpointRouter: ) return _resolve_oidc_reference(get_secret_str(_env_variable_name)) + def has_credentials(self, custom_llm_provider: str) -> bool: + if self._get_deployment_api_key(custom_llm_provider=custom_llm_provider, region_name=None) is not None: + return True + env_variable_name: Final = self._get_default_env_variable_name_passthrough_endpoint( + custom_llm_provider=custom_llm_provider, + ) + return bool(os.environ.get(env_variable_name)) + def _get_deployment_api_key( self, custom_llm_provider: str, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c04e79d0934..d6ec62b25c2 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -12390,6 +12390,12 @@ async def model_list( hidden_names: Final = blocked_names | unhealthy_names + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import typesafe_allowlist_model_names + + asks_for_allowlist_names: Final = ( + include_model_access_groups is True and return_wildcard_routes is True and only_model_access_groups is not True + ) + # If scope=expand and user has admin privileges, return all proxy models if should_expand_scope: # Get all proxy models as if user is a proxy admin @@ -12410,7 +12416,10 @@ async def model_list( all_models = get_complete_model_list( key_models=[], team_models=[], - proxy_model_list=proxy_model_list, + proxy_model_list=[ + *proxy_model_list, + *(typesafe_allowlist_model_names() if asks_for_allowlist_names else ()), + ], user_model=None, infer_model_from_keys=False, return_wildcard_routes=include_wildcard_routes, @@ -12471,7 +12480,7 @@ async def model_list( # Otherwise, use the normal behavior (current implementation) # Get available models for the user - all_models = await get_available_models_for_user( + available_models: Final = await get_available_models_for_user( user_api_key_dict=user_api_key_dict, llm_router=llm_router, general_settings=general_settings, @@ -12484,6 +12493,11 @@ async def model_list( return_wildcard_routes=include_wildcard_routes, user_api_key_cache=user_api_key_cache, ) + offers_typesafe_allowlist_names: Final = ( + asks_for_allowlist_names and team_id is None and user_api_key_has_admin_view(user_api_key_dict) + ) + typesafe_names_to_offer: Final = typesafe_allowlist_model_names() if offers_typesafe_allowlist_names else () + all_models = [*available_models, *(name for name in typesafe_names_to_offer if name not in available_models)] undiscoverable_names: Final = undiscoverable_model_names( all_models, llm_router, user_api_key_dict, team_id or user_api_key_dict.team_id diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index dccf24fd63a..23c6ccf9c0a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -63057,6 +63057,14 @@ "litellm_provider": "anthropic" } }, + { + "name": "typesafe-jev-ids", + "pattern": "^jev-(?:latest|preview|1\\.13\\.0|1\\.14\\.0)$", + "description": "A bare Jev id that has a typesafe/ price entry (jev-latest, jev-preview, jev-1.13.0, jev-1.14.0), the name the /typesafe pass-through forwards to TypeSafe. Listed by name, matched case-insensitively like every rule here, so foo/jev-latest or a model group named like a Jev id TypeSafe does not serve, such as jev-1.0 or jev-internal-router, never resolves to typesafe. A new Jev version goes here together with its price entry. Provider inference feeds the proxy's provider/* wildcard access checks, so this rule is what lets typesafe/* on a key, team or project allowlist cover a jev-latest request.", + "model_info": { + "litellm_provider": "typesafe" + } + }, { "name": "claude-family-baseline", "pattern": "claude-[a-z]+-\\d+(?:[-.]\\d+)?(?:-\\d{8})?", diff --git a/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py b/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py index e9e3f7b8174..25de5ef60d9 100644 --- a/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py +++ b/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py @@ -714,6 +714,48 @@ class TestClaudeModelPatternMatching: litellm.get_llm_provider(model="claude-opus-4-9") +@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") +class TestTypesafeJevPatternMatching: + def test_every_priced_jev_id_routes_to_typesafe( + self, shipped_generalizations: None + ) -> None: + from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap + + priced_ids: Final = tuple( + key.removeprefix("typesafe/") + for key in GetModelCostMap.load_local_model_cost_map() + if key.startswith("typesafe/") + ) + assert "jev-latest" in priced_ids + assert { + model: litellm.get_llm_provider(model=model)[1] for model in priced_ids + } == dict.fromkeys(priced_ids, "typesafe") + + @pytest.mark.parametrize( + "model", + [ + "foo/jev-latest", + "openjev-latest", + "jev-", + "jev-latest model", + "jev-internal-router", + "jev-2.0-internal", + "jev-1.0-acme.router", + "jev-1.0", + "jev-2.0", + "jev-1.13", + ], + ) + def test_namespaced_or_partial_jev_name_does_not_match_rule( + self, model: str, shipped_generalizations: None + ) -> None: + from litellm.litellm_core_utils.fallback_generalizations import ( + match_routing_generalization, + ) + + assert match_routing_generalization(model) is None + + @pytest.mark.parametrize("prefix", ["openai", "anthropic", "custom_openai", "openai_like"]) def test_opencode_affinity_resolution_preserves_existing_transport(prefix: str) -> None: api_base: Final = "https://opencode.ai/zen/go/v1" diff --git a/tests/unit/proxy/auth/test_auth_checks.py b/tests/unit/proxy/auth/test_auth_checks.py index e1afa81d372..e783f01769e 100644 --- a/tests/unit/proxy/auth/test_auth_checks.py +++ b/tests/unit/proxy/auth/test_auth_checks.py @@ -12,7 +12,7 @@ load_dotenv() from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence from copy import deepcopy from types import SimpleNamespace -from typing import Final +from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch import pytest, litellm @@ -721,6 +721,51 @@ async def test_can_key_call_model_wildcard_access(key_models, model, expect_to_w ) +@pytest.fixture +def shipped_routing_rules() -> Iterator[None]: + from litellm.litellm_core_utils.fallback_generalizations import ( + get_fallback_generalization_rules, + set_fallback_generalizations, + ) + from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap + + previous_rules: Final = list(get_fallback_generalization_rules()) + set_fallback_generalizations(GetModelCostMap.load_local_model_cost_map()["fallback_generalizations"]["rules"]) + yield + set_fallback_generalizations(previous_rules) + + +@pytest.mark.parametrize("object_type", ["team", "project"]) +@pytest.mark.parametrize( + "allowed_models, model, expect_to_work", + [ + (["typesafe/*"], "jev-latest", True), + (["typesafe/*"], "jev-1.14.0", True), + (["jev-latest"], "jev-latest", True), + (["openai/*"], "jev-latest", False), + (["typesafe/*"], "foo/jev-latest", False), + (["typesafe/*"], "jev-internal-router", False), + (["typesafe/*"], "jev-2.0-internal", False), + (["typesafe/*"], "jev-1.0", False), + ], +) +def test_typesafe_provider_wildcard_covers_bare_jev_model( + shipped_routing_rules: None, + object_type: Literal["team", "project"], + allowed_models: list[str], + model: str, + expect_to_work: bool, +) -> None: + from litellm.proxy._types import ProxyException + from litellm.proxy.auth.auth_checks import can_object_call_model + + if expect_to_work: + assert can_object_call_model(model=model, llm_router=None, models=allowed_models, object_type=object_type) + return + with pytest.raises(ProxyException): + can_object_call_model(model=model, llm_router=None, models=allowed_models, object_type=object_type) + + @pytest.mark.parametrize( "key_models, model, expect_to_work", [ diff --git a/tests/unit/proxy/proxy_server/conftest.py b/tests/unit/proxy/proxy_server/conftest.py index 9baf3206fc4..3f48a3b8558 100644 --- a/tests/unit/proxy/proxy_server/conftest.py +++ b/tests/unit/proxy/proxy_server/conftest.py @@ -11,11 +11,14 @@ import contextlib import os import sys from pathlib import Path -from typing import Any, AsyncIterator, Callable, Dict, Iterator, List, Optional +from typing import TYPE_CHECKING, Any, AsyncIterator, Callable, Dict, Final, Iterator, List, Optional from unittest.mock import AsyncMock, MagicMock import pytest +if TYPE_CHECKING: + from litellm import Router + # Repo root, anchored to this file (not CWD) so the path is correct no # matter where pytest is invoked from. With the project installed via # uv this is defensive — `litellm` already resolves through site-packages @@ -536,3 +539,25 @@ def reset_login_throttle(monkeypatch): _drop_throttle_keys() yield _drop_throttle_keys _drop_throttle_keys() + + +@pytest.fixture +def openai_only_router(monkeypatch: pytest.MonkeyPatch) -> Router: + from litellm import Router + from litellm.proxy import proxy_server + + router: Final = Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-fake"}, + "model_info": {"id": "gpt-4-id"}, + } + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "llm_model_list", router.model_list) + monkeypatch.setattr(proxy_server, "prisma_client", None) + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(proxy_server, "user_model", None) + return router diff --git a/tests/unit/proxy/proxy_server/test_routes_models.py b/tests/unit/proxy/proxy_server/test_routes_models.py index 7be30c61f0c..36f2a8d5acd 100644 --- a/tests/unit/proxy/proxy_server/test_routes_models.py +++ b/tests/unit/proxy/proxy_server/test_routes_models.py @@ -9,15 +9,20 @@ Pins (PR2): from __future__ import annotations +from typing import Final from unittest.mock import MagicMock import pytest import litellm +from litellm import Router +from litellm.integrations.custom_secret_manager import CustomSecretManager from litellm.proxy import proxy_server +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy import utils as proxy_utils from litellm.proxy.utils import create_model_info_response from litellm.types.router import DeploymentModelListingInfo +from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem from .conftest import normalize # type: ignore[import-not-found] @@ -237,6 +242,7 @@ def test_anthropic_display_name_resolved_via_internal_team_key( return_value=[ { "model_name": internal_name, + "litellm_params": {"model": "openai/gpt-4"}, "model_info": { "team_id": "team-1", "team_public_model_name": "gpt-4-team", @@ -320,6 +326,7 @@ def test_anthropic_format_returns_public_team_model_name( return_value=[ { "model_name": internal_name, + "litellm_params": {"model": "openai/gpt-4"}, "model_info": { "team_id": "team-1", "team_public_model_name": "gpt-4-team", @@ -494,7 +501,11 @@ def test_codex_service_tiers_resolved_via_internal_team_key(client, auth_as, pat patched_models.get_model_list = MagicMock( return_value=[ - {"model_name": internal_name, "model_info": {"team_id": "team-1", "team_public_model_name": "gpt-4-team"}} + { + "model_name": internal_name, + "litellm_params": {"model": "openai/gpt-4"}, + "model_info": {"team_id": "team-1", "team_public_model_name": "gpt-4-team"}, + } ] ) patched_models.get_model_names = MagicMock(return_value=[internal_name]) @@ -514,3 +525,206 @@ def test_codex_service_tiers_resolved_via_internal_team_key(client, auth_as, pat assert response.status_code == 200 (entry,) = response.json()["models"] assert (entry["slug"], [tier["id"] for tier in entry["service_tiers"]]) == ("gpt-4-team", ["ultrafast"]) + + +def _priced_jev_models() -> frozenset[str]: + return frozenset(name.removeprefix("typesafe/") for name in litellm.model_cost if name.startswith("typesafe/")) + + +async def _admin_picker_models() -> set[str]: + admin: Final = UserAPIKeyAuth(api_key="sk-test", user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN) + response: Final = await proxy_server.model_list( + user_api_key_dict=admin, return_wildcard_routes=True, include_model_access_groups=True, scope="expand" + ) + return {model["id"] for model in response["data"]} + + +@pytest.mark.asyncio +async def test_scope_expand_offers_typesafe_models_when_typesafe_is_configured( + openai_only_router: Router, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake") + jev_models: Final = _priced_jev_models() + + picker_models: Final = await _admin_picker_models() + + assert jev_models + assert {"gpt-4", "typesafe/*", *jev_models} <= picker_models + + +@pytest.mark.asyncio +async def test_scope_expand_still_lists_models_when_the_typesafe_key_reference_cannot_resolve( + openai_only_router: Router, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("LITELLM_QA_UNSET_TYPESAFE_TOKEN", raising=False) + monkeypatch.setenv("TYPESAFE_API_KEY", "oidc/env/LITELLM_QA_UNSET_TYPESAFE_TOKEN") + + picker_models: Final = await _admin_picker_models() + + assert {"gpt-4", "typesafe/*"} <= picker_models + + +@pytest.mark.asyncio +async def test_scope_expand_offers_no_typesafe_models_without_typesafe_credentials( + openai_only_router: Router, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("TYPESAFE_API_KEY", raising=False) + + picker_models: Final = await _admin_picker_models() + + assert "gpt-4" in picker_models + assert picker_models.isdisjoint({"typesafe/*", *_priced_jev_models()}) + + +@pytest.mark.asyncio +async def test_scope_expand_does_not_read_the_secret_manager_for_typesafe_credentials( + openai_only_router: Router, monkeypatch: pytest.MonkeyPatch +) -> None: + secret_manager: Final = MagicMock(spec=CustomSecretManager) + secret_manager.sync_read_secret.return_value = None + monkeypatch.delenv("TYPESAFE_API_KEY", raising=False) + monkeypatch.setattr(litellm, "secret_manager_client", secret_manager) + monkeypatch.setattr(litellm, "_key_management_system", KeyManagementSystem.CUSTOM) + monkeypatch.setattr(litellm, "_key_management_settings", KeyManagementSettings(access_mode="read_only")) + + picker_models: Final = await _admin_picker_models() + + secrets_read: Final = { + call.kwargs["secret_name"] + for call in (*secret_manager.sync_read_secret.call_args_list, *secret_manager.async_read_secret.call_args_list) + } + assert "gpt-4" in picker_models + assert "TYPESAFE_API_KEY" not in secrets_read + + +@pytest.mark.asyncio +async def test_default_model_list_does_not_offer_typesafe_models( + openai_only_router: Router, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake") + admin: Final = UserAPIKeyAuth(api_key="sk-test", user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN) + + response: Final = await proxy_server.model_list(user_api_key_dict=admin) + + assert {model["id"] for model in response["data"]}.isdisjoint({"typesafe/*", *_priced_jev_models()}) + + +async def _key_dialog_models(user_role: LitellmUserRoles, team_id: str | None) -> set[str]: + caller: Final = UserAPIKeyAuth(api_key="sk-test", user_id="u", user_role=user_role) + response: Final = await proxy_server.model_list( + user_api_key_dict=caller, return_wildcard_routes=True, include_model_access_groups=True, team_id=team_id + ) + return {model["id"] for model in response["data"]} + + +@pytest.mark.asyncio +async def test_key_dialog_without_a_team_offers_typesafe_models_to_an_admin( + openai_only_router: Router, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake") + + dialog_models: Final = await _key_dialog_models(LitellmUserRoles.PROXY_ADMIN, team_id=None) + + assert {"gpt-4", "typesafe/*", *_priced_jev_models()} <= dialog_models + + +@pytest.mark.asyncio +@pytest.mark.parametrize("scope", ["expand", None]) +async def test_pickers_offer_typesafe_names_once_and_no_bare_name_for_registered_typesafe_deployments( + monkeypatch: pytest.MonkeyPatch, scope: str | None +) -> None: + monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake") + jev_models: Final = _priced_jev_models() + monkeypatch.setattr(litellm, "model_cost", {**litellm.model_cost}) + router: Final = Router( + model_list=[ + {"model_name": "typesafe/*", "litellm_params": {"model": "typesafe/*", "api_key": "sk-typesafe-fake"}}, + { + "model_name": "ticket-router", + "litellm_params": {"model": "typesafe/my-classifier", "api_key": "sk-typesafe-fake"}, + }, + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "llm_model_list", router.model_list) + monkeypatch.setattr(proxy_server, "prisma_client", None) + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(proxy_server, "user_model", None) + admin: Final = UserAPIKeyAuth(api_key="sk-test", user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN) + + response: Final = await proxy_server.model_list( + user_api_key_dict=admin, return_wildcard_routes=True, include_model_access_groups=True, scope=scope + ) + + listed: Final = [model["id"] for model in response["data"]] + assert {"ticket-router", *jev_models} <= set(listed) + assert listed.count("typesafe/*") == 1 + assert set(listed).isdisjoint({"*", "my-classifier"}) + + +@pytest.mark.asyncio +async def test_key_dialog_offers_no_typesafe_models_to_an_internal_user( + openai_only_router: Router, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake") + + dialog_models: Final = await _key_dialog_models(LitellmUserRoles.INTERNAL_USER, team_id=None) + + assert "gpt-4" in dialog_models + assert dialog_models.isdisjoint({"typesafe/*", *_priced_jev_models()}) + + +@pytest.mark.asyncio +async def test_key_dialog_for_a_team_offers_no_typesafe_models_the_team_lacks( + openai_only_router: Router, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake") + + dialog_models: Final = await _key_dialog_models(LitellmUserRoles.PROXY_ADMIN, team_id="team-without-typesafe") + + assert dialog_models.isdisjoint({"typesafe/*", *_priced_jev_models()}) + + +@pytest.mark.asyncio +async def test_scope_expand_lists_no_typesafe_models_to_a_client_not_asking_for_allowlist_names( + openai_only_router: Router, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake") + admin: Final = UserAPIKeyAuth(api_key="sk-test", user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN) + + response: Final = await proxy_server.model_list(user_api_key_dict=admin, scope="expand") + + listed: Final = {model["id"] for model in response["data"]} + assert "gpt-4" in listed + assert listed.isdisjoint({"typesafe/*", *_priced_jev_models()}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("scope", "return_wildcard_routes", "only_model_access_groups"), + [ + pytest.param(None, False, True, id="access-group-dropdown"), + pytest.param(None, None, None, id="dashboard-list-without-wildcard-routes"), + pytest.param("expand", None, None, id="expanded-list-without-wildcard-routes"), + pytest.param(None, True, True, id="access-groups-only-with-wildcard-routes"), + ], +) +async def test_dashboard_lists_not_asking_for_allowlist_names_offer_no_typesafe_models( + openai_only_router: Router, + monkeypatch: pytest.MonkeyPatch, + scope: str | None, + return_wildcard_routes: bool | None, + only_model_access_groups: bool | None, +) -> None: + monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake") + admin: Final = UserAPIKeyAuth(api_key="sk-test", user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN) + + response: Final = await proxy_server.model_list( + user_api_key_dict=admin, + include_model_access_groups=True, + return_wildcard_routes=return_wildcard_routes, + only_model_access_groups=only_model_access_groups, + scope=scope, + ) + + assert {model["id"] for model in response["data"]}.isdisjoint({"typesafe/*", *_priced_jev_models()}) From 56e60d3b7d4b472b0c641d47202369c3dc3ad9c8 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 28 Oct 2026 20:20:01 -0700 Subject: [PATCH 09/10] feat(router): daily_reset_time picks when deployment rpd and tpd daily limits start over (#48957) * feat(router): reset_time picks when deployment rpd and tpd daily limits start over reset_time is "HH:MM" in UTC next to rpd/tpd in litellm_params or model_info. The daily cache keys are dated by now minus that offset and the daily 429 retry-after counts to the next reset, so a free tier whose provider quota resets at 08:00 UTC comes back at 08:00 instead of midnight. Without it the window stays the UTC calendar day. * chore(ui): regenerate schema.d.ts with reset_time * refactor(router): rpd, tpd and reset_time are read from litellm_params only * feat(router): rename reset_time to daily_reset_time, drop the request-sent value, accept only HH:MM * chore(ui): regenerate schema.d.ts for daily_reset_time --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_core_utils/get_litellm_params.py | 1 + litellm/router.py | 1 + .../pre_call_checks/model_rate_limit_check.py | 64 ++++--- litellm/types/litellm_params.py | 1 + litellm/types/router.py | 23 +++ .../test_enforce_model_rate_limits.py | 158 ++++++++++++++++++ tests/unit/types/test_litellm_params.py | 1 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 + 8 files changed, 229 insertions(+), 24 deletions(-) diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index 8178b7e98b8..8792fd0f58a 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -77,6 +77,7 @@ OPTIONAL_KWARGS_KEYS: Final = ( "otpm", "tpd", "rpd", + "daily_reset_time", "use_xai_oauth", "chatgpt_auth_file", "github_copilot_auth_type", diff --git a/litellm/router.py b/litellm/router.py index 0408819e05c..b63130c3313 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -4921,6 +4921,7 @@ class Router: deployment=deployment, forwarded_keys=kwargs.pop(_ALIAS_MARKER_FORWARDED_PARAMS_KWARG, ()) ): kwargs.pop(key, None) + kwargs.pop("daily_reset_time", None) # pyright: ignore[reportUnknownMemberType] # kwargs is an untyped dict defaults: Final = _LITELLM_PARAMS_ADAPTER.validate_python(self.default_litellm_params) self._merge_tools_from_deployment(deployment=deployment, kwargs=kwargs, defaults=defaults) diff --git a/litellm/router_utils/pre_call_checks/model_rate_limit_check.py b/litellm/router_utils/pre_call_checks/model_rate_limit_check.py index 9eebcf39515..f4b41211c6e 100644 --- a/litellm/router_utils/pre_call_checks/model_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/model_rate_limit_check.py @@ -12,7 +12,7 @@ is logged the first time such a deployment is seen. """ import contextlib -from collections.abc import Callable +from collections.abc import Callable, Mapping from datetime import datetime, timedelta from typing import TYPE_CHECKING, Any, Final, Literal @@ -38,7 +38,7 @@ from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import ( ) from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET from litellm.types.caching import RedisPipelineIncrementOperation -from litellm.types.router import RouterErrors +from litellm.types.router import RouterErrors, parse_daily_reset_time from litellm.types.utils import StandardLoggingPayload from litellm.utils import get_utc_datetime @@ -58,42 +58,49 @@ class RoutingArgs: DAY_FORMAT: Final = "%Y-%m-%d" -def seconds_until_utc_midnight(now: datetime) -> int: - next_midnight: Final = (now + timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0) - return max(1, int((next_midnight - now).total_seconds())) +def daily_window_day(now: datetime, reset_offset: timedelta) -> str: + return (now - reset_offset).strftime(DAY_FORMAT) -def _first_set(*values: int | None) -> int | None: - return next((value for value in values if value is not None), None) +def seconds_until_daily_reset(now: datetime, reset_offset: timedelta = timedelta(0)) -> int: + window_start: Final = (now - reset_offset).replace(hour=0, minute=0, second=0, microsecond=0) + next_reset: Final = window_start + timedelta(days=1) + reset_offset + return max(1, int((next_reset - now).total_seconds())) -class _DailyLimitFields(BaseModel): +class _DailyLimitParams(BaseModel): + """rpd, tpd and daily_reset_time live in litellm_params only.""" + model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True) + model: str | None = None tpd: int | None = None rpd: int | None = None + daily_reset_time: str | None = None + + def reset_offset(self) -> timedelta: + return parse_daily_reset_time(self.daily_reset_time) -class _DailyLimitParams(_DailyLimitFields): - model: str | None = None - - -class _DailyLimitModelInfo(_DailyLimitFields): +class _DailyLimitModelInfo(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True) id: str | None = None -class _DailyLimitDeployment(_DailyLimitFields): +class _DailyLimitDeployment(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True) model_name: str = "" litellm_params: _DailyLimitParams = _DailyLimitParams() model_info: _DailyLimitModelInfo = _DailyLimitModelInfo() def limits(self) -> tuple[int | None, int | None]: - return ( - _first_set(self.tpd, self.litellm_params.tpd, self.model_info.tpd), - _first_set(self.rpd, self.litellm_params.rpd, self.model_info.rpd), - ) + return self.litellm_params.tpd, self.litellm_params.rpd - def cache_keys(self, current_day: str) -> tuple[str, str]: + def reset_offset(self) -> timedelta: + return self.litellm_params.reset_offset() + + def cache_keys(self, now: datetime) -> tuple[str, str]: prefix: Final = f"{self.model_info.id}:{self.litellm_params.model}" + current_day: Final = daily_window_day(now, self.reset_offset()) return f"{prefix}:tpd:{current_day}", f"{prefix}:rpd:{current_day}" def limit_error(self, kind: Literal["tpd", "rpd"], limit: int, usage: int, now: datetime) -> litellm.RateLimitError: @@ -104,12 +111,19 @@ class _DailyLimitDeployment(_DailyLimitFields): response=httpx.Response( status_code=429, content=f"{RouterErrors.user_defined_ratelimit_error.value} {kind} limit={limit}. current usage={usage}. id={self.model_info.id}, model_group={self.model_name}", - headers={"retry-after": str(seconds_until_utc_midnight(now))}, + headers={"retry-after": str(seconds_until_daily_reset(now, self.reset_offset()))}, request=httpx.Request(method="model_rate_limit_check", url="https://github.com/BerriAI/litellm"), ), ) +def _reset_offset_from_kwargs(kwargs: Mapping[str, object]) -> timedelta: + litellm_params: Final = kwargs.get("litellm_params") + if litellm_params is None: + return timedelta(0) + return _DailyLimitParams.model_validate(litellm_params).reset_offset() + + class ModelRateLimitingCheck(CustomLogger): """ Pre-call check that enforces TPM/RPM or ITPM/OTPM limits on model deployments. @@ -242,7 +256,7 @@ class ModelRateLimitingCheck(CustomLogger): tpd_limit, rpd_limit = deployment.limits() if tpd_limit is None and rpd_limit is None: return - tpd_key, rpd_key = deployment.cache_keys(now.strftime(DAY_FORMAT)) + tpd_key, rpd_key = deployment.cache_keys(now) if tpd_limit is not None: current_tpd: Final = self._get_current_tpm(tpd_key, tpd_limit) if current_tpd is not None and current_tpd >= tpd_limit: @@ -394,7 +408,7 @@ class ModelRateLimitingCheck(CustomLogger): current_minute: Final = dt.strftime("%H-%M") tpm_key, rpm_key = self._get_cache_keys(deployment, current_minute) - tpd_key, rpd_key = daily.cache_keys(dt.strftime(DAY_FORMAT)) + tpd_key, rpd_key = daily.cache_keys(dt) model_id: Final = deployment.get("model_info", {}).get("id") model_name: Final = deployment.get("litellm_params", {}).get("model") @@ -510,7 +524,8 @@ class ModelRateLimitingCheck(CustomLogger): dt: Final = self.clock() current_minute: Final = dt.strftime("%H-%M") tpm_key: Final = f"{model_id}:{model}:tpm:{current_minute}" - tpd_key: Final = f"{model_id}:{model}:tpd:{dt.strftime(DAY_FORMAT)}" + reset_offset: Final = _reset_offset_from_kwargs(kwargs) # pyright: ignore[reportUnknownArgumentType] # CustomLogger passes an untyped kwargs dict + tpd_key: Final = f"{model_id}:{model}:tpd:{daily_window_day(dt, reset_offset)}" verbose_router_logger.debug("[TPM TRACKING] Incrementing %s by %s", tpm_key, total_tokens) @@ -570,7 +585,8 @@ class ModelRateLimitingCheck(CustomLogger): dt: Final = self.clock() current_minute: Final = dt.strftime("%H-%M") tpm_key: Final = f"{model_id}:{model}:tpm:{current_minute}" - tpd_key: Final = f"{model_id}:{model}:tpd:{dt.strftime(DAY_FORMAT)}" + reset_offset: Final = _reset_offset_from_kwargs(kwargs) # pyright: ignore[reportUnknownArgumentType] # CustomLogger passes an untyped kwargs dict + tpd_key: Final = f"{model_id}:{model}:tpd:{daily_window_day(dt, reset_offset)}" tokens: Final = int(total_tokens) self._increment(tpm_key, tokens, RoutingArgs.ttl) diff --git a/litellm/types/litellm_params.py b/litellm/types/litellm_params.py index 9196b7a226f..2065093f295 100644 --- a/litellm/types/litellm_params.py +++ b/litellm/types/litellm_params.py @@ -191,6 +191,7 @@ class DeploymentOptions: otpm: int | None = None rpd: int | None = None tpd: int | None = None + daily_reset_time: str | None = None kubernetes_pod_discovery: bool | None = None default_api_key_rpm_limit: int | None = None default_api_key_tpm_limit: int | None = None diff --git a/litellm/types/router.py b/litellm/types/router.py index f8de2128a80..60c9e596588 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -4,6 +4,7 @@ litellm.Router Types - includes RouterConfig, UpdateRouterConfig, ModelInfo etc import datetime import enum +import re from collections.abc import Container, Mapping, Sequence from dataclasses import dataclass, field from typing import ( @@ -518,6 +519,19 @@ def holds_secret_pointer(param_name: str) -> bool: _RESERVED_INIT_KEYS: Final = frozenset({"self", "params", "__class__"}) +_DAILY_RESET_TIME_PATTERN: Final = re.compile(r"\d{2}:\d{2}") + + +def parse_daily_reset_time(value: str | None) -> datetime.timedelta: + """`daily_reset_time` is "HH:MM" in UTC, when the rpd/tpd window starts over; unset means midnight UTC.""" + if value is None: + return datetime.timedelta(0) + if _DAILY_RESET_TIME_PATTERN.fullmatch(value) is None: + raise ValueError(f"daily_reset_time must be HH:MM in UTC, got {value!r}") + parsed: Final = datetime.time.fromisoformat(value) + return datetime.timedelta(hours=parsed.hour, minutes=parsed.minute) + + class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): """ LiteLLM Params without 'model' arg (used across completion / assistants api) @@ -535,6 +549,7 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): otpm: int | None = None tpd: int | None = None rpd: int | None = None + daily_reset_time: str | None = None timeout: float | str | httpx.Timeout | SDKTimeout | None = None # if str, pass in as os.environ/ # read timeout of a live stream, a faked stream takes the longer of this and timeout. If str, pass in as os.environ/ stream_timeout: float | str | None = None @@ -626,6 +641,13 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): valkey_text_field: str | None = None valkey_embedding_field: str | None = None + @field_validator("daily_reset_time") + @classmethod + def _daily_reset_time_is_utc_hh_mm(cls, value: str | None) -> str | None: + if value is not None: + parse_daily_reset_time(value) + return value + @field_validator("timeout") @classmethod def normalize_timeout( @@ -742,6 +764,7 @@ class LiteLLMParamsTypedDict(TypedDict, total=False): otpm: int | None tpd: ReadOnly[int | None] rpd: ReadOnly[int | None] + daily_reset_time: ReadOnly[str | None] order: int | None weight: int | None max_parallel_requests: int | None diff --git a/tests/unit/test_router/test_enforce_model_rate_limits.py b/tests/unit/test_router/test_enforce_model_rate_limits.py index 2a260fdf0b8..ff3ffc0db0c 100644 --- a/tests/unit/test_router/test_enforce_model_rate_limits.py +++ b/tests/unit/test_router/test_enforce_model_rate_limits.py @@ -7,9 +7,11 @@ regardless of the routing strategy being used. import asyncio from datetime import datetime, timedelta, timezone +from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest +from pydantic import TypeAdapter import litellm from litellm import Router @@ -724,3 +726,159 @@ class TestDailyLimits: router.discard() assert served == ["a", "b", "b", "b"] + + +_LOGGED_TOKENS: Final = TypeAdapter[float | None](float | None) + + +async def _tpd_tokens_once_logged(cache: DualCache, key: str) -> int: + for _ in range(200): + logged: Final = _LOGGED_TOKENS.validate_python( + cache.in_memory_cache.get_cache(key) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # InMemoryCache.get_cache is untyped + ) + if logged is not None: + return int(logged) + await asyncio.sleep(0.01) + pytest.fail(f"the success callback never logged tpd tokens under {key}") + + +RESET_AT_EIGHT = datetime(2026, 10, 28, 8, 0, 0, tzinfo=timezone.utc) +DAILY_RESET_DEPLOYMENT = { + **RPD_DEPLOYMENT, + "litellm_params": {"model": "gpt-4.1-mini", "rpd": 2, "daily_reset_time": "08:00"}, +} + + +class TestDailyResetTime: + def test_rpd_window_starts_over_at_daily_reset_time_not_midnight(self): + clock = Clock(RESET_AT_EIGHT - timedelta(hours=9)) + check = ModelRateLimitingCheck(dual_cache=DualCache(), clock=clock) + + assert _pre_call(check, DAILY_RESET_DEPLOYMENT) == DAILY_RESET_DEPLOYMENT + clock.now = RESET_AT_EIGHT - timedelta(hours=7) + assert _pre_call(check, DAILY_RESET_DEPLOYMENT) == DAILY_RESET_DEPLOYMENT + clock.now = RESET_AT_EIGHT - timedelta(minutes=1) + with pytest.raises(litellm.RateLimitError, match="RPD limit=2, current usage=3") as exc_info: + _pre_call(check, DAILY_RESET_DEPLOYMENT) + assert exc_info.value.response.headers["retry-after"] == "60" + + clock.now = RESET_AT_EIGHT + assert _pre_call(check, DAILY_RESET_DEPLOYMENT) == DAILY_RESET_DEPLOYMENT + + @pytest.mark.asyncio + async def test_async_rpd_window_starts_over_at_daily_reset_time_not_midnight(self): + clock = Clock(RESET_AT_EIGHT - timedelta(hours=9)) + check = ModelRateLimitingCheck(dual_cache=DualCache(), clock=clock) + + assert await _async_pre_call(check, DAILY_RESET_DEPLOYMENT) == DAILY_RESET_DEPLOYMENT + clock.now = RESET_AT_EIGHT - timedelta(hours=7) + assert await _async_pre_call(check, DAILY_RESET_DEPLOYMENT) == DAILY_RESET_DEPLOYMENT + clock.now = RESET_AT_EIGHT - timedelta(minutes=1) + with pytest.raises(litellm.RateLimitError, match="RPD limit=2, current usage=3") as exc_info: + await _async_pre_call(check, DAILY_RESET_DEPLOYMENT) + assert exc_info.value.response.headers["retry-after"] == "60" + + clock.now = RESET_AT_EIGHT + assert await _async_pre_call(check, DAILY_RESET_DEPLOYMENT) == DAILY_RESET_DEPLOYMENT + + @pytest.mark.parametrize( + ("now", "seconds_to_reset"), + [ + (RESET_AT_EIGHT + timedelta(seconds=1), 86399), + (datetime(2026, 10, 28, 23, 59, 30, tzinfo=timezone.utc), 28830), + (datetime(2026, 10, 29, 7, 30, 0, tzinfo=timezone.utc), 1800), + ], + ) + def test_daily_429_retry_after_is_seconds_until_daily_reset_time(self, now: datetime, seconds_to_reset: int): + deployment = { + **RPD_DEPLOYMENT, + "litellm_params": {"model": "gpt-4.1-mini", "rpd": 0, "daily_reset_time": "08:00"}, + } + check = ModelRateLimitingCheck(dual_cache=DualCache(), clock=Clock(now)) + + with pytest.raises(litellm.RateLimitError) as exc_info: + _pre_call(check, deployment) + + assert exc_info.value.response.headers["retry-after"] == str(seconds_to_reset) + + @pytest.mark.asyncio + async def test_tokens_logged_before_daily_reset_time_do_not_count_after_it(self): + deployment = { + **RPD_DEPLOYMENT, + "litellm_params": {"model": "gpt-4.1-mini", "tpd": 100, "daily_reset_time": "08:00"}, + } + clock = Clock(RESET_AT_EIGHT - timedelta(minutes=1)) + check = ModelRateLimitingCheck(dual_cache=DualCache(), clock=clock) + await _async_log_success( + check, + { + "litellm_params": {"daily_reset_time": "08:00", "metadata": {}}, + "standard_logging_object": { + "model_id": "daily-test-id", + "total_tokens": 120, + "hidden_params": {"litellm_model_name": "gpt-4.1-mini"}, + }, + }, + ) + with pytest.raises(litellm.RateLimitError, match="TPD limit=100, current usage=120"): + await _async_pre_call(check, deployment) + + clock.now = RESET_AT_EIGHT + assert await _async_pre_call(check, deployment) == deployment + + @pytest.mark.parametrize( + "deployment", + [ + {"litellm_params": {"model": "gpt-4.1-mini"}, "model_info": {"id": "x", "rpd": 0}, "model_name": "m"}, + {"litellm_params": {"model": "gpt-4.1-mini"}, "model_info": {"id": "x"}, "model_name": "m", "rpd": 0}, + ], + ) + def test_rpd_outside_litellm_params_is_not_a_limit(self, deployment: dict[str, object]): + check = ModelRateLimitingCheck(dual_cache=DualCache(), clock=Clock(DAY_ONE)) + + assert _pre_call(check, deployment) == deployment + + @pytest.mark.parametrize("daily_reset_time", ["8am", "08:00:30", "08:00+05:00", "25:00"]) + def test_router_rejects_a_daily_reset_time_that_is_not_hh_mm(self, daily_reset_time: str): + with pytest.raises(Exception, match="daily_reset_time"): + Router( + model_list=[ + { + "model_name": "daily-group", + "litellm_params": {"model": "gpt-4.1-mini", "rpd": 2, "daily_reset_time": daily_reset_time}, + } + ] + ) + + @pytest.mark.asyncio + async def test_a_daily_reset_time_sent_in_the_request_does_not_move_the_tokens_out_of_the_deployment_window(self): + router = Router( + model_list=[ + { + "model_name": "daily-group", + "litellm_params": { + "model": "openai/gpt-4.1-mini", + "api_key": "k", + "mock_response": "a", + "tpd": 100, + }, + "model_info": {"id": "daily-test-id"}, + } + ] + ) + check = ModelRateLimitingCheck(dual_cache=router.cache, clock=Clock(RESET_AT_EIGHT - timedelta(minutes=1))) + router.optional_callbacks = [check] + litellm.logging_callback_manager.add_litellm_callback(check) + try: + await router.acompletion( # pyright: ignore[reportUnknownMemberType] # Router.acompletion kwargs are untyped + model="daily-group", messages=[{"role": "user", "content": "hi"}], daily_reset_time="08:00" + ) + logged = await _tpd_tokens_once_logged(router.cache, "daily-test-id:openai/gpt-4.1-mini:tpd:2026-10-28") + finally: + router.discard() + + assert logged > 0 + day_before: Final = _LOGGED_TOKENS.validate_python( + router.cache.in_memory_cache.get_cache("daily-test-id:openai/gpt-4.1-mini:tpd:2026-10-27") # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # InMemoryCache.get_cache is untyped + ) + assert day_before is None diff --git a/tests/unit/types/test_litellm_params.py b/tests/unit/types/test_litellm_params.py index cc1ea645012..e369b7e93ab 100644 --- a/tests/unit/types/test_litellm_params.py +++ b/tests/unit/types/test_litellm_params.py @@ -156,6 +156,7 @@ OPTION_NAMES: Final = ( "tpm", "rpd", "tpd", + "daily_reset_time", "itpm", "otpm", "default_api_key_rpm_limit", diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index a49f1099121..51348fa9c75 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -35397,6 +35397,8 @@ export interface components { cost_per_second?: number | null; /** Custom Llm Provider */ custom_llm_provider?: string | null; + /** Daily Reset Time */ + daily_reset_time?: string | null; /** Default Api Key Rpm Limit */ default_api_key_rpm_limit?: number | null; /** Default Api Key Tpm Limit */ @@ -50793,6 +50795,8 @@ export interface components { cost_per_second?: number | null; /** Custom Llm Provider */ custom_llm_provider?: string | null; + /** Daily Reset Time */ + daily_reset_time?: string | null; /** Default Api Key Rpm Limit */ default_api_key_rpm_limit?: number | null; /** Default Api Key Tpm Limit */ From a2ef18b257c8ef6b160ddf9707d070f89d0b605d Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:23:22 -0700 Subject: [PATCH 10/10] test(unit): keep unix socket paths in unit tests under the macOS AF_UNIX limit (#45040) * test(proxy): keep the spend sidecar socket path under the macOS AF_UNIX limit * test(unit): share the short socket fixtures across unit tests Move socket_dir and socket_path to tests/unit/conftest.py so the datadog redis test's absent socket fails for the right reason on a Mac, rename the pgbouncer helper's directory argument, and record the rule in tests/unit/AGENTS.md * test(unit): fall back to /tmp when TMPDIR leaves no room for a socket path socket_dir now uses /tmp when TMPDIR plus the longest socket suffix would pass macOS's sun_path limit, and its teardown ignores errors so a failed test reports only its own failure * test(unit): keep socket fixtures inside tmp_path behind a short link socket_dir now links a short temp directory to tmp_path, so artifacts keep pytest's retention and a killed run leaks only an empty link. The fallback to /tmp counts the temp directory in bytes, and socket_tempdir points tempfile at it for the pooler tests whose socket directory production picks --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- tests/unit/AGENTS.md | 4 + tests/unit/conftest.py | 34 +++++ tests/unit/gateway/test_launch.py | 12 +- .../unit/integrations/datadog/test_datadog.py | 4 +- tests/unit/proxy/db/test_pgbouncer.py | 122 +++++++++--------- .../hooks/test_proxy_track_cost_callback.py | 5 +- .../test_spend_event_producer.py | 40 +++--- tests/unit/proxy/test_collector.py | 22 ++-- 8 files changed, 142 insertions(+), 101 deletions(-) diff --git a/tests/unit/AGENTS.md b/tests/unit/AGENTS.md index ca717e8efb0..ee6648c0a82 100644 --- a/tests/unit/AGENTS.md +++ b/tests/unit/AGENTS.md @@ -8,6 +8,10 @@ the same tests against Google's live spec Unit tests must not depend on the current time or execution speed. Use fixed timestamps for fixtures and controlled clocks for time-dependent behavior. Incidental clock reads inside production code are allowed +A unix socket goes under the `socket_dir` or `socket_path` fixture, never `tmp_path`, whose paths run past macOS's +104-byte AF_UNIX limit and fail only on a Mac. Code that opens a socket in its own `tempfile` directory runs under +`socket_tempdir` + ## What good looks like ```python diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index e64f5cca5b8..3070ebb0235 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -2,6 +2,8 @@ import asyncio import base64 import importlib import os +import shutil +import tempfile from collections.abc import Callable, Coroutine, Iterator from dataclasses import dataclass, field from pathlib import Path @@ -217,6 +219,38 @@ def _flush_client_caches() -> None: _reset_aws_auth_caches() +_MACOS_LONGEST_SOCKET_PATH: Final = 103 +_LONGEST_SOCKET_SUFFIX: Final = len("/sock12345678/d/litellm-pgbouncer-12345678/.s.PGSQL.65535") + + +def _socket_root() -> str: + default: Final = tempfile.gettempdir() + fits: Final = len(os.fsencode(default)) + _LONGEST_SOCKET_SUFFIX <= _MACOS_LONGEST_SOCKET_PATH + return default if fits else "/tmp" + + +@pytest.fixture +def socket_dir(tmp_path: Path) -> Iterator[Path]: + """``tmp_path`` behind a short link, so unix socket paths fit the sun_path limit that it exceeds on macOS.""" + link_dir: Final = Path(tempfile.mkdtemp(prefix="sock", dir=_socket_root())) + link: Final = link_dir / "d" + link.symlink_to(tmp_path, target_is_directory=True) + yield link + shutil.rmtree(link_dir, ignore_errors=True) + + +@pytest.fixture +def socket_path(socket_dir: Path) -> Path: + return socket_dir / "spend.sock" + + +@pytest.fixture +def socket_tempdir(socket_dir: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Point ``tempfile``'s default directory at ``socket_dir``, for code that picks its own socket directory.""" + monkeypatch.setattr(tempfile, "tempdir", str(socket_dir)) + return socket_dir + + @pytest.fixture(autouse=True, scope="session") def bundled_tiktoken_cache() -> None: importlib.import_module("litellm.litellm_core_utils.default_encoding") diff --git a/tests/unit/gateway/test_launch.py b/tests/unit/gateway/test_launch.py index f76aea030fe..4f5298e830b 100644 --- a/tests/unit/gateway/test_launch.py +++ b/tests/unit/gateway/test_launch.py @@ -140,7 +140,7 @@ class TestPoolDatabaseUrl: assert "DATABASE_URL" in outcome.reason def test_token_auth_hands_the_workers_the_pool_user_not_the_token( - self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path + self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, socket_tempdir: Path ): monkeypatch.setenv("IAM_TOKEN_DB_AUTH", "true") monkeypatch.setenv("AWS_REGION_NAME", "us-east-1") @@ -149,7 +149,7 @@ class TestPoolDatabaseUrl: with _minted_iam_token("MINTED_TOKEN"): outcome: Final = pool_database_url( DatabaseURLSettings.from_env(), - PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pooler(tmp_path))), + PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pooler(socket_tempdir))), environ, ) assert isinstance(outcome, str), outcome @@ -160,12 +160,12 @@ class TestPoolDatabaseUrl: class TestMain: def test_workers_inherit_the_loopback_url_the_supervisor_installed( - self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path + self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, socket_tempdir: Path ): port: Final = _free_port() monkeypatch.setenv("LITELLM_PGBOUNCER_ENABLED", "true") monkeypatch.setenv("LITELLM_PGBOUNCER_PORT", str(port)) - monkeypatch.setenv("LITELLM_PGBOUNCER_BINARY", str(_fake_pooler(tmp_path))) + monkeypatch.setenv("LITELLM_PGBOUNCER_BINARY", str(_fake_pooler(socket_tempdir))) monkeypatch.setenv("KEEPALIVE_TIMEOUT", "75") served: Final[list[tuple[str, ...]]] = [] main(("--workers", "4"), serve=lambda argv: served.append(tuple(argv))) @@ -183,7 +183,7 @@ class TestMain: assert _query(os.environ["DATABASE_URL"])["pgbouncer"] == "true" def test_iam_workers_keep_the_loopback_url_instead_of_minting_their_own( - self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path + self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, socket_tempdir: Path ): port: Final = _free_port() monkeypatch.delenv("DATABASE_PASSWORD") @@ -191,7 +191,7 @@ class TestMain: monkeypatch.setenv("AWS_REGION_NAME", "us-east-1") monkeypatch.setenv("LITELLM_PGBOUNCER_ENABLED", "true") monkeypatch.setenv("LITELLM_PGBOUNCER_PORT", str(port)) - monkeypatch.setenv("LITELLM_PGBOUNCER_BINARY", str(_fake_pooler(tmp_path))) + monkeypatch.setenv("LITELLM_PGBOUNCER_BINARY", str(_fake_pooler(socket_tempdir))) served: Final[list[tuple[str, ...]]] = [] with _minted_iam_token("SUPERVISOR_TOKEN"): main(("--workers", "4"), serve=lambda argv: served.append(tuple(argv))) diff --git a/tests/unit/integrations/datadog/test_datadog.py b/tests/unit/integrations/datadog/test_datadog.py index 14cd376cbc8..a9fd5ce27aa 100644 --- a/tests/unit/integrations/datadog/test_datadog.py +++ b/tests/unit/integrations/datadog/test_datadog.py @@ -905,10 +905,10 @@ async def test_a_failed_request_is_delivered_as_an_error_log_that_keeps_the_erro @pytest.mark.asyncio async def test_a_failing_redis_cache_is_delivered_to_datadog_as_redis_warnings( - delivery: tuple[DataDogLogger, respx.Route], monkeypatch: pytest.MonkeyPatch, tmp_path: Path + delivery: tuple[DataDogLogger, respx.Route], monkeypatch: pytest.MonkeyPatch, socket_dir: Path ) -> None: datadog_logger, intake = delivery - absent_socket: Final = str(tmp_path / "absent.sock") + absent_socket: Final = str(socket_dir / "absent.sock") redis_cache: Final = Cache(type="redis", url=f"unix://{absent_socket}") monkeypatch.setattr(redis_cache.cache.service_logger_obj, "dd_logger", datadog_logger, raising=False) monkeypatch.setattr(litellm, "service_callback", ["datadog"]) diff --git a/tests/unit/proxy/db/test_pgbouncer.py b/tests/unit/proxy/db/test_pgbouncer.py index c69c8d015a3..08861073008 100644 --- a/tests/unit/proxy/db/test_pgbouncer.py +++ b/tests/unit/proxy/db/test_pgbouncer.py @@ -277,7 +277,7 @@ def _free_port() -> int: def _fake_pooler( - tmp_path: Path, + directory: Path, port: int, exit_immediately: bool = False, port_file: Path | None = None, @@ -288,13 +288,13 @@ def _fake_pooler( """An executable that listens like PgBouncer: on the TCP port first, then on ``.s.PGSQL.`` in the socket dir. Port and socket dir come from the ini it is given, else from ``port`` and - ``tmp_path``. With ``port_file`` each start reads the port from that file + ``directory``. With ``port_file`` each start reads the port from that file instead. ``bind_delay_seconds`` holds the bind back, like a slow start. ``--version`` prints ``version_banner``. With ``auth_log`` it appends the ``auth_file`` it reads at startup and on every SIGHUP, one line per read, like PgBouncer loading its credentials. """ - script: Final = tmp_path / "fake-pgbouncer" + script: Final = directory / "fake-pgbouncer" script.write_text( textwrap.dedent( f"""\ @@ -316,7 +316,7 @@ def _fake_pooler( port = ini.getint("pgbouncer", "listen_port", fallback={port}) if not {port_file is None!r}: port = int(pathlib.Path({str(port_file)!r}).read_text()) - socket_dir = ini.get("pgbouncer", "unix_socket_dir", fallback={str(tmp_path)!r}) + socket_dir = ini.get("pgbouncer", "unix_socket_dir", fallback={str(directory)!r}) socket_path = f"{{socket_dir}}/.s.PGSQL.{{port}}" time.sleep({bind_delay_seconds!r}) listener = socket.socket() @@ -357,10 +357,10 @@ def _wait_until(condition: Callable[[], bool], timeout_seconds: float = 5.0) -> class TestPgBouncerProcess: - def test_start_waits_for_the_listener_and_stop_ends_it(self, tmp_path: Path): + def test_start_waits_for_the_listener_and_stop_ends_it(self, socket_dir: Path): port: Final = _free_port() pooler: Final = PgBouncerProcess( - argv=(str(_fake_pooler(tmp_path, port)),), port=port, socket_path=unix_socket_path(tmp_path, port) + argv=(str(_fake_pooler(socket_dir, port)),), port=port, socket_path=unix_socket_path(socket_dir, port) ) assert pooler.start() is None assert _listening(port) @@ -371,12 +371,12 @@ class TestPgBouncerProcess: with pytest.raises(ProcessLookupError): os.kill(pid, 0) - def test_a_crashed_pooler_is_restarted_with_a_new_pid(self, tmp_path: Path): + def test_a_crashed_pooler_is_restarted_with_a_new_pid(self, socket_dir: Path): port: Final = _free_port() pooler: Final = PgBouncerProcess( - argv=(str(_fake_pooler(tmp_path, port)),), + argv=(str(_fake_pooler(socket_dir, port)),), port=port, - socket_path=unix_socket_path(tmp_path, port), + socket_path=unix_socket_path(socket_dir, port), restart_delay_seconds=0.1, ) assert pooler.start() is None @@ -388,17 +388,17 @@ class TestPgBouncerProcess: assert _wait_until(lambda: not _listening(port)) def test_a_failed_restart_is_retried_until_the_pooler_is_back( - self, tmp_path: Path, caplog: pytest.LogCaptureFixture + self, socket_dir: Path, caplog: pytest.LogCaptureFixture ): port: Final = _free_port() - script: Final = _fake_pooler(tmp_path, port) + script: Final = _fake_pooler(socket_dir, port) pooler: Final = PgBouncerProcess( - argv=(str(script),), port=port, socket_path=unix_socket_path(tmp_path, port), restart_delay_seconds=0.1 + argv=(str(script),), port=port, socket_path=unix_socket_path(socket_dir, port), restart_delay_seconds=0.1 ) assert pooler.start() is None first_pid: Final = pooler.pid assert first_pid is not None - hidden: Final = script.rename(tmp_path / "hidden") + hidden: Final = script.rename(socket_dir / "hidden") with caplog.at_level(logging.ERROR, logger=verbose_proxy_logger.name): os.kill(first_pid, signal.SIGKILL) assert _wait_until(lambda: any("could not be restarted" in record.message for record in caplog.records)) @@ -408,15 +408,17 @@ class TestPgBouncerProcess: pooler.stop() assert _wait_until(lambda: not _listening(port)) - def test_a_replacement_that_never_listens_is_replaced_again(self, tmp_path: Path, caplog: pytest.LogCaptureFixture): + def test_a_replacement_that_never_listens_is_replaced_again( + self, socket_dir: Path, caplog: pytest.LogCaptureFixture + ): port: Final = _free_port() - port_file: Final = tmp_path / "port" + port_file: Final = socket_dir / "port" port_file.write_text(str(port)) - script: Final = _fake_pooler(tmp_path, port, port_file=port_file) + script: Final = _fake_pooler(socket_dir, port, port_file=port_file) pooler: Final = PgBouncerProcess( argv=(str(script),), port=port, - socket_path=unix_socket_path(tmp_path, port), + socket_path=unix_socket_path(socket_dir, port), restart_delay_seconds=0.1, ready_timeout_seconds=2.0, ) @@ -435,12 +437,12 @@ class TestPgBouncerProcess: pooler.stop() assert _wait_until(lambda: not _listening(port)) - def test_stopping_during_the_restart_delay_leaves_no_pooler_behind(self, tmp_path: Path): + def test_stopping_during_the_restart_delay_leaves_no_pooler_behind(self, socket_dir: Path): port: Final = _free_port() pooler: Final = PgBouncerProcess( - argv=(str(_fake_pooler(tmp_path, port)),), + argv=(str(_fake_pooler(socket_dir, port)),), port=port, - socket_path=unix_socket_path(tmp_path, port), + socket_path=unix_socket_path(socket_dir, port), restart_delay_seconds=0.3, ) assert pooler.start() is None @@ -453,12 +455,12 @@ class TestPgBouncerProcess: assert not _listening(port) assert pooler.pid == first_pid - def test_a_stopped_pooler_is_not_restarted(self, tmp_path: Path, caplog: pytest.LogCaptureFixture): + def test_a_stopped_pooler_is_not_restarted(self, socket_dir: Path, caplog: pytest.LogCaptureFixture): port: Final = _free_port() pooler: Final = PgBouncerProcess( - argv=(str(_fake_pooler(tmp_path, port)),), + argv=(str(_fake_pooler(socket_dir, port)),), port=port, - socket_path=unix_socket_path(tmp_path, port), + socket_path=unix_socket_path(socket_dir, port), restart_delay_seconds=0.1, ) assert pooler.start() is None @@ -468,31 +470,31 @@ class TestPgBouncerProcess: assert not _listening(port) assert caplog.records == [] - def test_a_pooler_that_exits_during_startup_is_reported(self, tmp_path: Path): + def test_a_pooler_that_exits_during_startup_is_reported(self, socket_dir: Path): port: Final = _free_port() pooler: Final = PgBouncerProcess( - argv=(str(_fake_pooler(tmp_path, port, exit_immediately=True)),), + argv=(str(_fake_pooler(socket_dir, port, exit_immediately=True)),), port=port, - socket_path=unix_socket_path(tmp_path, port), + socket_path=unix_socket_path(socket_dir, port), ) outcome: Final = pooler.start() assert isinstance(outcome, PgBouncerError) assert "status 3" in outcome.reason - def test_a_missing_binary_is_reported(self, tmp_path: Path): + def test_a_missing_binary_is_reported(self, socket_dir: Path): outcome: Final = PgBouncerProcess( - argv=("/nonexistent/pgbouncer",), port=_free_port(), socket_path=tmp_path / "sock" + argv=("/nonexistent/pgbouncer",), port=_free_port(), socket_path=socket_dir / "sock" ).start() assert isinstance(outcome, PgBouncerError) assert "/nonexistent/pgbouncer" in outcome.reason - def test_a_port_owned_by_someone_else_is_refused_before_spawning(self, tmp_path: Path): + def test_a_port_owned_by_someone_else_is_refused_before_spawning(self, socket_dir: Path): with socket.socket() as squatter: squatter.bind(("127.0.0.1", 0)) squatter.listen() port: Final = _bound_port(squatter) pooler: Final = PgBouncerProcess( - argv=(str(_fake_pooler(tmp_path, port)),), port=port, socket_path=unix_socket_path(tmp_path, port) + argv=(str(_fake_pooler(socket_dir, port)),), port=port, socket_path=unix_socket_path(socket_dir, port) ) outcome: Final = pooler.start() assert isinstance(outcome, PgBouncerError) @@ -500,13 +502,13 @@ class TestPgBouncerProcess: assert pooler.pid is None def test_a_replacement_waits_until_a_squatter_leaves_the_port( - self, tmp_path: Path, caplog: pytest.LogCaptureFixture + self, socket_dir: Path, caplog: pytest.LogCaptureFixture ): port: Final = _free_port() pooler: Final = PgBouncerProcess( - argv=(str(_fake_pooler(tmp_path, port)),), + argv=(str(_fake_pooler(socket_dir, port)),), port=port, - socket_path=unix_socket_path(tmp_path, port), + socket_path=unix_socket_path(socket_dir, port), restart_delay_seconds=0.5, ) assert pooler.start() is None @@ -524,12 +526,12 @@ class TestPgBouncerProcess: pooler.stop() assert _wait_until(lambda: not _listening(port)) - def test_a_listener_that_grabs_the_port_after_the_spawn_is_not_taken_for_the_pooler(self, tmp_path: Path): + def test_a_listener_that_grabs_the_port_after_the_spawn_is_not_taken_for_the_pooler(self, socket_dir: Path): port: Final = _free_port() pooler: Final = PgBouncerProcess( - argv=(str(_fake_pooler(tmp_path, port, bind_delay_seconds=0.5)),), + argv=(str(_fake_pooler(socket_dir, port, bind_delay_seconds=0.5)),), port=port, - socket_path=unix_socket_path(tmp_path, port), + socket_path=unix_socket_path(socket_dir, port), ready_timeout_seconds=3.0, ) with socket.socket() as squatter, ThreadPoolExecutor(max_workers=1) as starter: @@ -542,12 +544,12 @@ class TestPgBouncerProcess: assert isinstance(outcome, PgBouncerError) assert "exited with status 1" in outcome.reason - def test_a_port_served_by_a_stranger_while_the_pooler_is_still_starting_is_reported(self, tmp_path: Path): + def test_a_port_served_by_a_stranger_while_the_pooler_is_still_starting_is_reported(self, socket_dir: Path): port: Final = _free_port() pooler: Final = PgBouncerProcess( - argv=(str(_fake_pooler(tmp_path, port, bind_delay_seconds=30.0)),), + argv=(str(_fake_pooler(socket_dir, port, bind_delay_seconds=30.0)),), port=port, - socket_path=unix_socket_path(tmp_path, port), + socket_path=unix_socket_path(socket_dir, port), ready_timeout_seconds=0.5, ) with socket.socket() as squatter, ThreadPoolExecutor(max_workers=1) as starter: @@ -564,12 +566,12 @@ class TestPgBouncerProcess: with pytest.raises(ProcessLookupError): os.kill(pid, 0) - def test_a_pooler_that_never_listens_times_out(self, tmp_path: Path): + def test_a_pooler_that_never_listens_times_out(self, socket_dir: Path): port: Final = _free_port() pooler: Final = PgBouncerProcess( - argv=(str(_fake_pooler(tmp_path, _free_port())),), + argv=(str(_fake_pooler(socket_dir, _free_port())),), port=port, - socket_path=unix_socket_path(tmp_path, port), + socket_path=unix_socket_path(socket_dir, port), ready_timeout_seconds=0.5, ) outcome: Final = pooler.start() @@ -704,10 +706,10 @@ def _runtime_dir_listening_on(port: int) -> Path: class TestStartInContainerPgBouncer: - def test_returns_the_loopback_url_once_the_pooler_listens(self, tmp_path: Path): + def test_returns_the_loopback_url_once_the_pooler_listens(self, socket_tempdir: Path): port: Final = _free_port() - auth_log: Final = tmp_path / "auth.log" - binary: Final = _fake_pooler(tmp_path, port, auth_log=auth_log) + auth_log: Final = socket_tempdir / "auth.log" + binary: Final = _fake_pooler(socket_tempdir, port, auth_log=auth_log) settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(binary)) pooled: Final = start_in_container_pgbouncer(settings, "postgresql://app:pw@db/litellm?connection_limit=5") assert isinstance(pooled, str), pooled @@ -723,9 +725,9 @@ class TestStartInContainerPgBouncer: assert auth_log.read_text() == repr(f'"app" "pw"\n"litellm_pgbouncer" "{parsed.password}"\n') + "\n" @pytest.mark.filterwarnings("ignore:This process .* is multi-threaded:DeprecationWarning") - def test_a_forked_worker_exiting_leaves_the_pooler_and_its_files_to_the_parent(self, tmp_path: Path): + def test_a_forked_worker_exiting_leaves_the_pooler_and_its_files_to_the_parent(self, socket_tempdir: Path): port: Final = _free_port() - settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pooler(tmp_path, port))) + settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pooler(socket_tempdir, port))) exit_hooks: Final[list[Callable[[], None]]] = [] pooled: Final = start_in_container_pgbouncer( settings, "postgresql://app:pw@db/litellm", register_exit_hook=exit_hooks.append @@ -766,10 +768,10 @@ class TestStartInContainerPgBouncer: assert "IAM_TOKEN_DB_AUTH" in outcome.reason assert not _listening(port) - def test_token_auth_mints_the_first_token_into_the_auth_file_before_the_pooler_starts(self, tmp_path: Path): + def test_token_auth_mints_the_first_token_into_the_auth_file_before_the_pooler_starts(self, socket_tempdir: Path): port: Final = _free_port() - auth_log: Final = tmp_path / "auth.log" - binary: Final = _fake_pooler(tmp_path, port, auth_log=auth_log) + auth_log: Final = socket_tempdir / "auth.log" + binary: Final = _fake_pooler(socket_tempdir, port, auth_log=auth_log) settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(binary)) token: Final = _entra_jwt(datetime.now(tz=timezone.utc) + timedelta(hours=1)) pooled: Final = start_in_container_pgbouncer( @@ -798,25 +800,25 @@ class TestStartInContainerPgBouncer: assert "no Azure credential" in outcome.reason assert not _listening(port) - def test_a_renewed_token_is_written_and_picked_up_by_the_running_and_by_a_restarted_pooler(self, tmp_path: Path): + def test_a_renewed_token_is_written_and_picked_up_by_the_running_and_by_a_restarted_pooler(self, socket_dir: Path): port: Final = _free_port() - auth_log: Final = tmp_path / "auth.log" + auth_log: Final = socket_dir / "auth.log" plan: Final = plan_pgbouncer( - "postgresql://app@db/litellm", PgBouncerSettings(enabled=True, port=port), tmp_path, None + "postgresql://app@db/litellm", PgBouncerSettings(enabled=True, port=port), socket_dir, None ) assert isinstance(plan, PgBouncerPlan), plan - ini_path: Final = write_pgbouncer_ini(plan, tmp_path, None) - write_userlist(plan.userlist("first"), tmp_path, None) + ini_path: Final = write_pgbouncer_ini(plan, socket_dir, None) + write_userlist(plan.userlist("first"), socket_dir, None) pooler: Final = PgBouncerProcess( - argv=(str(_fake_pooler(tmp_path, port, auth_log=auth_log)), str(ini_path)), + argv=(str(_fake_pooler(socket_dir, port, auth_log=auth_log)), str(ini_path)), port=port, - socket_path=unix_socket_path(tmp_path, port), + socket_path=unix_socket_path(socket_dir, port), restart_delay_seconds=0.1, ) assert pooler.start() is None first_pid: Final = pooler.pid assert first_pid is not None - install_pgbouncer_token(plan, tmp_path, None, pooler, "second") + install_pgbouncer_token(plan, socket_dir, None, pooler, "second") assert _wait_until(lambda: auth_log.read_text().count("\n") == 2) os.kill(first_pid, signal.SIGKILL) assert _wait_until(lambda: pooler.pid not in (None, first_pid) and _listening(port)) @@ -837,9 +839,9 @@ class TestStartInContainerPgBouncer: assert "1.19" in outcome.reason assert not _listening(port) - def test_the_first_version_that_dies_on_a_failed_tcp_bind_is_accepted(self, tmp_path: Path): + def test_the_first_version_that_dies_on_a_failed_tcp_bind_is_accepted(self, socket_tempdir: Path): port: Final = _free_port() - binary: Final = _fake_pooler(tmp_path, port, version_banner="PgBouncer 1.19.0") + binary: Final = _fake_pooler(socket_tempdir, port, version_banner="PgBouncer 1.19.0") settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(binary)) pooled: Final = start_in_container_pgbouncer(settings, "postgresql://app:pw@db/litellm") assert isinstance(pooled, str), pooled diff --git a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py index 79193d37b04..4b5957a1237 100644 --- a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py @@ -3,6 +3,7 @@ import json import logging from collections.abc import Mapping from datetime import datetime, timedelta, timezone +from pathlib import Path from types import SimpleNamespace from typing import Final, cast from unittest.mock import AsyncMock, MagicMock, create_autospec, patch @@ -3062,10 +3063,10 @@ async def _no_fallback(line: bytes) -> None: @pytest.mark.asyncio -async def test_async_log_success_event_hands_the_sidecar_a_compact_event_and_skips_the_pipeline(tmp_path): +async def test_async_log_success_event_hands_the_sidecar_a_compact_event_and_skips_the_pipeline(socket_path: Path): handler = _RecordingHandler() consumer = SpendEventConsumer(handler) - address = UnixAddress(path=str(tmp_path / "spend.sock")) + address = UnixAddress(path=str(socket_path)) server = await consumer.serve(address) producer = SpendEventProducer( address=address, on_unavailable="fallback", buffer_size=10, connect_timeout=1.0, fallback=_no_fallback diff --git a/tests/unit/proxy/spend_tracking/test_spend_event_producer.py b/tests/unit/proxy/spend_tracking/test_spend_event_producer.py index adfb1251d1c..585616677e1 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_event_producer.py +++ b/tests/unit/proxy/spend_tracking/test_spend_event_producer.py @@ -177,9 +177,9 @@ def test_settings_read_the_documented_env(monkeypatch): @pytest.mark.asyncio -async def test_events_reach_the_sidecar_once_and_in_order(tmp_path: Path): +async def test_events_reach_the_sidecar_once_and_in_order(socket_path: Path): fallback: Final = _Fallback() - async with _Sidecar(tmp_path / "spend.sock") as sidecar: + async with _Sidecar(socket_path) as sidecar: producer: Final = _producer(sidecar.path, fallback) outcomes: Final = [await producer.publish(f"event-{i}\n".encode()) for i in range(20)] await producer.close(drain_timeout=5.0) @@ -193,9 +193,9 @@ async def test_events_reach_the_sidecar_once_and_in_order(tmp_path: Path): @pytest.mark.asyncio -async def test_unreachable_sidecar_falls_back_in_process_and_backs_off(tmp_path: Path): +async def test_unreachable_sidecar_falls_back_in_process_and_backs_off(socket_path: Path): fallback: Final = _Fallback() - producer: Final = _producer(tmp_path / "missing.sock", fallback) + producer: Final = _producer(socket_path.with_name("missing.sock"), fallback) first: Final = await producer.publish(b"event-1\n") await asyncio.sleep(0.05) second: Final = await producer.publish(b"event-2\n") @@ -210,11 +210,11 @@ async def test_unreachable_sidecar_falls_back_in_process_and_backs_off(tmp_path: @pytest.mark.parametrize("loop_factory", [asyncio.new_event_loop, uvloop.new_event_loop], ids=["asyncio", "uvloop"]) def test_sidecar_hang_up_falls_back_instead_of_losing_events( - tmp_path: Path, loop_factory: Callable[[], asyncio.AbstractEventLoop] + socket_path: Path, loop_factory: Callable[[], asyncio.AbstractEventLoop] ): async def scenario() -> tuple[list[bytes], list[bytes], tuple[int, int, int]]: fallback: Final = _Fallback() - sidecar: Final = _Sidecar(tmp_path / "spend.sock") + sidecar: Final = _Sidecar(socket_path) async with sidecar: producer: Final = _producer(sidecar.path, fallback) await producer.publish(b"event-1\n") @@ -236,7 +236,7 @@ def test_sidecar_hang_up_falls_back_instead_of_losing_events( @pytest.mark.parametrize("loop_factory", [asyncio.new_event_loop, uvloop.new_event_loop], ids=["asyncio", "uvloop"]) def test_mid_stream_crash_never_bills_an_event_on_both_sides( - tmp_path: Path, loop_factory: Callable[[], asyncio.AbstractEventLoop] + socket_path: Path, loop_factory: Callable[[], asyncio.AbstractEventLoop] ): """Events large enough to straddle the kernel buffer, a sidecar that reads some and then drops the socket: a failed write may only fall back when the sidecar cannot have read the whole line.""" @@ -244,7 +244,7 @@ def test_mid_stream_crash_never_bills_an_event_on_both_sides( async def scenario() -> tuple[list[bytes], list[bytes], tuple[int, int, int]]: fallback: Final = _Fallback() - sidecar: Final = _CrashingSidecar(tmp_path / "spend.sock", lines_before_crash=3) + sidecar: Final = _CrashingSidecar(socket_path, lines_before_crash=3) async with sidecar: producer: Final = _producer(sidecar.path, fallback) for event in events: @@ -266,11 +266,11 @@ def test_mid_stream_crash_never_bills_an_event_on_both_sides( @pytest.mark.asyncio -async def test_drain_timeout_hands_the_in_flight_event_to_fallback(tmp_path: Path): +async def test_drain_timeout_hands_the_in_flight_event_to_fallback(socket_path: Path): """A sidecar that stops reading leaves one event half-written; cancelling the writer must not lose it.""" fallback: Final = _Fallback() stuck: Final = b"x" * (4 * 1024 * 1024) + b"\n" - async with _Sidecar(tmp_path / "spend.sock", reads=False) as sidecar: + async with _Sidecar(socket_path, reads=False) as sidecar: producer: Final = _producer(sidecar.path, fallback) assert await producer.publish(stuck) == "queued" await asyncio.sleep(0.1) @@ -282,10 +282,10 @@ async def test_drain_timeout_hands_the_in_flight_event_to_fallback(tmp_path: Pat @pytest.mark.asyncio -async def test_shutdown_lets_the_writer_finish_a_fallback_already_in_progress(tmp_path: Path): +async def test_shutdown_lets_the_writer_finish_a_fallback_already_in_progress(socket_path: Path): """Cancelling the writer while it runs the pipeline in-process must neither lose nor repeat that event.""" fallback: Final = _GatedFallback() - producer: Final = _producer(tmp_path / "missing.sock", fallback) + producer: Final = _producer(socket_path.with_name("missing.sock"), fallback) assert await producer.publish(b"event-1\n") == "queued" await asyncio.wait_for(fallback.started.wait(), 5.0) closing: Final = asyncio.ensure_future(producer.close(drain_timeout=0.05)) @@ -299,10 +299,10 @@ async def test_shutdown_lets_the_writer_finish_a_fallback_already_in_progress(tm @pytest.mark.asyncio -async def test_shutdown_does_not_replay_an_event_the_kernel_already_took(tmp_path: Path): +async def test_shutdown_does_not_replay_an_event_the_kernel_already_took(socket_path: Path): """Cancelling a drain whose bytes already left the process must not run the event a second time in-process.""" fallback: Final = _Fallback() - async with _Sidecar(tmp_path / "spend.sock") as sidecar: + async with _Sidecar(socket_path) as sidecar: producer: Final = _producer(sidecar.path, fallback, open_connection=_open_with_stalled_drain) assert await producer.publish(b"event-1\n") == "queued" await asyncio.sleep(0.1) @@ -316,9 +316,9 @@ async def test_shutdown_does_not_replay_an_event_the_kernel_already_took(tmp_pat @pytest.mark.asyncio -async def test_drop_policy_counts_instead_of_running_in_process(tmp_path: Path): +async def test_drop_policy_counts_instead_of_running_in_process(socket_path: Path): fallback: Final = _Fallback() - producer: Final = _producer(tmp_path / "missing.sock", fallback, on_unavailable="drop") + producer: Final = _producer(socket_path.with_name("missing.sock"), fallback, on_unavailable="drop") await producer.publish(b"event-1\n") await producer.close(drain_timeout=5.0) assert await producer.publish(b"event-2\n") == "dropped" @@ -328,9 +328,9 @@ async def test_drop_policy_counts_instead_of_running_in_process(tmp_path: Path): @pytest.mark.asyncio -async def test_full_buffer_applies_the_unavailable_policy_immediately(tmp_path: Path): +async def test_full_buffer_applies_the_unavailable_policy_immediately(socket_path: Path): fallback: Final = _Fallback() - async with _Sidecar(tmp_path / "spend.sock") as sidecar: + async with _Sidecar(socket_path) as sidecar: producer: Final = _producer(sidecar.path, fallback, buffer_size=2) outcomes: Final = [await producer.publish(f"event-{i}\n".encode()) for i in range(3)] await producer.close(drain_timeout=5.0) @@ -342,9 +342,9 @@ async def test_full_buffer_applies_the_unavailable_policy_immediately(tmp_path: @pytest.mark.asyncio -async def test_close_flushes_buffered_events_then_refuses_new_ones(tmp_path: Path): +async def test_close_flushes_buffered_events_then_refuses_new_ones(socket_path: Path): fallback: Final = _Fallback() - async with _Sidecar(tmp_path / "spend.sock") as sidecar: + async with _Sidecar(socket_path) as sidecar: producer: Final = _producer(sidecar.path, fallback) for i in range(50): await producer.publish(f"event-{i}\n".encode()) diff --git a/tests/unit/proxy/test_collector.py b/tests/unit/proxy/test_collector.py index ac3e1f5e566..4dbaccc3cbf 100644 --- a/tests/unit/proxy/test_collector.py +++ b/tests/unit/proxy/test_collector.py @@ -49,14 +49,14 @@ class _Fallback: @pytest.mark.asyncio @pytest.mark.parametrize("transport", ["unix", "tcp"]) -async def test_consumer_handles_each_producer_line_once_in_order(tmp_path: Path, transport: str): +async def test_consumer_handles_each_producer_line_once_in_order(socket_path: Path, transport: str): handler: Final = _Handler(fail_on=b"event-3\n") consumer: Final = SpendEventConsumer(handler) server: Final = await consumer.serve( - UnixAddress(path=str(tmp_path / "spend.sock")) if transport == "unix" else TcpAddress("127.0.0.1", 0) + UnixAddress(path=str(socket_path)) if transport == "unix" else TcpAddress("127.0.0.1", 0) ) address: Final = ( - UnixAddress(path=str(tmp_path / "spend.sock")) + UnixAddress(path=str(socket_path)) if transport == "unix" else TcpAddress("127.0.0.1", server.sockets[0].getsockname()[1]) ) @@ -74,10 +74,10 @@ async def test_consumer_handles_each_producer_line_once_in_order(tmp_path: Path, @pytest.mark.asyncio -async def test_consumer_discards_a_truncated_trailing_event(tmp_path: Path): +async def test_consumer_discards_a_truncated_trailing_event(socket_path: Path): handler: Final = _Handler() consumer: Final = SpendEventConsumer(handler) - address: Final = UnixAddress(path=str(tmp_path / "spend.sock")) + address: Final = UnixAddress(path=str(socket_path)) server: Final = await consumer.serve(address) _, writer = await open_collector_connection(address, timeout=1.0) writer.write(b"whole\npartial-without-newline") @@ -93,9 +93,9 @@ async def test_consumer_discards_a_truncated_trailing_event(tmp_path: Path): @pytest.mark.asyncio -async def test_drain_reports_producers_still_connected_after_the_timeout(tmp_path: Path): +async def test_drain_reports_producers_still_connected_after_the_timeout(socket_path: Path): consumer: Final = SpendEventConsumer(_Handler()) - address: Final = UnixAddress(path=str(tmp_path / "spend.sock")) + address: Final = UnixAddress(path=str(socket_path)) server: Final = await consumer.serve(address) _, writer = await open_collector_connection(address, timeout=1.0) await asyncio.sleep(0.05) @@ -108,11 +108,11 @@ async def test_drain_reports_producers_still_connected_after_the_timeout(tmp_pat @pytest.mark.asyncio -async def test_graceful_stop_hands_the_producer_over_to_its_fallback_without_losing_events(tmp_path: Path): +async def test_graceful_stop_hands_the_producer_over_to_its_fallback_without_losing_events(socket_path: Path): handler: Final = _Handler() fallback: Final = _Fallback() consumer: Final = SpendEventConsumer(handler) - address: Final = UnixAddress(path=str(tmp_path / "spend.sock")) + address: Final = UnixAddress(path=str(socket_path)) server: Final = await consumer.serve(address) producer: Final = SpendEventProducer( address=address, on_unavailable="fallback", buffer_size=100, connect_timeout=1.0, fallback=fallback @@ -134,7 +134,7 @@ async def test_graceful_stop_hands_the_producer_over_to_its_fallback_without_los @pytest.mark.parametrize("loop_factory", [asyncio.new_event_loop, uvloop.new_event_loop], ids=["asyncio", "uvloop"]) def test_drain_still_hands_over_live_producers_when_another_connection_already_died( - tmp_path: Path, loop_factory: Callable[[], asyncio.AbstractEventLoop] + socket_path: Path, loop_factory: Callable[[], asyncio.AbstractEventLoop] ): """A transport the loop force-closed under a busy handler must not abort the half-close of the others.""" @@ -145,7 +145,7 @@ def test_drain_still_hands_over_live_producers_when_another_connection_already_d await release.wait() consumer: Final = SpendEventConsumer(slow_handler) - address: Final = UnixAddress(path=str(tmp_path / "spend.sock")) + address: Final = UnixAddress(path=str(socket_path)) server: Final = await consumer.serve(address) _, dead = await open_collector_connection(address, timeout=1.0) dead.write(b"stuck\n")