diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 158110cbbab..2826098fd1f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,5 +1,6 @@ /model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri /litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri +/ci_cd/cost_map_pins.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri /helm/ @yassin-berriai /terraform/ @yassin-berriai /.github/workflows/zizmor.yml @yuneng-berri diff --git a/.github/PULL_REQUEST_TEMPLATE/general.md b/.github/PULL_REQUEST_TEMPLATE/general.md index 243d121d626..86a23966671 100644 --- a/.github/PULL_REQUEST_TEMPLATE/general.md +++ b/.github/PULL_REQUEST_TEMPLATE/general.md @@ -84,9 +84,10 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac ## Screenshots / Proof of Fix \n\n| a |", + "contentFormat": "markdown", + "stringIndexType": "unicodeCodePoint", + "pages": [ + {"pageNumber": 1, "spans": [{"offset": 0, "length": 29}], "lines": [{"content": "Tïtle"}]}, + {"pageNumber": 2, "spans": [{"offset": 29, "length": 5}], "lines": [{"content": "a"}]} + ] + } + }))]) + .await; + + let response = perform(ocr_request( + LAYOUT_MODEL, + &upstream.uri(), + json!({"outputContentFormat": output_content_format, "pages": [0, 1]}), + )) + .await + .unwrap(); + + let sent = only_request(&upstream).await; + assert_eq!( + sent.query("outputContentFormat").as_deref(), + Some("markdown") + ); + assert_eq!( + sent.query("stringIndexType").as_deref(), + Some("unicodeCodePoint") + ); + assert_eq!(sent.query("pages").as_deref(), Some("1,2")); + assert_eq!(sent.json(), json!({"base64Source": "YWJj"})); + assert_eq!( + response.content.as_deref(), + Some("# Tïtle\n\n\n\n| a |") + ); + let pages = response + .pages + .iter() + .map(|page| page.markdown.as_str()) + .collect::>(); + assert_eq!(pages, ["# Tïtle\n\n\n\n", "| a |"]); + assert_eq!( + response.extra_fields.get("stringIndexType"), + Some(&json!("unicodeCodePoint")) + ); + assert_eq!( + response.usage_info.as_ref().unwrap().pages_processed, + Some(2) + ); +} + +#[rstest] +#[case::native_markdown(LAYOUT_MODEL, json!({"outputContentFormat": "markdown", "req_format": "native"}), Some("markdown"))] +#[case::markdown_on_read(MODEL, json!({"outputContentFormat": "markdown"}), Some("markdown"))] +#[case::text(LAYOUT_MODEL, json!({"outputContentFormat": "text"}), Some("text"))] +#[case::omitted(LAYOUT_MODEL, json!({}), None)] +#[tokio::test] +async fn only_layout_markdown_in_litellm_format_asks_for_code_point_offsets( + #[case] model: &str, + #[case] options: Value, + #[case] output_content_format: Option<&str>, +) { + let upstream = upstream([json_response(json!({ + "status": "succeeded", + "analyzeResult": { + "contentFormat": "text", + "stringIndexType": "textElements", + "pages": [{"pageNumber": 1, "lines": [{"content": "hello"}]}] + } + }))]) + .await; + + let response = perform(ocr_request(model, &upstream.uri(), options)) + .await + .unwrap(); + + let sent = only_request(&upstream).await; + assert_eq!( + sent.query("outputContentFormat").as_deref(), + output_content_format + ); + assert_eq!(sent.query("stringIndexType"), None); + assert_eq!(sent.json(), json!({"base64Source": "YWJj"})); + assert_eq!(response.pages[0].markdown, "hello"); + assert_eq!(response.extra_fields.get("stringIndexType"), None); +} + +#[rstest] +#[case::snake_case(json!({"output_content_format": "markdown"}), Some("markdown"))] +#[case::padded(json!({"outputContentFormat": " markdown "}), Some("markdown"))] +#[case::empty(json!({"outputContentFormat": ""}), None)] +#[case::empty_camel_case_falls_back_to_snake_case( + json!({"outputContentFormat": "", "output_content_format": "markdown"}), + Some("markdown") +)] +#[case::dropped(json!({"outputContentFormat": "html", "drop_params": true}), None)] +#[tokio::test] +async fn the_requested_output_content_format_reaches_azure_in_its_query_spelling( + #[case] options: Value, + #[case] output_content_format: Option<&str>, +) { + let upstream = upstream([json_response(json!({ + "status": "succeeded", + "analyzeResult": {"pages": [{"pageNumber": 1, "lines": [{"content": "hello"}]}]} + }))]) + .await; + + let response = perform(ocr_request(MODEL, &upstream.uri(), options)) + .await + .unwrap(); + + let sent = only_request(&upstream).await; + assert_eq!( + sent.query("outputContentFormat").as_deref(), + output_content_format + ); + assert_eq!(sent.query("output_content_format"), None); + assert_eq!(sent.query("drop_params"), None); + assert_eq!(sent.json(), json!({"base64Source": "YWJj"})); + assert_eq!(response.pages[0].markdown, "hello"); +} + #[tokio::test] async fn native_format_normalizes_pages_and_keeps_the_provider_response() { let operation = json!({ diff --git a/litellm-rust/crates/llms-types/src/formats/audio_transcription/response.rs b/litellm-rust/crates/llms-types/src/formats/audio_transcription/response.rs index 624eb7be4e4..211d4d0f648 100644 --- a/litellm-rust/crates/llms-types/src/formats/audio_transcription/response.rs +++ b/litellm-rust/crates/llms-types/src/formats/audio_transcription/response.rs @@ -1,14 +1,31 @@ use serde_json::Value; +#[macro_rules_attribute::apply(crate::wire_type)] +pub struct AudioTranscriptionUsage { + pub input_tokens: u64, + pub output_tokens: u64, + pub total_tokens: u64, +} + #[macro_rules_attribute::apply(crate::wire_type)] pub struct AudioTranscriptionResponseData { pub text: String, + pub usage: Option, } impl AudioTranscriptionResponseData { pub fn into_json(self) -> Value { - serde_json::json!({ - "text": self.text, - }) + match self.usage { + None => serde_json::json!({"text": self.text}), + Some(usage) => serde_json::json!({ + "text": self.text, + "usage": { + "type": "tokens", + "input_tokens": usage.input_tokens, + "output_tokens": usage.output_tokens, + "total_tokens": usage.total_tokens, + }, + }), + } } } diff --git a/litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs b/litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs index d7233db0a76..36dfe2c890e 100644 --- a/litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs +++ b/litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs @@ -1,4 +1,7 @@ -use std::{collections::BTreeSet, time::Duration}; +use std::{ + collections::{BTreeMap, BTreeSet}, + time::Duration, +}; use base64::{Engine, engine::general_purpose::STANDARD}; use litellm_auth::{InputSource, Sourced}; @@ -8,7 +11,7 @@ use litellm_llms_types::serde_compat::{FiniteF64, LaxI64}; use reqwest::Url; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; -use serde_with::serde_as; +use serde_with::{DefaultOnError, serde_as}; use tokio::time::Instant; use crate::base_llm::ocr::{ @@ -17,9 +20,10 @@ use crate::base_llm::ocr::{ handler::{CallHooks, OcrClient, read_json_response}, settings::OcrSettings, transformation::{ - BaseOcrConfig, DecodedOcrResponse, OCR_INLINE_MAX_BYTES, OCR_POLL_RETRY_SECS, - OcrConnection, OcrCredentialInputs, OcrResponseContext, PreparedOcrRequest, - ResolvedOcrCredentials, decode_and_normalize_response, decode_response, + BaseOcrConfig, DROP_PARAMS_PARAM, DecodedOcrResponse, OCR_INLINE_MAX_BYTES, + OCR_POLL_RETRY_SECS, OcrConnection, OcrCredentialInputs, OcrResponseContext, + PreparedOcrRequest, ResolvedOcrCredentials, decode_and_normalize_response, decode_response, + drops_unsupported_params, response_format, }, }; use litellm_llms_types::formats::ocr::{ @@ -32,6 +36,30 @@ const AZURE_DI_DEFAULT_HEIGHT: f64 = 11.0; const AZURE_DI_API_KEY_ENV: &str = "AZURE_DOCUMENT_INTELLIGENCE_API_KEY"; const AZURE_DI_ENDPOINT_ENV: &str = "AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT"; +const OUTPUT_CONTENT_FORMAT_PARAM: &str = "outputContentFormat"; +const OUTPUT_CONTENT_FORMAT_ALIAS: &str = "output_content_format"; +const CODE_POINT_STRING_INDEX: &str = "unicodeCodePoint"; +const STRING_INDEX_TYPE_FIELD: &str = "stringIndexType"; +const MARKDOWN_MODEL_ID: &str = "prebuilt-layout"; + +#[derive( + Clone, + Copy, + Debug, + PartialEq, + strum::EnumString, + strum::IntoStaticStr, + strum::Display, + strum::VariantArray, + serde_with::SerializeDisplay, +)] +#[strum(ascii_case_insensitive)] +pub enum OutputContentFormat { + #[strum(serialize = "text")] + Text, + #[strum(serialize = "markdown")] + Markdown, +} #[derive(Clone, Debug, PartialEq, Serialize)] pub struct DocumentIntelligenceParams { @@ -39,6 +67,13 @@ pub struct DocumentIntelligenceParams { pub pages: Option, #[serde(skip_serializing_if = "Option::is_none")] pub features: Option, + #[serde( + rename = "outputContentFormat", + skip_serializing_if = "Option::is_none" + )] + pub output_content_format: Option, + #[serde(rename = "stringIndexType", skip_serializing_if = "Option::is_none")] + pub string_index_type: Option<&'static str>, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -80,6 +115,10 @@ pub struct AzureDocumentIntelligenceOperation { #[derive(Clone, Debug, Default, Deserialize)] struct AzureDocumentIntelligenceAnalyzeResult { pub content: Option, + #[serde(rename = "contentFormat")] + pub content_format: Option, + #[serde(rename = "stringIndexType")] + pub string_index_type: Option, #[serde(default)] pub pages: Vec, pub tables: Option>>, @@ -100,6 +139,9 @@ struct AzureDocumentIntelligencePage { pub unit: Option, #[serde(default)] pub lines: Vec, + #[serde(default)] + #[serde_as(deserialize_as = "DefaultOnError")] + pub spans: Option>, } #[derive(Clone, Debug, Deserialize)] @@ -107,6 +149,18 @@ struct AzureDocumentIntelligenceLine { pub content: Option, } +#[derive(Clone, Debug, Deserialize)] +struct AzureDocumentIntelligenceSpan { + pub offset: usize, + pub length: usize, +} + +impl AzureDocumentIntelligenceSpan { + fn end(&self) -> usize { + self.offset.saturating_add(self.length) + } +} + #[derive(Clone, Debug)] pub struct AzureDocumentIntelligenceOcrConfig; @@ -116,7 +170,14 @@ impl BaseOcrConfig for AzureDocumentIntelligenceOcrConfig { type Environment = Vec<(String, String)>; fn get_supported_ocr_params(&self, _model: &str) -> &'static [&'static str] { - &["pages", "features", "req_format"] + &[ + "pages", + "features", + OUTPUT_CONTENT_FORMAT_PARAM, + OUTPUT_CONTENT_FORMAT_ALIAS, + "req_format", + DROP_PARAMS_PARAM, + ] } fn get_api_key_env_var(&self) -> Option<&'static str> { @@ -154,11 +215,17 @@ impl BaseOcrConfig for AzureDocumentIntelligenceOcrConfig { fn map_ocr_params( &self, non_default_params: &CallArguments, - _model: &str, + model: &str, ) -> Result { + let output_content_format = requested_output_content_format(non_default_params)?; + let slices_markdown_pages = output_content_format == Some(OutputContentFormat::Markdown) + && response_format(non_default_params)? != OcrResponseFormat::Native + && model_id(model).is_ok_and(|id| id == MARKDOWN_MODEL_ID); Ok(DocumentIntelligenceParams { pages: normalize_pages_param(non_default_params.get("pages"))?, features: normalize_features_param(non_default_params.get("features"))?, + output_content_format, + string_index_type: slices_markdown_pages.then_some(CODE_POINT_STRING_INDEX), }) } @@ -337,6 +404,36 @@ fn normalize_features_param(features: Option<&Value>) -> Result, Ok(Some(normalized.join(","))) } +fn requested_output_content_format( + params: &CallArguments, +) -> Result, Error> { + let requested = [OUTPUT_CONTENT_FORMAT_PARAM, OUTPUT_CONTENT_FORMAT_ALIAS] + .into_iter() + .filter_map(|name| params.get(name)) + .find(|format| !is_unset(format)); + match requested.map(parse_output_content_format).transpose() { + Err(Error::OutputContentFormat) if drops_unsupported_params(params) => Ok(None), + parsed => parsed, + } +} + +fn is_unset(format: &Value) -> bool { + match format { + Value::Null => true, + Value::String(format) => format.trim().is_empty(), + _ => false, + } +} + +fn parse_output_content_format(format: &Value) -> Result { + format + .as_str() + .ok_or(Error::OutputContentFormat)? + .trim() + .parse() + .map_err(|_| Error::OutputContentFormat) +} + fn build_request(document: OcrDocument) -> Result { let source = document.source(); if source.is_empty() { @@ -367,10 +464,27 @@ fn transform_completed_response( )); } let result = response.analyze_result.unwrap_or_default(); + let markdown_content = result + .content + .as_deref() + .filter(|_| { + result.content_format.as_deref() == Some(<&str>::from(OutputContentFormat::Markdown)) + && result.string_index_type.as_deref() == Some(CODE_POINT_STRING_INDEX) + && result.pages.iter().all(|page| page.spans.is_some()) + }) + .map(|content| { + CodePointText::new( + content, + result + .pages + .iter() + .flat_map(|page| page.spans.iter().flatten()), + ) + }); let pages = result .pages .into_iter() - .map(|page| transform_azure_page(page, dpi)) + .map(|page| transform_azure_page(page, markdown_content.as_ref(), dpi)) .collect::, _>>()?; let pages_processed = i64::try_from(pages.len()).map_err(|_| Error::NumericRange("pages"))?; Ok(LiteLLMOcrResponse { @@ -381,11 +495,20 @@ fn transform_completed_response( pages_processed: Some(pages_processed), ..Default::default() }), + extra_fields: result + .string_index_type + .filter(|unit| unit == CODE_POINT_STRING_INDEX) + .map(|unit| Map::from_iter([(STRING_INDEX_TYPE_FIELD.to_owned(), Value::String(unit))])) + .unwrap_or_default(), ..LiteLLMOcrResponse::new(model, pages) }) } -fn transform_azure_page(page: AzureDocumentIntelligencePage, dpi: i64) -> Result { +fn transform_azure_page( + page: AzureDocumentIntelligencePage, + markdown_content: Option<&CodePointText<'_>>, + dpi: i64, +) -> Result { let index = page .page_number .unwrap_or(1) @@ -397,12 +520,20 @@ fn transform_azure_page(page: AzureDocumentIntelligencePage, dpi: i64) -> Result page.unit.as_deref().unwrap_or("inch"), dpi, )?; - let markdown = page - .lines - .iter() - .map(|line| line.content.as_deref().unwrap_or_default()) - .collect::>() - .join("\n"); + let markdown = match markdown_content { + Some(content) => page + .spans + .iter() + .flatten() + .map(|span| content.slice(span)) + .collect::(), + None => page + .lines + .iter() + .map(|line| line.content.as_deref().unwrap_or_default()) + .collect::>() + .join("\n"), + }; Ok(OcrPage { index, markdown, @@ -411,6 +542,43 @@ fn transform_azure_page(page: AzureDocumentIntelligencePage, dpi: i64) -> Result }) } +struct CodePointText<'a> { + text: &'a str, + byte_offsets: BTreeMap, +} + +impl<'a> CodePointText<'a> { + fn new<'s>( + text: &'a str, + spans: impl Iterator, + ) -> Self { + let span_bounds = spans + .flat_map(|span| [span.offset, span.end()]) + .collect::>(); + Self { + text, + byte_offsets: text + .char_indices() + .map(|(byte_offset, _)| byte_offset) + .enumerate() + .filter(|(code_point, _)| span_bounds.contains(code_point)) + .collect(), + } + } + + fn slice(&self, span: &AzureDocumentIntelligenceSpan) -> &'a str { + let byte_offset = |code_point: usize| { + self.byte_offsets + .get(&code_point) + .copied() + .unwrap_or(self.text.len()) + }; + self.text + .get(byte_offset(span.offset)..byte_offset(span.end())) + .unwrap_or_default() + } +} + fn convert_dimensions( width: f64, height: f64, @@ -578,6 +746,16 @@ impl AzureDocumentIntelligenceOcrConfig { .features .iter() .map(|features| ("features", features.as_str())), + ) + .chain( + params + .output_content_format + .map(|format| (OUTPUT_CONTENT_FORMAT_PARAM, format.into())), + ) + .chain( + params + .string_index_type + .map(|index_type| ("stringIndexType", index_type)), ), ) .into_string() @@ -697,8 +875,12 @@ mod tests { } fn map(value: Value) -> Result { + map_for_model(value, "azure_ai/doc-intelligence/prebuilt-layout") + } + + fn map_for_model(value: Value, model: &str) -> Result { let arguments = serde_json::from_value(value).unwrap(); - AzureDocumentIntelligenceOcrConfig.map_ocr_params(&arguments, "model") + AzureDocumentIntelligenceOcrConfig.map_ocr_params(&arguments, model) } #[test] @@ -828,6 +1010,218 @@ mod tests { assert_eq!(map(json!({"features": []})).unwrap().features, None); } + #[rstest] + #[case::markdown( + json!({"outputContentFormat": "markdown"}), + Some(OutputContentFormat::Markdown), + Some("unicodeCodePoint") + )] + #[case::markdown_litellm_format( + json!({"outputContentFormat": "markdown", "req_format": "litellm"}), + Some(OutputContentFormat::Markdown), + Some("unicodeCodePoint") + )] + #[case::markdown_native_format( + json!({"outputContentFormat": "markdown", "req_format": "native"}), + Some(OutputContentFormat::Markdown), + None + )] + #[case::capitalized_markdown( + json!({"outputContentFormat": "Markdown"}), + Some(OutputContentFormat::Markdown), + Some("unicodeCodePoint") + )] + #[case::text(json!({"outputContentFormat": "text"}), Some(OutputContentFormat::Text), None)] + #[case::upper_case_text(json!({"outputContentFormat": "TEXT"}), Some(OutputContentFormat::Text), None)] + #[case::padded_markdown( + json!({"outputContentFormat": " markdown\n"}), + Some(OutputContentFormat::Markdown), + Some("unicodeCodePoint") + )] + #[case::snake_case_markdown( + json!({"output_content_format": "markdown"}), + Some(OutputContentFormat::Markdown), + Some("unicodeCodePoint") + )] + #[case::camel_case_wins_over_snake_case( + json!({"outputContentFormat": "text", "output_content_format": "markdown"}), + Some(OutputContentFormat::Text), + None + )] + #[case::empty_camel_case_falls_back_to_snake_case( + json!({"outputContentFormat": "", "output_content_format": "markdown"}), + Some(OutputContentFormat::Markdown), + Some("unicodeCodePoint") + )] + #[case::null_camel_case_falls_back_to_snake_case( + json!({"outputContentFormat": null, "output_content_format": "markdown"}), + Some(OutputContentFormat::Markdown), + Some("unicodeCodePoint") + )] + #[case::empty(json!({"outputContentFormat": ""}), None, None)] + #[case::blank(json!({"outputContentFormat": " "}), None, None)] + #[case::null(json!({"outputContentFormat": null}), None, None)] + #[case::omitted(json!({}), None, None)] + fn output_content_format_maps_to_query_fields( + #[case] options: Value, + #[case] format: Option, + #[case] string_index_type: Option<&str>, + ) { + let mapped = map(options).unwrap(); + + assert_eq!(mapped.output_content_format, format); + assert_eq!(mapped.string_index_type, string_index_type); + } + + #[rstest] + #[case::layout("azure_ai/doc-intelligence/prebuilt-layout", Some("unicodeCodePoint"))] + #[case::bare_layout("prebuilt-layout", Some("unicodeCodePoint"))] + #[case::read("azure_ai/doc-intelligence/prebuilt-read", None)] + #[case::invoice("azure_ai/doc-intelligence/prebuilt-invoice", None)] + #[case::custom("azure_ai/doc-intelligence/my-extraction-model", None)] + fn only_the_layout_model_asks_for_code_point_offsets( + #[case] model: &str, + #[case] string_index_type: Option<&str>, + ) { + let mapped = map_for_model(json!({"outputContentFormat": "markdown"}), model).unwrap(); + + assert_eq!( + mapped.output_content_format, + Some(OutputContentFormat::Markdown) + ); + assert_eq!(mapped.string_index_type, string_index_type); + } + + #[rstest] + #[case::unknown(json!({"outputContentFormat": "html"}))] + #[case::number(json!({"outputContentFormat": 1}))] + #[case::boolean(json!({"outputContentFormat": true}))] + #[case::list(json!({"outputContentFormat": ["markdown"]}))] + #[case::snake_case_unknown(json!({"output_content_format": "html"}))] + #[case::drop_params_off(json!({"outputContentFormat": "html", "drop_params": false}))] + fn invalid_output_content_format_is_a_request_error(#[case] options: Value) { + let error = map(options).unwrap_err(); + + assert!(matches!(error, Error::OutputContentFormat), "{error:?}"); + assert_eq!(error.http_status_code(), Some(400)); + } + + #[rstest] + #[case::unknown(json!({"outputContentFormat": "html", "drop_params": true, "pages": [0]}))] + #[case::number(json!({"outputContentFormat": 1, "drop_params": true, "pages": [0]}))] + #[case::snake_case_unknown( + json!({"output_content_format": "html", "drop_params": true, "pages": [0]}) + )] + fn drop_params_drops_an_invalid_output_content_format(#[case] options: Value) { + let mapped = map(options).unwrap(); + + assert_eq!(serde_json::to_value(mapped).unwrap(), json!({"pages": "1"})); + } + + #[rstest] + #[case::camel_case(json!({"outputContentFormat": "markdown", "drop_params": true}))] + #[case::snake_case(json!({"output_content_format": "markdown", "drop_params": true}))] + fn drop_params_keeps_a_valid_output_content_format(#[case] options: Value) { + let mapped = map(options).unwrap(); + + assert_eq!( + mapped.output_content_format, + Some(OutputContentFormat::Markdown) + ); + } + + fn markdown_pages(analyze_result: Value) -> Vec { + let raw = + serde_json::to_vec(&json!({"status": "succeeded", "analyzeResult": analyze_result})) + .unwrap(); + AzureDocumentIntelligenceOcrConfig + .transform_ocr_response("model", &raw, OcrResponseFormat::Litellm) + .unwrap() + .pages + .into_iter() + .map(|page| page.markdown) + .collect() + } + + #[rstest] + fn markdown_pages_are_code_point_slices_of_the_content() { + let content = "# Ünïcode क्ष\n\n\n\n## Två 🙂\n\n|a|\n\n> tail"; + let second_page_start = "# Ünïcode क्ष\n\n\n\n".chars().count(); + let pages = markdown_pages(json!({ + "content": content, + "contentFormat": "markdown", + "stringIndexType": "unicodeCodePoint", + "pages": [ + {"pageNumber": 1, "spans": [{"offset": 0, "length": second_page_start}], + "lines": [{"content": "Ünïcode क्ष"}]}, + {"pageNumber": 2, "spans": [ + {"offset": second_page_start, "length": 8}, + {"offset": second_page_start + 10, "length": 3} + ], "lines": [{"content": "Två"}]} + ] + })); + + assert_eq!( + pages, + ["# Ünïcode क्ष\n\n\n\n", "## Två 🙂|a|"] + ); + } + + #[rstest] + fn spans_past_the_content_end_are_clamped() { + let pages = markdown_pages(json!({ + "content": "ab", + "contentFormat": "markdown", + "stringIndexType": "unicodeCodePoint", + "pages": [{"spans": [{"offset": 1, "length": 9}, {"offset": 7, "length": 1}]}] + })); + + assert_eq!(pages, ["b"]); + } + + #[rstest] + #[case::text_content(json!("text"), json!("unicodeCodePoint"))] + #[case::grapheme_offsets(json!("markdown"), json!("textElements"))] + #[case::no_index_type(json!("markdown"), Value::Null)] + #[case::no_content_format(Value::Null, json!("unicodeCodePoint"))] + fn other_responses_keep_line_joined_pages( + #[case] content_format: Value, + #[case] string_index_type: Value, + ) { + let pages = markdown_pages(json!({ + "content": "# A\n\nB", + "contentFormat": content_format, + "stringIndexType": string_index_type, + "pages": [{"spans": [{"offset": 0, "length": 6}], + "lines": [{"content": "A"}, {"content": "B"}]}] + })); + + assert_eq!(pages, ["A\nB"]); + } + + #[rstest] + #[case::null_spans(json!("markdown"), json!(null))] + #[case::fractional_offset(json!("markdown"), json!([{"offset": 0.5, "length": 1}]))] + #[case::missing_length(json!("markdown"), json!([{"offset": 0}]))] + #[case::negative_offset(json!("markdown"), json!([{"offset": -1, "length": 1}]))] + #[case::text_content(json!("text"), json!([{"offset": "0", "length": 1}]))] + fn unusable_page_spans_keep_line_joined_pages( + #[case] content_format: Value, + #[case] broken_spans: Value, + ) { + let pages = markdown_pages(json!({ + "content": "# A\n\nB", + "contentFormat": content_format, + "stringIndexType": "unicodeCodePoint", + "pages": [ + {"spans": [{"offset": 0, "length": 5}], "lines": [{"content": "A"}]}, + {"spans": broken_spans, "lines": [{"content": "B"}]} + ] + })); + + assert_eq!(pages, ["A", "B"]); + } + #[tokio::test] async fn request_endpoint_cannot_receive_environment_key() { let connection = OcrConnection { diff --git a/litellm-rust/crates/llms/src/base_llm/messages/transformation.rs b/litellm-rust/crates/llms/src/base_llm/messages/transformation.rs index 8a88af40d6b..744ce89afdb 100644 --- a/litellm-rust/crates/llms/src/base_llm/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/base_llm/messages/transformation.rs @@ -77,6 +77,15 @@ pub trait BaseMessagesConfig: Sync { headers } + fn request_body( + &self, + request: MessagesRequest, + _headers: &Headers, + _drop_params: bool, + ) -> Result { + Ok(request) + } + /// The JSON that goes on the wire, for a host whose body differs from the typed request: /// Python's configs `pop("model")` when the model is addressed by the URL. fn wire_body(&self, body: Value) -> Value { diff --git a/litellm-rust/crates/llms/src/base_llm/ocr/error.rs b/litellm-rust/crates/llms/src/base_llm/ocr/error.rs index d51fe996dc4..e5f1c3339cf 100644 --- a/litellm-rust/crates/llms/src/base_llm/ocr/error.rs +++ b/litellm-rust/crates/llms/src/base_llm/ocr/error.rs @@ -59,6 +59,10 @@ pub enum Error { Pages(PagesFault), #[error("invalid OCR features")] Features, + #[error( + "invalid OCR outputContentFormat, expected text or markdown. To drop unsupported params, set `litellm.drop_params = True`." + )] + OutputContentFormat, #[error("OCR model cannot be a dot segment")] DotModel, #[error("OCR response exceeds the size limit of {limit} bytes")] @@ -200,6 +204,7 @@ impl Error { | Self::InlineDocumentTooLarge | Self::Pages(_) | Self::Features + | Self::OutputContentFormat | Self::DotModel | Self::TooLarge { .. } | Self::ResponseField { .. } @@ -242,6 +247,7 @@ impl Error { | Self::InlineDocumentTooLarge | Self::Pages(_) | Self::Features + | Self::OutputContentFormat | Self::DotModel | Self::TooLarge { .. } | Self::ResponseField { .. } @@ -291,6 +297,7 @@ impl Error { | Self::InlineDocumentTooLarge | Self::Pages(_) | Self::Features + | Self::OutputContentFormat | Self::DotModel | Self::InvalidProvider(_) | Self::InvalidModel { .. } @@ -345,6 +352,7 @@ impl Error { | Self::InlineDocumentTooLarge | Self::Pages(_) | Self::Features + | Self::OutputContentFormat | Self::DotModel | Self::InvalidProvider(_) | Self::InvalidModel { .. } diff --git a/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs b/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs index 19dd8e24226..c90fd99809e 100644 --- a/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs @@ -21,6 +21,7 @@ pub const OCR_RESPONSE_MAX_BYTES: usize = 64 * 1024 * 1024; pub const OCR_INLINE_MAX_BYTES: usize = 50 * 1024 * 1024; pub const OCR_MAX_FETCH_REDIRECTS: usize = 10; pub const OCR_POLL_RETRY_SECS: u64 = 2; +pub const DROP_PARAMS_PARAM: &str = "drop_params"; #[derive(Clone, Default)] pub struct OcrCredentialInputs { @@ -185,6 +186,10 @@ pub fn response_format(optional_params: &CallArguments) -> Result bool { + optional_params.get(DROP_PARAMS_PARAM) == Some(&Value::Bool(true)) +} + #[derive(Debug)] pub struct DecodedOcrResponse { pub data: T, diff --git a/litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs b/litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs index 0f8703f223b..1c976c1f73d 100644 --- a/litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs +++ b/litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs @@ -6,8 +6,8 @@ use litellm_auth_aws::{ }; use litellm_core_utils::core_helpers::json_type_name; use litellm_llms_types::{ - formats::audio_transcription::AudioTranscriptionResponseData, - providers::bedrock::ConverseResponse, + formats::audio_transcription::{AudioTranscriptionResponseData, AudioTranscriptionUsage}, + providers::bedrock::{ConverseResponse, ConverseUsage}, }; use serde::Deserialize; use serde_json::{Map, Value, json}; @@ -101,10 +101,10 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig { if let Some(prompt) = optional_string(&optional_params, "prompt") { instruction.push_str(&format!(" Additional context: {prompt}")); } - let mut inference_config = Map::from_iter([("maxTokens".to_string(), json!(4096))]); - if let Some(temperature) = optional_params.get("temperature") { - inference_config.insert("temperature".to_string(), temperature.clone()); - } + let temperature = optional_params + .get("temperature") + .cloned() + .unwrap_or(json!(0)); Ok(AudioTranscriptionRequestData { body: json!({ "messages": [{ @@ -114,8 +114,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig { {"text": instruction} ] }], - "system": [{"text": "You are a transcription assistant."}], - "inferenceConfig": inference_config, + "inferenceConfig": {"maxTokens": 4096, "temperature": temperature}, }), }) } @@ -125,6 +124,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig { _model: &str, response_json: Value, ) -> Result { + let reports_usage = response_json.get("usage").is_some(); let response: ConverseResponse = serde_json::from_value(response_json).map_err(|error| { Error::InvalidResponse(ErrorDetail::invalid("Converse response", error)) @@ -137,6 +137,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig { } Ok(AudioTranscriptionResponseData { text: response.content_text(), + usage: reports_usage.then(|| transcription_usage(&response.usage)), }) } @@ -191,22 +192,41 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig { } } +fn transcription_usage(usage: &ConverseUsage) -> AudioTranscriptionUsage { + AudioTranscriptionUsage { + input_tokens: usage.input_tokens, + output_tokens: usage.output_tokens, + total_tokens: usage + .total_tokens + .unwrap_or(usage.input_tokens + usage.output_tokens), + } +} + #[cfg(test)] mod tests { use super::*; + use rstest::rstest; fn no_env(_: &str) -> Option { None } - #[test] - fn request_matches_python_shape() { - let params = Map::from_iter([ - ("language".to_string(), json!("en")), - ("prompt".to_string(), json!("Speaker names")), - ("temperature".to_string(), json!(0)), - ("timestamp_granularities".to_string(), json!(["word"])), - ]); + #[rstest] + #[case::caller_temperature(Some(json!(0.4)), json!(0.4))] + #[case::default_temperature(None, json!(0))] + fn request_sends_audio_and_instruction_without_system_block( + #[case] temperature: Option, + #[case] expected_temperature: Value, + ) { + let params = Map::from_iter( + [ + ("language".to_string(), json!("en")), + ("prompt".to_string(), json!("Speaker names")), + ("timestamp_granularities".to_string(), json!(["word"])), + ] + .into_iter() + .chain(temperature.map(|value| ("temperature".to_string(), value))), + ); let params = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG.map_transcription_params(¶ms); let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG .transform_audio_transcription_request( @@ -225,22 +245,39 @@ mod tests { {"text": "Transcribe the audio. Respond with only the transcript. The audio language is en. Additional context: Speaker names"} ] }], - "system": [{"text": "You are a transcription assistant."}], - "inferenceConfig": {"maxTokens": 4096, "temperature": 0} + "inferenceConfig": {"maxTokens": 4096, "temperature": expected_temperature} }) ); } - #[test] - fn response_concatenates_content_blocks() { + #[rstest] + #[case::reported_total( + json!({"inputTokens": 390, "outputTokens": 60, "totalTokens": 450}), + json!({"text": "hello world", "usage": {"type": "tokens", "input_tokens": 390, "output_tokens": 60, "total_tokens": 450}}) + )] + #[case::derived_total( + json!({"inputTokens": 1, "outputTokens": 2}), + json!({"text": "hello world", "usage": {"type": "tokens", "input_tokens": 1, "output_tokens": 2, "total_tokens": 3}}) + )] + fn response_carries_text_and_converse_usage(#[case] usage: Value, #[case] expected: Value) { let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG .transform_audio_transcription_response( "model", - json!({"output": {"message": {"content": [{"text": "hello "}, {"text": "world"}]}}, "usage": {"inputTokens": 1, "outputTokens": 2}}), + json!({"output": {"message": {"content": [{"text": "hello "}, {"text": "world"}]}}, "usage": usage}), ) .expect("response"); - assert_eq!(result.text, "hello world"); - assert_eq!(result.into_json(), json!({"text": "hello world"})); + assert_eq!(result.into_json(), expected); + } + + #[rstest] + fn response_without_usage_reports_none() { + let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG + .transform_audio_transcription_response( + "model", + json!({"output": {"message": {"content": [{"text": "hello"}]}}}), + ) + .expect("response"); + assert_eq!(result.into_json(), json!({"text": "hello"})); } #[test] diff --git a/litellm-rust/crates/llms/src/vertex_ai/messages/AGENTS.md b/litellm-rust/crates/llms/src/vertex_ai/messages/AGENTS.md index b5d637d6bc7..d95da65ac9d 100644 --- a/litellm-rust/crates/llms/src/vertex_ai/messages/AGENTS.md +++ b/litellm-rust/crates/llms/src/vertex_ai/messages/AGENTS.md @@ -1,6 +1,7 @@ # rules -- Claude on Vertex AI: reuses `anthropic/messages` shaping, moves the model into the publisher URL, puts the Vertex `anthropic_version` in the body, and authenticates with a Google access token +- Claude on Vertex AI: reuses `anthropic/messages` shaping, moves the model into the publisher URL, and authenticates with a Google access token +- Vertex reads the API version only from the body's `anthropic_version` and answers 400 when an `anthropic-version` header names a different one, so the caller's header never reaches Vertex. A version Vertex accepts in the body is carried there, `2023-06-01`, `2023-01-01`, or no header get `vertex-2023-10-16`, and any other version drops under `drop_params` and is refused without it - Applies Vertex's beta policy and strips `output_config.effort` for models that do not accept it # references diff --git a/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs b/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs index 2e77dbc1ca8..46e5a68dcb8 100644 --- a/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs @@ -3,10 +3,10 @@ use litellm_auth_gcp::{ VertexConfig, get_vertex_ai_location, get_vertex_ai_project, get_vertex_ai_project_from_credentials, secret_names, }; -use litellm_http::request::has_header; +use litellm_http::request::{has_header, header_value, without_headers}; use litellm_llms_types::{ formats::messages::{MessagesOptionalParams, MessagesRequest, MessagesTool, OutputConfig}, - providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet}, + providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet, VERSION_HEADER}, recognized::Recognized, }; use litellm_router_types::LitellmParams; @@ -36,6 +36,8 @@ use litellm_llms_types::providers::vertex_ai::{ }; const WEB_SEARCH_TOOL_PREFIX: &str = "web_search"; +const VERSIONS_SERVED_AS_VERTEX_DEFAULT: [&str; 3] = ["", "2023-01-01", "2023-06-01"]; +const VERSIONS_VERTEX_ACCEPTS_IN_BODY: [&str; 2] = [ANTHROPIC_VERSION, "2026-10-21"]; /// Claude on Vertex AI, Python's `VertexAIPartnerModelsAnthropicMessagesConfig`: the Anthropic /// payload addressed to a project and location, authenticated with a Google access token. @@ -90,12 +92,6 @@ impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig { .params .output_config .and_then(|config| sanitize_output_config(config, accepts_effort)), - extra: request - .params - .extra - .into_iter() - .chain([("anthropic_version".to_string(), json!(ANTHROPIC_VERSION))]) - .collect(), ..request.params }, ..request @@ -133,7 +129,35 @@ impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig { } fn request_headers(&self, headers: Headers, request: &MessagesRequest) -> Headers { - merge_beta_headers(headers, vertex_feature_betas(request)) + merge_beta_headers( + without_headers(headers, &[VERSION_HEADER]), + vertex_feature_betas(request), + ) + } + + fn request_body( + &self, + request: MessagesRequest, + headers: &Headers, + drop_params: bool, + ) -> Result { + let version = vertex_version( + &request.model, + header_value(headers, VERSION_HEADER), + drop_params, + )?; + Ok(MessagesRequest { + params: MessagesOptionalParams { + extra: request + .params + .extra + .into_iter() + .chain([("anthropic_version".to_string(), json!(version))]) + .collect(), + ..request.params + }, + ..request + }) } /// Vertex addresses the model in the URL and rejects it in the body. @@ -225,6 +249,26 @@ fn vertex_feature_betas(request: &MessagesRequest) -> BetaSet { ) } +fn vertex_version<'a>( + model: &str, + requested: Option<&'a str>, + drop_params: bool, +) -> Result<&'a str, Error> { + match requested.map(str::trim).unwrap_or_default() { + version if VERSIONS_SERVED_AS_VERTEX_DEFAULT.contains(&version) => Ok(ANTHROPIC_VERSION), + version if VERSIONS_VERTEX_ACCEPTS_IN_BODY.contains(&version) => Ok(version), + _ if drop_params => Ok(ANTHROPIC_VERSION), + version => Err(Error::InvalidRequest( + crate::ErrorDetail::UnsupportedParameter { + model: model.into(), + param: VERSION_HEADER.into(), + value: version.into(), + hint: String::new(), + }, + )), + } +} + #[cfg(test)] mod tests { use rstest::rstest; @@ -471,6 +515,28 @@ mod tests { ); } + #[rstest] + #[case::sdk_spelling("anthropic-version")] + #[case::any_case("Anthropic-Version")] + fn a_forwarded_anthropic_version_never_reaches_vertex(#[case] name: &str) { + let request: MessagesRequest = serde_json::from_value(json!({ + "model": "claude-sonnet-4-5", + "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}], + })) + .unwrap(); + assert_eq!( + VERTEX_ANTHROPIC_MESSAGES_CONFIG.request_headers( + vec![ + (name.to_string(), "2026-10-21".to_string()), + ("x-request-id".to_string(), "req-1".to_string()), + ], + &request, + ), + vec![("x-request-id".to_string(), "req-1".to_string())] + ); + } + fn transformed(value: Value, context: &MessagesTransformContext) -> Value { let request: MessagesRequest = serde_json::from_value(value).unwrap(); serde_json::to_value( @@ -482,16 +548,87 @@ mod tests { } #[rstest] - fn the_body_carries_the_vertex_version_and_the_wire_drops_the_model() { - let body = transformed( - json!({ - "model": "claude-sonnet-4-5@20250929", - "max_tokens": 16, - "messages": [{"role": "user", "content": "hi"}] - }), - &MessagesTransformContext::default(), + #[case::sdk_2(&[("anthropic-version", "2026-10-21")], false, "2026-10-21")] + #[case::any_case(&[("Anthropic-Version", "2026-10-21")], false, "2026-10-21")] + #[case::vertex_name(&[("anthropic-version", "vertex-2023-10-16")], false, "vertex-2023-10-16")] + #[case::sdk_1(&[("anthropic-version", "2023-06-01")], false, "vertex-2023-10-16")] + #[case::first_version(&[("anthropic-version", "2023-01-01")], false, "vertex-2023-10-16")] + #[case::blank(&[("anthropic-version", " ")], false, "vertex-2023-10-16")] + #[case::no_header(&[], false, "vertex-2023-10-16")] + #[case::unknown_under_drop_params(&[("anthropic-version", "latest")], true, "vertex-2023-10-16")] + fn the_requested_version_goes_in_the_body( + #[case] forwarded: &[(&str, &str)], + #[case] drop_params: bool, + #[case] body_version: &str, + ) { + let request: MessagesRequest = serde_json::from_value(json!({ + "model": "claude-sonnet-4-5", + "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}], + })) + .unwrap(); + let headers: Headers = forwarded + .iter() + .map(|(name, value)| (name.to_string(), value.to_string())) + .collect(); + let body = serde_json::to_value( + VERTEX_ANTHROPIC_MESSAGES_CONFIG + .request_body(request, &headers, drop_params) + .unwrap(), + ) + .unwrap(); + assert_eq!(body["anthropic_version"], json!(body_version)); + } + + #[rstest] + #[case::unreleased("2025-01-01")] + #[case::alias("latest")] + fn an_unsupported_version_is_refused_without_drop_params(#[case] requested: &str) { + let request: MessagesRequest = serde_json::from_value(json!({ + "model": "claude-sonnet-4-5", + "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}], + })) + .unwrap(); + let headers: Headers = vec![("anthropic-version".to_string(), requested.to_string())]; + let Error::InvalidRequest(detail) = VERTEX_ANTHROPIC_MESSAGES_CONFIG + .request_body(request, &headers, false) + .unwrap_err() + else { + panic!("expected an invalid request"); + }; + assert_eq!( + detail.to_string(), + format!( + "claude-sonnet-4-5 does not support anthropic-version={requested}. \ + To drop unsupported params, set `litellm.drop_params = True`." + ) ); - assert_eq!(body["anthropic_version"], json!("vertex-2023-10-16")); + } + + #[rstest] + fn the_body_carries_the_vertex_version_and_the_wire_drops_the_model() { + let request: MessagesRequest = serde_json::from_value(json!({ + "model": "claude-sonnet-4-5@20250929", + "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}] + })) + .unwrap(); + let body = serde_json::to_value( + VERTEX_ANTHROPIC_MESSAGES_CONFIG + .request_body( + VERTEX_ANTHROPIC_MESSAGES_CONFIG + .transform_anthropic_messages_request( + request, + &MessagesTransformContext::default(), + ) + .unwrap(), + &Vec::new(), + false, + ) + .unwrap(), + ) + .unwrap(); assert_eq!(body["model"], json!("claude-sonnet-4-5@20250929")); let wire = VERTEX_ANTHROPIC_MESSAGES_CONFIG.wire_body(body); assert_eq!(wire.get("model"), None); diff --git a/litellm-rust/crates/model-catalog-refresh/Cargo.toml b/litellm-rust/crates/model-catalog-refresh/Cargo.toml new file mode 100644 index 00000000000..27219042c6c --- /dev/null +++ b/litellm-rust/crates/model-catalog-refresh/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "litellm-model-catalog-refresh" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +litellm-http.workspace = true +litellm-model-catalog.workspace = true +thiserror.workspace = true +url.workspace = true + +[dev-dependencies] +litellm-error.workspace = true +litellm-http = { workspace = true, features = ["test-support"] } +reqwest.workspace = true +rstest.workspace = true +tokio.workspace = true diff --git a/litellm-rust/crates/model-catalog-refresh/src/error.rs b/litellm-rust/crates/model-catalog-refresh/src/error.rs new file mode 100644 index 00000000000..656f32370cb --- /dev/null +++ b/litellm-rust/crates/model-catalog-refresh/src/error.rs @@ -0,0 +1,105 @@ +use std::io; + +use thiserror::Error; + +#[derive(Debug, Error)] +#[error(transparent)] +pub struct Error(Box); + +#[derive(Debug, Error)] +enum Repr { + #[error(transparent)] + Http(#[from] litellm_http::Error), + #[error("could not read the catalog")] + Read(#[source] io::Error), + #[error("invalid catalog URL")] + InvalidUrl(#[source] url::ParseError), + #[error("unsupported catalog URL scheme {scheme}")] + UnsupportedScheme { scheme: String }, +} + +impl Error { + pub fn unsupported_scheme(scheme: String) -> Self { + Self(Box::new(Repr::UnsupportedScheme { scheme })) + } +} + +impl From for Error { + fn from(cause: litellm_http::Error) -> Self { + Self(Box::new(Repr::Http(cause))) + } +} + +impl From for Error { + fn from(cause: io::Error) -> Self { + Self(Box::new(Repr::Read(cause))) + } +} + +impl From for Error { + fn from(cause: url::ParseError) -> Self { + Self(Box::new(Repr::InvalidUrl(cause))) + } +} + +#[cfg(test)] +mod tests { + use litellm_error::{Report, find}; + use rstest::rstest; + + use super::*; + use crate::{DoneReason, Reply, Step}; + + #[rstest] + fn exhausted_transport_keeps_its_typed_cause() { + let reply = Reply::Transport { + error: io::Error::from(io::ErrorKind::ConnectionReset).into(), + }; + let Reply::Transport { error } = reply else { + panic!("transport failure") + }; + let step = Step::Done { + reason: DoneReason::Exhausted(error), + }; + let Step::Done { + reason: DoneReason::Exhausted(error), + } = step + else { + panic!("exhausted attempts") + }; + let cause = find::(&error).expect("typed I/O cause"); + assert_eq!(cause.kind(), io::ErrorKind::ConnectionReset); + assert_eq!( + Report::new(&error) + .to_string() + .matches(&cause.to_string()) + .count(), + 1 + ); + assert!(!error.to_string().contains(&cause.to_string())); + } + + #[rstest] + fn invalid_source_keeps_its_typed_cause() { + let error = Error::from(url::Url::parse("http://[invalid").unwrap_err()); + assert!(find::(&error).is_some()); + } + + #[rstest] + #[tokio::test] + async fn http_failures_keep_the_cause_without_the_url() { + let cause = litellm_http::Client::for_test() + .get("http://localhost:invalid/private?api_key=secret") + .send() + .await + .unwrap_err(); + let error = Error::from(litellm_http::Error::from(cause)); + assert!(find::(&error).is_some()); + let report = Report::new(&error).to_string(); + assert!(!report.contains("secret"), "{report}"); + assert!(!report.contains("private"), "{report}"); + let debug = format!("{error:?}"); + assert!(!debug.contains("secret"), "{debug}"); + assert!(!debug.contains("private"), "{debug}"); + } +} diff --git a/litellm-rust/crates/model-catalog-refresh/src/lib.rs b/litellm-rust/crates/model-catalog-refresh/src/lib.rs new file mode 100644 index 00000000000..006c4887fad --- /dev/null +++ b/litellm-rust/crates/model-catalog-refresh/src/lib.rs @@ -0,0 +1,9 @@ +mod error; +mod policy; +mod source; +mod step; + +pub use error::Error; +pub use policy::*; +pub use source::*; +pub use step::*; diff --git a/litellm-rust/crates/model-catalog-refresh/src/policy.rs b/litellm-rust/crates/model-catalog-refresh/src/policy.rs new file mode 100644 index 00000000000..bc32e40408e --- /dev/null +++ b/litellm-rust/crates/model-catalog-refresh/src/policy.rs @@ -0,0 +1,20 @@ +use std::time::Duration; + +use litellm_model_catalog::IntegrityLimits; + +/// Main's defaults, 3 attempts, 30 s max wait, 5 s timeout, backoff base 2.0, jitter 0.0 to 1.0 s +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct RetryPolicy { + pub max_attempts: u32, + pub max_wait: Duration, + pub timeout: Duration, + pub backoff_base_seconds: f64, + pub jitter_min_seconds: f64, + pub jitter_max_seconds: f64, + /// Main's 50 model floor and 0.5 shrink ratio + pub limits: IntegrityLimits, +} + +/// Main's default `LITELLM_MODEL_COST_MAP_URL` +pub const DEFAULT_REGISTRY_URL: &str = + "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json"; diff --git a/litellm-rust/crates/model-catalog-refresh/src/source.rs b/litellm-rust/crates/model-catalog-refresh/src/source.rs new file mode 100644 index 00000000000..713e0386941 --- /dev/null +++ b/litellm-rust/crates/model-catalog-refresh/src/source.rs @@ -0,0 +1,17 @@ +use std::path::PathBuf; + +use url::Url; + +/// Where a fetched catalog comes from +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Source { + Http(Url), + File(PathBuf), + S3(S3Object), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct S3Object { + pub bucket: String, + pub key: String, +} diff --git a/litellm-rust/crates/model-catalog-refresh/src/step.rs b/litellm-rust/crates/model-catalog-refresh/src/step.rs new file mode 100644 index 00000000000..822d1807916 --- /dev/null +++ b/litellm-rust/crates/model-catalog-refresh/src/step.rs @@ -0,0 +1,66 @@ +use std::time::Duration; + +use litellm_model_catalog::Catalog; + +use crate::Error; +use crate::source::Source; + +/// One attempt. Where to fetch from is the `Source`, so the request carries no URL +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FetchRequest { + pub if_none_match: Option, + pub timeout: Duration, +} + +#[derive(Debug)] +pub enum Reply { + /// An HTTP, S3 or file answer. The body is read only on 200 + Response { + status: u16, + etag: Option, + retry_after: Option, + body: Option>, + }, + /// No answer reached us + Transport { error: Error }, +} + +/// What the fetch state machine asks its driver to do next +#[derive(Debug)] +pub enum Step { + Fetch(FetchRequest), + Sleep(Duration), + /// A parsed document to install, with its rejected row errors + Adopt { + catalog: Box, + installed: Installed, + errors: Vec, + }, + /// 304, a final status, or retries exhausted + Done { + reason: DoneReason, + }, +} + +#[derive(Debug)] +pub enum DoneReason { + NotModified, + Status(u16), + Exhausted(Error), +} + +/// What is installed now, the input to the next revalidation +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Installed { + pub origin: Origin, + pub etag: Option, + /// `git_blob_id` of the installed bytes + pub revision: String, + pub model_count: usize, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Origin { + Bundled, + Remote { source: Source }, +} diff --git a/litellm-rust/crates/model-catalog/AGENTS.md b/litellm-rust/crates/model-catalog/AGENTS.md index 0a380788c19..fc4b28d4ef0 100644 --- a/litellm-rust/crates/model-catalog/AGENTS.md +++ b/litellm-rust/crates/model-catalog/AGENTS.md @@ -1,5 +1,27 @@ -## Validation +## Layout -For `model_prices_and_context_window.json` validation, we should eventually: +`wire/` reads one entry as written. `EntryWire::read` makes one pass over the entry's keys and puts every non-null key in exactly one place, which is an attribute, a flat price parsed by `FlatKey::parse`, a price block, a `supports_*` capability or `unknown`. A known key with a bad value becomes an `Error` that names the key, and the rest of the entry is still read -- Stop skipping this crate's tests +`lower.rs` is the only code that knows both the wire types and `litellm-cost`. It lowers an `EntryWire` into `Prices`, `Adjustments` and `Model` and checks the rules that span keys, such as one price per meter and condition, contiguous tiers, no suffix inside a block, and off-peak needing hours and prices. Lowering only goes one way. Nothing turns `Prices` back into JSON, since the raw fields are kept + +`catalog.rs` reads the document, indexes aliases and runs the checks across entries. A bad entry keeps its raw fields and its errors are listed in `Loaded.errors`. Deciding what to do with them is the caller's job. `Catalog::written` gives back every top-level key and value as the document wrote them, in document order. `Catalog::served` gives back every model and alias as Python serves it, without `sample_spec` and `fallback_generalizations` + +`Billing` on `Prices` holds the row attributes pricing reads (text unit, band edge, cache-read fallback, wall clock, web search unit, tier scope). `lower.rs` derives it from the row and its `litellm_provider`, so `litellm-cost` holds no provider list + +`store/` is the catalog's in-memory store and the only place that indexes model names. Its rules are in [src/store/AGENTS.md](src/store/AGENTS.md) + +`layers.rs`, `resolve/`, `model_info.rs` and `register.rs` hold types only. The functions over them land with their first caller + +Catalog errors use one opaque `Error` with typed causes and JSON Pointer accessors (`path`, `key`, `field`). `Error::rejects` alone decides whether an error fails the document, rejects its row or neither, and `ErrorKind` is the stable name hosts report it by + +## Prices + +The price vocabulary lives in `litellm-cost`, and only this crate knows JSON key names. Every price is keyed by `Meter`, `Channel` and `When`. A variation only one meter has stays in the meter (cache TTL, image size, resolution, `_above_Ns_interval`, `max_results_range`). Service tier and prompt band make up `When`. Batch and decisions are a `Channel` because Python bills them with their own rules. Off-peak is an overlay on `Prices`, and regional uplifts are `Adjustments`, never prices + +A new `N`, image size or resolution in an existing key shape parses with no Rust change. A new unit, token kind, feature or tier needs a variant in `litellm-cost` plus its spelling in `wire/flat_key.rs` and `FLAT_KEY_PATTERN` in `schema.rs` + +`Prices::band` and `Prices::rate` are pinned to Python's `_get_token_base_cost` by `tests/price_lookup_reference.json`. After a billing change in Python, regenerate it from the repo root with `PYTHONPATH=. python litellm-rust/crates/model-catalog/tests/generate_price_lookup_reference.py` + +## Tests + +`BUNDLED_REGISTRY` embeds the root `model_prices_and_context_window.json`, so the tests over it fail a JSON change with a bad value or a key no type reads diff --git a/litellm-rust/crates/model-catalog/Cargo.toml b/litellm-rust/crates/model-catalog/Cargo.toml index 2c2866d7090..45d16515dcd 100644 --- a/litellm-rust/crates/model-catalog/Cargo.toml +++ b/litellm-rust/crates/model-catalog/Cargo.toml @@ -6,15 +6,19 @@ license.workspace = true repository.workspace = true [features] -schema = ["dep:schemars", "litellm-llms-types/schema"] +schema = ["dep:schemars", "litellm-cost/schema"] [dependencies] -litellm-llms-types.workspace = true +litellm-cost = { workspace = true, features = ["serde"] } indexmap = { version = "2.14.0", features = ["serde"] } +jiff = { version = "0.2.37", default-features = false, features = ["std", "tzdb-bundle-always"] } +macro_rules_attribute.workspace = true schemars = { workspace = true, optional = true } serde.workspace = true -serde_json.workspace = true +serde_json = { workspace = true, features = ["preserve_order"] } +serde_with.workspace = true +strum.workspace = true thiserror.workspace = true time.workspace = true diff --git a/litellm-rust/crates/model-catalog/src/bundled.rs b/litellm-rust/crates/model-catalog/src/bundled.rs new file mode 100644 index 00000000000..d71a2d57ebb --- /dev/null +++ b/litellm-rust/crates/model-catalog/src/bundled.rs @@ -0,0 +1,3 @@ +/// The registry document this build serves when no other one is loaded. +pub const BUNDLED_REGISTRY: &[u8] = + include_bytes!("../../../../model_prices_and_context_window.json"); diff --git a/litellm-rust/crates/model-catalog/src/capabilities.rs b/litellm-rust/crates/model-catalog/src/capabilities.rs deleted file mode 100644 index 30724766aeb..00000000000 --- a/litellm-rust/crates/model-catalog/src/capabilities.rs +++ /dev/null @@ -1,67 +0,0 @@ -use serde::{Deserialize, Serialize}; - -/// Primary API surface / task type of the model. -#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(rename_all = "snake_case")] -pub enum Mode { - AudioSpeech, - AudioTranscription, - Chat, - Completion, - Decisions, - Embedding, - Evaluation, - Guardrail, - ImageEdit, - ImageGeneration, - Moderation, - Ocr, - Realtime, - Rerank, - Responses, - Search, - VectorStore, - VideoGeneration, -} - -/// Gemini audio generation API the model is served through. -#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(rename_all = "snake_case")] -pub enum VertexAiAudioApi { - LyriaPredict, - LyriaInteractions, -} - -/// Audio container format the model can return. -#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(rename_all = "snake_case")] -pub enum AudioFormat { - Mp3, - Wav, -} - -/// Input modality the model accepts. -#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(rename_all = "snake_case")] -pub enum InputModality { - Text, - Image, - Audio, - Video, -} - -/// Output modality the model can produce. -#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(rename_all = "snake_case")] -pub enum OutputModality { - Text, - Image, - Audio, - Video, - Code, -} diff --git a/litellm-rust/crates/model-catalog/src/catalog.rs b/litellm-rust/crates/model-catalog/src/catalog.rs index 37412ef1e14..ac5f4ffcf25 100644 --- a/litellm-rust/crates/model-catalog/src/catalog.rs +++ b/litellm-rust/crates/model-catalog/src/catalog.rs @@ -1,54 +1,29 @@ -use crate::error::Error; -use crate::fallback::{FallbackGeneralizations, FallbackRule}; -use crate::model_info::ModelInfo; +use std::borrow::Cow; +use std::collections::HashSet; + use indexmap::IndexMap; use serde::Deserialize; -use serde_json::{Map, Value}; -use std::collections::HashMap; +use serde_json::Value; -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct Provenance { - pub source: Option, - pub revision: Option, - pub etag: Option, -} +use crate::error::{Error, Loaded, Rejects, Repr}; +use crate::fallback::{FallbackGeneralizations, FallbackRule}; +use crate::model::Model; +use crate::store::{Rows, Store}; +use crate::wire::{EntryWire, Fields}; + +const SAMPLE_SPEC: &str = "sample_spec"; +const FALLBACK_GENERALIZATIONS: &str = "fallback_generalizations"; #[derive(Clone, Copy, Debug, PartialEq)] pub struct IntegrityLimits { - pub reference_model_count: usize, - pub min_model_count: usize, - pub min_reference_ratio: f64, -} - -#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] -pub enum AliasIssue { - #[error("model {model:?} has aliases that are not an array")] - InvalidList { model: String }, - #[error("model {model:?} has an alias that is not a string")] - InvalidName { model: String }, - #[error("alias {alias:?} of model {model:?} is already a model name")] - CanonicalCollision { model: String, alias: String }, - #[error("alias {alias:?} of model {model:?} is already an alias of another model")] - AliasCollision { model: String, alias: String }, + min_model_count: usize, + min_reference_ratio: f64, } #[derive(Clone, Debug)] pub struct ModelEntry { - fields: Map, - info: ModelInfo, -} - -impl ModelEntry { - pub fn field(&self, name: &str) -> Option<&Value> { - self.fields.get(name) - } - pub fn fields(&self) -> &Map { - &self.fields - } - /// The entry deserialized into the typed mirror of the catalog schema. - pub fn info(&self) -> &ModelInfo { - &self.info - } + fields: Fields, + model: Option, } #[derive(Clone, Copy, Debug)] @@ -58,142 +33,312 @@ pub struct ModelMatch<'a> { pub entry: &'a ModelEntry, } -#[derive(Debug)] +#[derive(Clone, Debug)] +struct Section { + index: usize, + key: String, + value: Value, +} + +/// One fetched key kept off the served map because its row failed validation +#[derive(Clone, Debug)] +pub struct Rejection<'a> { + pub key: String, + pub errors: Vec<&'a Error>, + /// The bundled row for the key is served instead. `false` means the key is left out + pub replaced_by_bundled: bool, +} + +#[derive(Clone, Copy, Debug, serde::Serialize)] +#[serde(untagged)] +pub enum Written<'a> { + Entry(&'a Fields), + Section(&'a Value), +} + +#[derive(Clone, Debug)] pub struct Catalog { + store: Store, + sections: Vec
, + fallback_generalizations: Option, +} + +/// Models and aliases, an alias served as its owner's row +#[derive(Clone, Debug)] +struct CatalogRows { entries: IndexMap, aliases: IndexMap, - lowercase_keys: HashMap, - sample_spec: Option, - fallback_generalizations: Option, - provenance: Provenance, - alias_issues: Vec, +} + +impl CatalogRows { + fn owner<'a>(&'a self, key: &'a str) -> &'a str { + self.aliases.get(key).map_or(key, String::as_str) + } + + fn fields(&self, key: &str) -> Option<&Fields> { + Some(self.entries.get(self.owner(key))?.fields()) + } +} + +/// Model names match ignoring case, and keys are grouped by their raw `litellm_provider` +impl Rows for CatalogRows { + type Row = Fields; + type Stamp = (); + + fn fold(name: &str) -> String { + name.to_lowercase() + } + + fn stamp(&self) -> Self::Stamp {} + + fn contains(&self, key: &str) -> bool { + self.entries.contains_key(key) || self.aliases.contains_key(key) + } + + fn row(&self, key: &str) -> Option> { + self.fields(key).map(Cow::Borrowed) + } + + fn scan(&self, visit: &mut dyn FnMut(&str, Option<&str>)) { + let keys = self.entries.keys().chain(self.aliases.keys()); + for key in keys { + let provider = self + .fields(key) + .and_then(|fields| fields.get("litellm_provider")?.as_str()); + visit(key, provider); + } + } +} + +impl IntegrityLimits { + pub fn new(min_model_count: usize, min_reference_ratio: f64) -> Option { + (0.0..=1.0).contains(&min_reference_ratio).then_some(Self { + min_model_count, + min_reference_ratio, + }) + } +} + +impl ModelEntry { + pub fn field(&self, name: &str) -> Option<&Value> { + self.fields.get(name) + } + + pub fn fields(&self) -> &Fields { + &self.fields + } + + pub fn model(&self) -> Option<&Model> { + self.model.as_ref() + } } impl Catalog { - pub fn parse(body: &[u8], provenance: Provenance) -> Result { - let root: IndexMap = serde_json::from_slice(body)?; - if root.is_empty() { - return Err(Error::Empty); - } + pub fn parse(body: &[u8]) -> Result, Error> { + let document = serde_json::from_slice::(body) + .map_err(|error| Error::new("", Repr::Json(error)))?; - let mut entries = IndexMap::with_capacity(root.len()); + let mut problems = Vec::new(); + let mut sections = Vec::new(); + let mut entries = IndexMap::with_capacity(document.len()); let mut alias_lists = Vec::new(); - let mut alias_issues = Vec::new(); - let mut sample_spec = None; - let mut fallback_generalizations = None; - for (name, value) in root { - match name.as_str() { - "sample_spec" => { - sample_spec = Some(value); + for (index, (name, value)) in document.into_iter().enumerate() { + let fields = match value { + Value::Object(fields) if !is_section(&name) => fields, + value => { + if !is_section(&name) { + problems.push(Error::from_key(&name, Repr::NotAnObject)); + } + sections.push(Section { + index, + key: name, + value, + }); continue; } - "fallback_generalizations" => { - fallback_generalizations = - Some(serde_json::from_value::(value)?); - continue; - } - _ => {} - } - let Value::Object(mut fields) = value else { - return Err(Error::EntryNotObject { model: name }); }; - let info = ModelInfo::deserialize(&fields)?; - if let Some(aliases) = fields.remove("aliases") { - match aliases { - Value::Array(names) => alias_lists.push((name.clone(), names)), - _ => alias_issues.push(AliasIssue::InvalidList { - model: name.clone(), - }), - } + let (model, aliases, row_problems) = read_row(&fields); + problems.extend( + row_problems + .into_iter() + .map(|problem| problem.prepend(&name)), + ); + if let Some(aliases) = aliases { + alias_lists.push((name.clone(), aliases)); } - entries.insert(name, ModelEntry { fields, info }); + entries.insert(name, ModelEntry { fields, model }); + } + if entries.is_empty() { + return Err(Error::new("", Repr::Empty)); } - let mut aliases = IndexMap::new(); + let mut aliases: IndexMap = IndexMap::new(); for (model, names) in alias_lists { - for name in names { - let Value::String(alias) = name else { - alias_issues.push(AliasIssue::InvalidName { - model: model.clone(), - }); - continue; - }; + for alias in names { if entries.contains_key(&alias) { - alias_issues.push(AliasIssue::CanonicalCollision { - model: model.clone(), - alias, - }); - } else if aliases.contains_key(&alias) { - alias_issues.push(AliasIssue::AliasCollision { - model: model.clone(), - alias, - }); + problems.push(Error::from_key(&model, Repr::AliasShadowsModel { alias })); + } else if let Some(taken_by) = aliases.get(&alias) { + problems.push(Error::from_key( + &model, + Repr::AliasTaken { + alias, + taken_by: taken_by.clone(), + }, + )); } else { aliases.insert(alias, model.clone()); } } } - let lowercase_keys = entries - .keys() - .chain(aliases.keys()) - .map(|key| (key.to_lowercase(), key.clone())) - .collect(); - Ok(Self { - entries, - aliases, - lowercase_keys, - sample_spec, - fallback_generalizations, - provenance, - alias_issues, + let fallback_generalizations = sections + .iter() + .find(|section| section.key == FALLBACK_GENERALIZATIONS) + .and_then(|section| { + FallbackGeneralizations::deserialize(§ion.value) + .map_err(|error| { + problems.push(Error::from_key(§ion.key, Repr::Invalid(error))) + }) + .ok() + }); + Ok(Loaded { + value: Self { + store: Store::new(CatalogRows { entries, aliases }), + sections, + fallback_generalizations, + }, + errors: problems, }) } - pub fn validate(&self, limits: IntegrityLimits) -> Result<(), Error> { - if !limits.min_reference_ratio.is_finite() - || !(0.0..=1.0).contains(&limits.min_reference_ratio) + pub fn row_errors(model: &str, fields: &Fields) -> Vec { + let (_, _, errors) = read_row(fields); + errors + .into_iter() + .map(|error| error.prepend(model)) + .collect() + } + + pub fn written(&self) -> impl Iterator)> { + let mut sections = self.sections.iter().peekable(); + let mut entries = self.rows().entries.iter(); + (0..self.rows().entries.len() + self.sections.len()).map_while(move |index| match sections + .next_if(|section| section.index == index) { - return Err(Error::InvalidRatio); - } - let actual = self.entries.len(); + Some(section) => Some((section.key.as_str(), Written::Section(§ion.value))), + None => entries + .next() + .map(|(name, entry)| (name.as_str(), Written::Entry(&entry.fields))), + }) + } + + pub fn served(&self) -> impl Iterator { + let rows = self.rows(); + rows.entries + .iter() + .map(|(name, entry)| (name.as_str(), entry.fields())) + .chain( + rows.aliases + .iter() + .map(|(alias, owner)| (alias.as_str(), rows.entries[owner].fields())), + ) + } + + pub fn check_size(&self, reference: usize, limits: IntegrityLimits) -> Result<(), Error> { + let actual = self.model_count(); if actual < limits.min_model_count { - return Err(Error::BelowMinimum { - actual, - minimum: limits.min_model_count, - }); + return Err(Error::new( + "", + Repr::BelowMinimum { + actual, + minimum: limits.min_model_count, + }, + )); } - if limits.reference_model_count > 0 - && (actual as f64) < (limits.reference_model_count as f64) * limits.min_reference_ratio - { - return Err(Error::Shrunk { - actual, - reference: limits.reference_model_count, - ratio: limits.min_reference_ratio, - }); + if (actual as f64) < (reference as f64) * limits.min_reference_ratio { + return Err(Error::new( + "", + Repr::Shrunk { + actual, + reference, + ratio: limits.min_reference_ratio, + }, + )); } Ok(()) } - pub fn lookup(&self, key: &str) -> Option> { - let matched_key = if self.entries.contains_key(key) || self.aliases.contains_key(key) { - key - } else { - self.lowercase_keys.get(&key.to_lowercase())?.as_str() - }; - let canonical_key = self - .aliases - .get(matched_key) - .map(String::as_str) - .unwrap_or(matched_key); - let (canonical_key, entry) = self.entries.get_key_value(canonical_key)?; - let matched_key = self + pub fn without_rejected(self, errors: &[Error]) -> (Self, Vec<&Error>) { + let row_names: HashSet = self + .rows() .entries - .get_key_value(matched_key) + .keys() + .cloned() + .chain( + self.sections + .iter() + .filter(|section| !is_section(§ion.key)) + .map(|section| section.key.clone()), + ) + .collect(); + let rejected: HashSet = errors + .iter() + .filter(|error| error.rejects() == Rejects::Row) + .filter_map(|error| crate::error::pointer_root(error.path())) + .filter(|root| row_names.contains(root)) + .collect(); + let removed_errors = errors + .iter() + .filter(|error| { + crate::error::pointer_root(error.path()) + .is_some_and(|root| rejected.contains(&root)) + }) + .collect(); + let removed_positions: HashSet = self + .written() + .enumerate() + .filter(|(_, (key, _))| rejected.contains(*key)) + .map(|(index, _)| index) + .collect(); + let mut catalog = self; + catalog.store.edit(|rows| { + rows.entries.retain(|name, _| !rejected.contains(name)); + rows.aliases.retain(|_, owner| !rejected.contains(owner)); + }); + catalog + .sections + .retain(|section| is_section(§ion.key) || !rejected.contains(§ion.key)); + for section in &mut catalog.sections { + let removed_before = removed_positions + .iter() + .filter(|index| **index < section.index) + .count(); + section.index -= removed_before; + } + (catalog, removed_errors) + } + + pub fn unrecognized(&self) -> impl Iterator { + self.rows().entries.iter().flat_map(|(name, entry)| { + entry + .model + .iter() + .flat_map(|model| &model.unrecognized) + .map(move |key| (name.as_str(), key.as_str())) + }) + } + + pub fn lookup(&self, key: &str) -> Option> { + let rows = self.rows(); + let found = self.store.find_key(key)?; + let (canonical_key, entry) = rows.entries.get_key_value(rows.owner(&found))?; + let matched_key = rows + .entries + .get_key_value(found.as_str()) .map(|(key, _)| key.as_str()) .or_else(|| { - self.aliases - .get_key_value(matched_key) + rows.aliases + .get_key_value(found.as_str()) .map(|(key, _)| key.as_str()) })?; Some(ModelMatch { @@ -204,30 +349,58 @@ impl Catalog { } pub fn model_count(&self) -> usize { - self.entries.len() + self.rows().entries.len() } + pub fn model_names(&self) -> impl Iterator { - self.entries.keys().map(String::as_str) - } - pub fn alias_count(&self) -> usize { - self.aliases.len() + self.rows().entries.keys().map(String::as_str) } + pub fn aliases(&self) -> &IndexMap { - &self.aliases + &self.rows().aliases } - pub fn alias_issues(&self) -> &[AliasIssue] { - &self.alias_issues + + fn rows(&self) -> &CatalogRows { + self.store.rows() } + pub fn sample_spec(&self) -> Option<&Value> { - self.sample_spec.as_ref() - } - pub fn fallback_generalizations(&self) -> Option<&FallbackGeneralizations> { - self.fallback_generalizations.as_ref() + self.sections + .iter() + .find(|section| section.key == SAMPLE_SPEC) + .map(|section| §ion.value) } + pub fn fallback_rules(&self) -> Option<&[FallbackRule]> { Some(self.fallback_generalizations.as_ref()?.rules.as_slice()) } - pub fn provenance(&self) -> &Provenance { - &self.provenance +} + +fn read_row(fields: &Fields) -> (Option, Option>, Vec) { + let wire = EntryWire::read(fields); + let aliases = wire.attributes.aliases.clone(); + let timezone_error = unknown_timezone(&wire); + match Model::try_from(wire) { + Ok(model) => (Some(model), aliases, timezone_error.into_iter().collect()), + Err(mut errors) => { + errors.extend(timezone_error); + (None, aliases, errors) + } } } + +fn is_section(key: &str) -> bool { + key == SAMPLE_SPEC || key == FALLBACK_GENERALIZATIONS +} + +fn unknown_timezone(wire: &EntryWire) -> Option { + let zone = wire.off_peak.as_ref()?.weekday_timezone.as_deref()?.trim(); + (!zone.is_empty() && jiff::tz::TimeZone::get(zone).is_err()).then(|| { + Error::from_path( + "off_peak_pricing.weekday_timezone", + Repr::UnknownTimezone { + zone: zone.to_owned(), + }, + ) + }) +} diff --git a/litellm-rust/crates/model-catalog/src/error.rs b/litellm-rust/crates/model-catalog/src/error.rs index 00902858be4..425ae47687f 100644 --- a/litellm-rust/crates/model-catalog/src/error.rs +++ b/litellm-rust/crates/model-catalog/src/error.rs @@ -1,21 +1,392 @@ +use std::num::ParseIntError; + +use strum::{EnumDiscriminants, IntoStaticStr, VariantArray}; use thiserror::Error; #[derive(Debug, Error)] -pub enum Error { - #[error("invalid catalog JSON")] - Json(#[from] serde_json::Error), - #[error("catalog is empty")] +#[error(transparent)] +pub struct Error(Box); + +#[derive(Debug, Error)] +enum Context { + #[error(transparent)] + Bare(Repr), + #[error("{}", display_path(path))] + At { + path: String, + #[source] + cause: Repr, + }, +} + +#[derive(Debug, Error)] +pub(crate) enum DateCause { + #[error("invalid date component")] + Number(#[from] ParseIntError), + #[error("calendar component out of range")] + Range(#[from] time::error::ComponentRange), +} + +#[derive(Debug, Error, EnumDiscriminants)] +#[strum_discriminants(name(ErrorKind), derive(IntoStaticStr, VariantArray), vis(pub))] +pub(crate) enum Repr { + #[error("invalid JSON")] + #[strum_discriminants(strum(serialize = "json"))] + Json(#[source] serde_json::Error), + #[error("has no models")] + #[strum_discriminants(strum(serialize = "empty"))] Empty, - #[error("model {model:?} must be an object")] - EntryNotObject { model: String }, - #[error("catalog has {actual} models, below minimum {minimum}")] + #[error("has {actual} models, below minimum {minimum}")] + #[strum_discriminants(strum(serialize = "below_minimum"))] BelowMinimum { actual: usize, minimum: usize }, - #[error("catalog has {actual} models, below {ratio} of reference count {reference}")] + #[error("has {actual} models, below {ratio} of reference count {reference}")] + #[strum_discriminants(strum(serialize = "shrunk"))] Shrunk { actual: usize, reference: usize, ratio: f64, }, - #[error("minimum reference ratio must be finite and between zero and one")] - InvalidRatio, + #[error("must be a JSON object")] + #[strum_discriminants(strum(serialize = "not_an_object"))] + NotAnObject, + #[error("invalid value")] + #[strum_discriminants(strum(serialize = "invalid"))] + Invalid(#[source] serde_json::Error), + #[error("not a field this block reads")] + #[strum_discriminants(strum(serialize = "unknown_block_key"))] + UnknownBlockKey, + #[error("required")] + #[strum_discriminants(strum(serialize = "missing_provider"))] + MissingProvider, + #[error("{zone:?} is not an IANA time zone")] + #[strum_discriminants(strum(serialize = "unknown_timezone"))] + UnknownTimezone { zone: String }, + #[error("alias {alias:?} is already a model name")] + #[strum_discriminants(strum(serialize = "alias_shadows_model"))] + AliasShadowsModel { alias: String }, + #[error("alias {alias:?} already belongs to {taken_by}")] + #[strum_discriminants(strum(serialize = "alias_taken"))] + AliasTaken { alias: String, taken_by: String }, + #[error("{first} and {second} price the same meter under the same conditions")] + #[strum_discriminants(strum(serialize = "duplicate_price"))] + DuplicatePrice { first: String, second: String }, + #[error("a price inside {block} takes no band or tier suffix")] + #[strum_discriminants(strum(serialize = "suffix_inside_block"))] + SuffixInsideBlock { block: &'static str }, + #[error("a tier needs exactly one of range and max_results_range")] + #[strum_discriminants(strum(serialize = "tier_range"))] + TierRange, + #[error("the range ends below its start")] + #[strum_discriminants(strum(serialize = "tier_reversed"))] + TierReversed, + #[error("starts at {start}, but the tiers before it end at {expected}")] + #[strum_discriminants(strum(serialize = "tier_gap"))] + TierGap { start: u64, expected: u64 }, + #[error("tiered_pricing mixes token ranges and result ranges")] + #[strum_discriminants(strum(serialize = "mixed_tier_basis"))] + MixedTierBasis, + #[error("a tier by result count only prices input_cost_per_query")] + #[strum_discriminants(strum(serialize = "not_priced_by_results"))] + NotPricedByResults, + #[error("tiered_pricing and prompt token bands are set in two places")] + #[strum_discriminants(strum(serialize = "bands_from_two_sources"))] + BandsFromTwoSources, + #[error("has no off-peak rate")] + #[strum_discriminants(strum(serialize = "not_an_off_peak_rate"))] + NotAnOffPeakRate, + #[error("off_peak_pricing has no prices")] + #[strum_discriminants(strum(serialize = "no_off_peak_prices"))] + NoOffPeakPrices, + #[error("off_peak_pricing needs hours_utc or windows")] + #[strum_discriminants(strum(serialize = "no_off_peak_hours"))] + NoOffPeakHours, + #[error("must list at least one entry")] + #[strum_discriminants(strum(serialize = "empty_list"))] + EmptyList, + #[error("expected a calendar date YYYY-MM-DD")] + #[strum_discriminants(strum(serialize = "invalid_date"))] + InvalidDate { + #[source] + cause: Option, + }, +} + +/// What a problem takes out of the served map +#[derive(Clone, Copy, Debug, PartialEq, Eq, VariantArray)] +pub enum Rejects { + Document, + Row, + Nothing, +} + +impl Error { + pub fn kind(&self) -> ErrorKind { + ErrorKind::from(self.repr()) + } + + pub fn rejects(&self) -> Rejects { + match self.repr() { + Repr::Json(_) | Repr::Empty | Repr::BelowMinimum { .. } | Repr::Shrunk { .. } => { + Rejects::Document + } + Repr::NotAnObject + | Repr::Invalid(_) + | Repr::UnknownBlockKey + | Repr::UnknownTimezone { .. } + | Repr::AliasShadowsModel { .. } + | Repr::AliasTaken { .. } + | Repr::DuplicatePrice { .. } + | Repr::SuffixInsideBlock { .. } + | Repr::TierRange + | Repr::TierReversed + | Repr::TierGap { .. } + | Repr::MixedTierBasis + | Repr::NotPricedByResults + | Repr::BandsFromTwoSources + | Repr::NotAnOffPeakRate + | Repr::NoOffPeakPrices + | Repr::NoOffPeakHours + | Repr::EmptyList + | Repr::InvalidDate { .. } => Rejects::Row, + Repr::MissingProvider => Rejects::Nothing, + } + } +} + +#[derive(Debug)] +pub struct Loaded { + pub value: T, + pub errors: Vec, +} + +impl From for Error { + fn from(cause: Repr) -> Self { + Self(Box::new(Context::Bare(cause))) + } +} + +impl From for Error { + fn from(cause: DateCause) -> Self { + Repr::InvalidDate { cause: Some(cause) }.into() + } +} + +impl Error { + fn repr(&self) -> &Repr { + match &*self.0 { + Context::Bare(cause) | Context::At { cause, .. } => cause, + } + } + + pub(crate) fn new(path: impl Into, cause: Repr) -> Self { + Self(Box::new(Context::At { + path: path.into(), + cause, + })) + } + + pub(crate) fn from_key(key: &str, cause: Repr) -> Self { + Self::new(pointer_for_key(key), cause) + } + + pub(crate) fn from_path(path: &str, cause: Repr) -> Self { + Self::new(path_to_pointer(path), cause) + } + + pub fn path(&self) -> &str { + match &*self.0 { + Context::Bare(_) => "", + Context::At { path, .. } => path, + } + } + + pub fn key(&self) -> Option { + pointer_root(self.path()) + } + + pub fn field(&self) -> Option { + let segments: Vec<_> = self + .path() + .split('/') + .skip(2) + .map(unescape_pointer_segment) + .collect(); + (!segments.is_empty()).then(|| segments.join("/")) + } + + pub(crate) fn prepend(self, key: &str) -> Self { + let (path, cause) = match *self.0 { + Context::Bare(cause) => (String::new(), cause), + Context::At { path, cause } => (path, cause), + }; + Self::new(format!("/{}{path}", escape_pointer_segment(key)), cause) + } +} + +pub(crate) fn pointer_root(path: &str) -> Option { + path.strip_prefix('/') + .and_then(|path| path.split('/').next()) + .map(unescape_pointer_segment) +} + +fn escape_pointer_segment(segment: &str) -> String { + segment.replace('~', "~0").replace('/', "~1") +} + +pub(crate) fn pointer_for_key(key: &str) -> String { + format!("/{}", escape_pointer_segment(key)) +} + +pub(crate) fn pointer_child(parent: &str, key: &str) -> String { + format!("{parent}/{}", escape_pointer_segment(key)) +} + +fn unescape_pointer_segment(segment: &str) -> String { + segment.replace("~1", "/").replace("~0", "~") +} + +fn path_to_pointer(key: &str) -> String { + let mut pointer = String::new(); + let mut segment = String::new(); + let mut chars = key.chars(); + let mut push_segment = |segment: &mut String| { + if !segment.is_empty() { + pointer.push('/'); + pointer.push_str(&escape_pointer_segment(segment)); + segment.clear(); + } + }; + while let Some(character) = chars.next() { + match character { + '.' => push_segment(&mut segment), + '[' => { + push_segment(&mut segment); + let mut index = String::new(); + for character in chars.by_ref() { + if character == ']' { + break; + } + index.push(character); + } + push_segment(&mut index); + } + character => segment.push(character), + } + } + push_segment(&mut segment); + pointer +} + +fn display_path(path: &str) -> String { + if path.is_empty() { + return "catalog".to_owned(); + } + path.split('/') + .skip(1) + .map(unescape_pointer_segment) + .enumerate() + .fold(String::new(), |mut display, (index, segment)| { + if index == 0 { + display.push_str(&segment); + } else if segment.bytes().all(|byte| byte.is_ascii_digit()) { + display.push('['); + display.push_str(&segment); + display.push(']'); + } else { + display.push('.'); + display.push_str(&segment); + } + display + }) +} + +#[cfg(test)] +mod tests { + use litellm_error::{Report, find}; + use rstest::rstest; + use serde_json::Value; + + use super::*; + use crate::{Catalog, wire::IsoDate}; + + #[rstest] + #[case::json(Repr::Json(serde_json::from_str::("{").unwrap_err()), Rejects::Document)] + #[case::empty(Repr::Empty, Rejects::Document)] + #[case::below_minimum(Repr::BelowMinimum { actual: 1, minimum: 2 }, Rejects::Document)] + #[case::shrunk(Repr::Shrunk { actual: 1, reference: 3, ratio: 0.5 }, Rejects::Document)] + #[case::not_an_object(Repr::NotAnObject, Rejects::Row)] + #[case::invalid(Repr::Invalid(serde_json::from_str::("x").unwrap_err()), Rejects::Row)] + #[case::unknown_block_key(Repr::UnknownBlockKey, Rejects::Row)] + #[case::missing_provider(Repr::MissingProvider, Rejects::Nothing)] + #[case::unknown_timezone(Repr::UnknownTimezone { zone: "invalid".into() }, Rejects::Row)] + #[case::alias_shadows_model(Repr::AliasShadowsModel { alias: "model".into() }, Rejects::Row)] + #[case::alias_taken(Repr::AliasTaken { alias: "alias".into(), taken_by: "owner".into() }, Rejects::Row)] + #[case::duplicate_price(Repr::DuplicatePrice { first: "a".into(), second: "b".into() }, Rejects::Row)] + #[case::suffix_inside_block(Repr::SuffixInsideBlock { block: "tiered_pricing" }, Rejects::Row)] + #[case::tier_range(Repr::TierRange, Rejects::Row)] + #[case::tier_reversed(Repr::TierReversed, Rejects::Row)] + #[case::tier_gap(Repr::TierGap { start: 1, expected: 0 }, Rejects::Row)] + #[case::mixed_tier_basis(Repr::MixedTierBasis, Rejects::Row)] + #[case::not_priced_by_results(Repr::NotPricedByResults, Rejects::Row)] + #[case::bands_from_two_sources(Repr::BandsFromTwoSources, Rejects::Row)] + #[case::not_an_off_peak_rate(Repr::NotAnOffPeakRate, Rejects::Row)] + #[case::no_off_peak_prices(Repr::NoOffPeakPrices, Rejects::Row)] + #[case::no_off_peak_hours(Repr::NoOffPeakHours, Rejects::Row)] + #[case::empty_list(Repr::EmptyList, Rejects::Row)] + #[case::invalid_date(Repr::InvalidDate { cause: None }, Rejects::Row)] + fn policy_classifies_each_error(#[case] error: Repr, #[case] expected: Rejects) { + assert_eq!(Error::from(error).rejects(), expected); + } + + #[rstest] + #[case::root("", "x", "/x")] + #[case::empty_model("/", "x", "//x")] + #[case::empty_parent_segment("/m/", "x", "/m//x")] + #[case::escaped("/m~1n", "a~/b", "/m~1n/a~0~1b")] + fn pointers_preserve_segments(#[case] parent: &str, #[case] key: &str, #[case] expected: &str) { + assert_eq!(pointer_child(parent, key), expected); + } + + #[rstest] + #[case::malformed(br#"{"#)] + #[case::bad_price(br#"{"m":{"litellm_provider":"test","input_cost_per_token":"free"}}"#)] + fn json_causes_are_typed_and_rendered_once(#[case] body: &[u8]) { + let error = match Catalog::parse(body) { + Ok(loaded) => loaded.errors.into_iter().next().expect("invalid price"), + Err(error) => error, + }; + let cause = find::(&error) + .expect("typed JSON cause") + .to_string(); + let report = Report::new(&error).to_string(); + assert_eq!(report.matches(&cause).count(), 1, "{report}"); + assert!(!error.to_string().contains(&cause)); + } + + #[rstest] + #[case::year("xxxx-01-01")] + #[case::month("2026-xx-01")] + #[case::day("2026-01-xx")] + fn date_number_errors_keep_the_cause(#[case] value: &str) { + let error = value.parse::().unwrap_err(); + let cause = find::(&error) + .expect("typed integer cause") + .to_string(); + assert_eq!(error.rejects(), Rejects::Row); + assert_eq!(Report::new(&error).to_string().matches(&cause).count(), 1); + assert!(!error.to_string().contains(&cause)); + } + + #[rstest] + #[case::month("2026-13-01")] + #[case::day("2026-02-30")] + fn date_range_errors_keep_the_cause(#[case] value: &str) { + let error = value.parse::().unwrap_err(); + let cause = find::(&error) + .expect("typed calendar cause") + .to_string(); + assert_eq!(error.rejects(), Rejects::Row); + assert_eq!(Report::new(&error).to_string().matches(&cause).count(), 1); + assert!(!error.to_string().contains(&cause)); + } } diff --git a/litellm-rust/crates/model-catalog/src/fallback.rs b/litellm-rust/crates/model-catalog/src/fallback.rs index 62291a84929..f5a0cadb6b9 100644 --- a/litellm-rust/crates/model-catalog/src/fallback.rs +++ b/litellm-rust/crates/model-catalog/src/fallback.rs @@ -1,10 +1,9 @@ -use serde::{Deserialize, Serialize}; use serde_json::Value; use std::collections::BTreeMap; /// One regex rule generalizing unknown model ids to known families. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[macro_rules_attribute::apply(crate::wire_type)] +#[derive(Eq, Hash)] pub struct FallbackRule { pub name: String, pub pattern: String, @@ -15,8 +14,8 @@ pub struct FallbackRule { } /// Regex rules that generalize unknown model ids to known families; not a model entry. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[macro_rules_attribute::apply(crate::wire_type)] +#[derive(Eq, Hash)] #[serde(deny_unknown_fields)] pub struct FallbackGeneralizations { pub rules: Vec, diff --git a/litellm-rust/crates/model-catalog/src/layers.rs b/litellm-rust/crates/model-catalog/src/layers.rs new file mode 100644 index 00000000000..881f0f6af9b --- /dev/null +++ b/litellm-rust/crates/model-catalog/src/layers.rs @@ -0,0 +1,32 @@ +use std::sync::Arc; + +use indexmap::IndexMap; +use serde_json::Value; + +use crate::catalog::Catalog; +use crate::register::Registrations; +use crate::wire::Fields; + +/// One field of a fetched row over the layer below, RFC 7386. An absent field inherits, JSON `null` is `Remove` +#[derive(Clone, Debug, PartialEq)] +pub enum Patch { + Set(Value), + Remove, +} + +/// The four layers served rows are composed from, lowest first +#[derive(Clone, Debug)] +#[expect( + dead_code, + reason = "02 and 07 add the functions that read these layers" +)] +pub struct Layers { + /// 0, the wheel's embedded map + bundled: Arc, + /// 1, the last installed fetch, rejected rows left out + fetched: Option>, + /// 2, `register(persist=False)`, cleared by every install and refilled by the router replay + transient: IndexMap, + /// 3, `register(persist=True)`, applied over every install + persisted: Registrations, +} diff --git a/litellm-rust/crates/model-catalog/src/lib.rs b/litellm-rust/crates/model-catalog/src/lib.rs index d42c05e6fe7..fe419dde054 100644 --- a/litellm-rust/crates/model-catalog/src/lib.rs +++ b/litellm-rust/crates/model-catalog/src/lib.rs @@ -7,20 +7,18 @@ //! in `litellm-router-core`, which reads this catalog through its dynamic config. //! //! # Structure -//! - `Catalog`: the parsed map. `Catalog::parse` takes the JSON body and a `Provenance` (where -//! the body came from: the bundled copy in the repo or the remote copy, with revision and etag), -//! and keeps every entry in file order with its raw fields. +//! - `Catalog`: the parsed map. `Catalog::parse` takes the JSON body and returns `Loaded`, +//! keeping every entry in file order with its raw fields and collecting row and alias errors. //! - `ModelEntry`: one model. `ModelEntry::fields` is the raw JSON object, so unknown fields are -//! never lost; `ModelEntry::info` is the typed mirror `ModelInfo` (pricing in `pricing`, -//! modalities and modes in `capabilities`). +//! never lost; `ModelEntry::model` is the optional typed `Model` with attributes, prices, +//! adjustments and capabilities. Invalid rows retain their raw fields without a typed model. //! - Aliases: an entry may list `aliases`; each alias resolves to its canonical key. Problems -//! found while indexing them (bad lists, collisions) are kept as `AliasIssue`s instead of -//! failing the parse. +//! found while indexing them (bad lists, collisions) are kept in `Loaded::errors` instead of +//! failing the document. //! - `FallbackGeneralizations` / `FallbackRule`: regex rules that map unknown model ids onto known //! families. They are data in the file, not entries. -//! - Validation: `Catalog::validate` checks a catalog against `IntegrityLimits` (a remote copy that -//! lost most of its models is refused), and `validate_registry` checks every entry against the -//! checked-in contract. +//! - Validation: `Catalog::check_size` checks a catalog against `IntegrityLimits` (a remote copy +//! that lost most of its models is refused), and `Catalog::row_errors` validates one row. //! //! # Lookups //! `Catalog::lookup` resolves a key exactly first (entry or alias), then case-insensitively, and @@ -31,21 +29,37 @@ //! Cost tracking, the router (through `RouterDynamicConfig`) and the gateway. Nothing in this crate //! performs I/O; loading and refreshing the file is the caller's job. -mod capabilities; +macro_rules_attribute::attribute_alias! { + #[apply(wire_type)] = + #[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)] + #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]; +} + +mod bundled; mod catalog; mod error; mod fallback; +mod layers; +mod lower; +mod model; mod model_info; -mod pricing; -mod validation; +mod register; +mod resolve; +mod store; +mod vocabulary; +pub mod wire; -pub use capabilities::*; +pub use bundled::*; pub use catalog::*; pub use error::*; pub use fallback::*; +pub use layers::*; +pub use model::*; pub use model_info::*; -pub use pricing::*; -pub use validation::*; +pub use register::*; +pub use resolve::*; +pub use store::*; +pub use vocabulary::*; #[cfg(feature = "schema")] mod schema; diff --git a/litellm-rust/crates/model-catalog/src/lower.rs b/litellm-rust/crates/model-catalog/src/lower.rs new file mode 100644 index 00000000000..4c3fc38a01a --- /dev/null +++ b/litellm-rust/crates/model-catalog/src/lower.rs @@ -0,0 +1,855 @@ +use std::collections::BTreeMap; +use std::collections::btree_map::Entry; +use std::num::NonZeroU64; + +use litellm_cost::money::UnitPrice; +use litellm_cost::price::{ + Adjustments, Band, BandEdge, Billing, CacheReadFallback, CacheReadMode, CacheTtl, + GuardrailUnit, InputUnit, Meter, OffPeak, OffPeakRates, OutputUnit, Price, Prices, Residency, + SearchContextSize, TextUnit, TierScope, TokenKind, Uplift, WallClock, WebSearchUnit, When, +}; +use litellm_cost::schedule::{Schedule, Window}; + +use crate::error::{Error, Repr, pointer_child, pointer_for_key}; +use crate::model::Model; +use crate::vocabulary::Mode; +use crate::wire::{EntryWire, FlatPrice, OffPeakBlock, TierRow, WebSearchBillingUnit}; + +impl TryFrom<&EntryWire> for Prices { + type Error = Vec; + + fn try_from(wire: &EntryWire) -> Result { + let mut problems = Vec::new(); + let sourced = flat_prices(&wire.prices) + .chain(tier_prices(wire, &mut problems)) + .chain(search_context_prices(wire)) + .chain(guardrail_prices(wire)); + let rates = dedupe(sourced, &mut problems); + let off_peak = wire + .off_peak + .as_ref() + .and_then(|block| off_peak(block, &mut problems)); + if problems.is_empty() { + let billing = billing(wire, &rates); + Ok(Prices::new(rates, off_peak, billing)) + } else { + Err(problems) + } + } +} + +fn billing(wire: &EntryWire, rates: &BTreeMap) -> Billing { + let provider = wire.attributes.litellm_provider.as_deref(); + let prices_meter = |priced: fn(&Meter) -> bool| rates.keys().any(|price| priced(&price.meter)); + let prices_tokens = prices_meter(|meter| { + matches!( + meter, + Meter::Input(InputUnit::Token(_)) | Meter::Output(OutputUnit::Token(_)) + ) + }); + let prices_characters = prices_meter(|meter| { + matches!( + meter, + Meter::Input(InputUnit::Character) | Meter::Output(OutputUnit::Character) + ) + }); + let prices_cache_read = prices_meter(|meter| { + matches!( + meter, + Meter::CacheRead { + kind: TokenKind::Text, + .. + } + ) + }); + Billing { + text_unit: if prices_characters && !prices_tokens { + TextUnit::Character + } else { + TextUnit::Token + }, + band_edge: if provider == Some("xai") { + BandEdge::Inclusive + } else { + BandEdge::Exclusive + }, + cache_read_fallback: if provider == Some("fireworks_ai") && !prices_cache_read { + CacheReadFallback::HalfInputRate + } else { + CacheReadFallback::InputRate + }, + wall_clock: match wire.attributes.mode { + None | Some(Mode::Chat | Mode::Completion | Mode::Embedding | Mode::Responses) => { + WallClock::ResponseTime + } + Some(Mode::AudioTranscription) => WallClock::AudioFile, + Some(_) => WallClock::Never, + }, + web_search: match wire.attributes.web_search_billing_unit { + Some(WebSearchBillingUnit::PerQuery) => WebSearchUnit::PerQuery, + Some(WebSearchBillingUnit::PerPrompt) | None => WebSearchUnit::PerPrompt, + }, + tier_scope: if provider == Some("dashscope") && wire.tier_rows.is_some() { + TierScope::WholeCall + } else { + TierScope::PerMeter + }, + } +} + +impl From<&EntryWire> for Adjustments { + fn from(wire: &EntryWire) -> Self { + Adjustments::new( + [ + (Uplift::NonGlobalEndpoint, wire.regional_endpoint_uplift), + (Uplift::Residency(Residency::Eu), wire.residency_uplift_eu), + (Uplift::Residency(Residency::Us), wire.residency_uplift_us), + ] + .into_iter() + .filter_map(|(uplift, factor)| Some((uplift, factor?))), + ) + } +} + +impl TryFrom for Model { + type Error = Vec; + + fn try_from(wire: EntryWire) -> Result { + let prices = Prices::try_from(&wire); + let adjustments = Adjustments::from(&wire); + let EntryWire { + attributes, + capabilities, + unknown, + problems, + .. + } = wire; + let missing_provider = attributes.litellm_provider.is_none().then_some(Error::new( + pointer_for_key("litellm_provider"), + Repr::MissingProvider, + )); + match prices { + Ok(prices) if problems.is_empty() && missing_provider.is_none() => Ok(Model { + attributes, + prices, + adjustments, + capabilities, + unrecognized: unknown.into_iter().map(|(key, _)| key).collect(), + }), + prices => Err(problems + .into_iter() + .chain(missing_provider) + .chain(prices.err().into_iter().flatten()) + .collect()), + } + } +} + +struct Sourced { + price: Price, + rate: UnitPrice, + source: String, + path: String, +} + +fn flat_prices(prices: &[FlatPrice]) -> impl Iterator + '_ { + prices.iter().map(|flat| { + let (channel, when) = flat.flat.channel_and_when(); + Sourced { + price: Price { + meter: flat.flat.meter.clone(), + channel, + when, + }, + rate: flat.price, + source: flat.key.clone(), + path: flat.path.clone(), + } + }) +} + +fn dedupe( + sourced: impl Iterator, + problems: &mut Vec, +) -> BTreeMap { + let mut seen: BTreeMap = BTreeMap::new(); + for Sourced { + price, + rate, + source, + path, + } in sourced + { + match seen.entry(price) { + Entry::Occupied(first) if first.get().0 == rate => {} + Entry::Occupied(first) => problems.push(Error::new( + path, + Repr::DuplicatePrice { + first: first.get().1.clone(), + second: source.clone(), + }, + )), + Entry::Vacant(slot) => { + slot.insert((rate, source, path)); + } + } + } + seen.into_iter() + .map(|(price, (rate, _, _))| (price, rate)) + .collect() +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum TierBasis { + PromptTokens, + Results, +} + +fn tier_basis(row: &TierRow) -> Result<(TierBasis, [u64; 2]), Error> { + match (row.range, row.max_results_range) { + (Some(range), None) => Ok((TierBasis::PromptTokens, range)), + (None, Some(range)) => Ok((TierBasis::Results, range)), + _ => Err(Error::new(row.path.clone(), Repr::TierRange)), + } +} + +fn tier_prices(wire: &EntryWire, problems: &mut Vec) -> Vec { + let Some(rows) = wire.tier_rows.as_deref().filter(|rows| !rows.is_empty()) else { + return Vec::new(); + }; + let based: Vec<_> = rows + .iter() + .filter_map(|row| match tier_basis(row) { + Ok((basis, range)) => Some((basis, range, row)), + Err(problem) => { + problems.push(problem); + None + } + }) + .collect(); + let Some(&(basis, ..)) = based.first() else { + return Vec::new(); + }; + if based.iter().any(|(other, ..)| *other != basis) { + problems.push(Error::from_key("tiered_pricing", Repr::MixedTierBasis)); + return Vec::new(); + } + if basis == TierBasis::PromptTokens { + problems.extend( + wire.prices + .iter() + .filter(|flat| flat.flat.band != Band::Base) + .map(|flat| Error::new(flat.path.clone(), Repr::BandsFromTwoSources)), + ); + } + let mut ordered = based; + ordered.sort_by_key(|(_, [start, _], _)| *start); + problems.extend(ladder_problems(basis, &ordered)); + ordered + .iter() + .flat_map(|(basis, [start, _], row)| { + row.prices.iter().map(move |flat| (*basis, *start, flat)) + }) + .filter_map( + |(basis, start, flat)| match tier_price(basis, start, flat) { + Ok(sourced) => Some(sourced), + Err(problem) => { + problems.push(problem); + None + } + }, + ) + .collect() +} + +fn ladder_problems(basis: TierBasis, ordered: &[(TierBasis, [u64; 2], &TierRow)]) -> Vec { + let step = match basis { + TierBasis::PromptTokens => 0, + TierBasis::Results => 1, + }; + let reversed = ordered + .iter() + .filter(|(_, [start, end], _)| end < start) + .map(|(.., row)| Error::new(row.path.clone(), Repr::TierReversed)); + let first = ordered + .first() + .filter(|(_, [start, _], _)| *start > step) + .map(|(_, [start, _], row)| { + Error::new( + row.path.clone(), + Repr::TierGap { + start: *start, + expected: 0, + }, + ) + }); + let gaps = ordered.windows(2).filter_map(|pair| { + let (_, [_, end], _) = pair[0]; + let (_, [start, _], row) = pair[1]; + (end.checked_add(step) != Some(start)).then(|| { + Error::new( + row.path.clone(), + Repr::TierGap { + start, + expected: end, + }, + ) + }) + }); + reversed.chain(first).chain(gaps).collect() +} + +fn tier_price(basis: TierBasis, start: u64, flat: &FlatPrice) -> Result { + if flat.flat.has_suffix() { + return Err(Error::new( + flat.path.clone(), + Repr::SuffixInsideBlock { + block: "tiered_pricing", + }, + )); + } + let (meter, band) = match (basis, &flat.flat.meter) { + (TierBasis::PromptTokens, meter) => (meter.clone(), Band::over_prompt_tokens(start)), + (TierBasis::Results, Meter::Input(InputUnit::Query { over_results: None })) => ( + Meter::Input(InputUnit::Query { + over_results: NonZeroU64::new(start.saturating_sub(1)), + }), + Band::Base, + ), + (TierBasis::Results, _) => { + return Err(Error::new(flat.path.clone(), Repr::NotPricedByResults)); + } + }; + Ok(Sourced { + price: Price::online( + meter, + When { + band, + ..When::default() + }, + ), + rate: flat.price, + source: flat.key.clone(), + path: flat.path.clone(), + }) +} + +fn search_context_prices(wire: &EntryWire) -> impl Iterator + '_ { + wire.search_context.iter().flat_map(|costs| { + [ + (SearchContextSize::Low, "low", costs.search_context_size_low), + ( + SearchContextSize::Medium, + "medium", + costs.search_context_size_medium, + ), + ( + SearchContextSize::High, + "high", + costs.search_context_size_high, + ), + ] + .into_iter() + .filter_map(|(size, name, rate)| { + Some(Sourced { + price: Price::online(Meter::WebSearch(size), When::default()), + rate: rate?, + source: format!("search_context_cost_per_query.search_context_size_{name}"), + path: pointer_child( + &pointer_for_key("search_context_cost_per_query"), + &format!("search_context_size_{name}"), + ), + }) + }) + }) +} + +fn guardrail_prices(wire: &EntryWire) -> impl Iterator + '_ { + wire.guardrail.iter().flatten().map(|(unit, rate)| Sourced { + price: Price::online(Meter::Guardrail(GuardrailUnit::new(unit)), When::default()), + rate: *rate, + source: format!("guardrail_cost_per_unit.{unit}"), + path: pointer_child(&pointer_for_key("guardrail_cost_per_unit"), unit), + }) +} + +fn off_peak(block: &OffPeakBlock, problems: &mut Vec) -> Option { + let rates = off_peak_rates(&block.prices, problems); + let schedule = schedule(block, problems)?; + let off_peak = OffPeak::new(schedule, rates); + if off_peak.is_none() { + problems.push(Error::from_key("off_peak_pricing", Repr::NoOffPeakPrices)); + } + off_peak +} + +fn off_peak_rates(prices: &[FlatPrice], problems: &mut Vec) -> OffPeakRates { + let mut rates = OffPeakRates::default(); + for flat in prices { + if flat.flat.has_suffix() { + problems.push(Error::new( + flat.path.clone(), + Repr::SuffixInsideBlock { + block: "off_peak_pricing", + }, + )); + continue; + } + let slot = match flat.flat.meter { + Meter::Input(InputUnit::Token(TokenKind::Text)) => &mut rates.input, + Meter::Output(OutputUnit::Token(TokenKind::Text)) => &mut rates.output, + Meter::Output(OutputUnit::ReasoningToken) => &mut rates.reasoning, + Meter::CacheRead { + kind: TokenKind::Text, + mode: CacheReadMode::Implicit, + } => &mut rates.cache_read, + Meter::CacheRead { + kind: TokenKind::Text, + mode: CacheReadMode::Explicit, + } => &mut rates.explicit_cache_read, + Meter::CacheWrite { + kind: TokenKind::Text, + ttl: CacheTtl::Default, + } => &mut rates.cache_write, + _ => { + problems.push(Error::new(flat.path.clone(), Repr::NotAnOffPeakRate)); + continue; + } + }; + *slot = Some(flat.price); + } + rates +} + +fn schedule(block: &OffPeakBlock, problems: &mut Vec) -> Option { + let empty = |key: String| Error::from_path(&key, Repr::EmptyList); + let every_day = block + .hours_utc + .iter() + .map(|hours| ("off_peak_pricing.hours_utc".to_owned(), hours, None)); + let windows = block + .windows + .iter() + .flatten() + .enumerate() + .map(|(index, window)| { + ( + format!("off_peak_pricing.windows[{index}].hours_utc"), + &window.hours_utc, + Some((index, window.weekdays.as_ref())), + ) + }); + let windows: Vec = every_day + .chain(windows) + .map(|(key, hours, weekdays)| { + if hours.windows().is_empty() { + problems.push(empty(key)); + } + if let Some((index, Some(days))) = weekdays + && days.is_empty() + { + problems.push(empty(format!("off_peak_pricing.windows[{index}].weekdays"))); + } + Window { + hours: hours.windows().to_vec(), + weekdays: weekdays.and_then(|(_, days)| days.cloned()), + } + }) + .collect(); + if block.windows.as_ref().is_some_and(Vec::is_empty) { + problems.push(empty("off_peak_pricing.windows".to_owned())); + } + if block.hours_utc.is_none() && block.windows.is_none() { + problems.push(Error::from_key("off_peak_pricing", Repr::NoOffPeakHours)); + return None; + } + Some(Schedule { + windows, + weekday_timezone: block.weekday_timezone.clone(), + }) +} + +#[cfg(test)] +mod tests { + use litellm_cost::money::Multiplier; + use litellm_cost::price::{Channel, ServiceTier}; + use litellm_cost::schedule::{UtcWindow, Weekday}; + use rstest::rstest; + use serde_json::{Value, json}; + + use super::*; + + const TEXT_INPUT: Meter = Meter::Input(InputUnit::Token(TokenKind::Text)); + + fn wire(entry: Value) -> EntryWire { + let Value::Object(fields) = entry else { + panic!("entry must be an object") + }; + let wire = EntryWire::read(&fields); + assert!(wire.problems.is_empty(), "{:?}", wire.problems); + wire + } + + fn prices(entry: Value) -> Result> { + Prices::try_from(&wire(entry)) + } + + fn online(meter: Meter, tier: ServiceTier, band: Band) -> Price { + Price::online(meter, When { tier, band }) + } + + #[rstest] + #[case::flat_key(json!({"input_cost_per_token": 1e-6}), online(TEXT_INPUT, ServiceTier::Standard, Band::Base))] + #[case::band_and_tier( + json!({"input_cost_per_token_above_272k_tokens_priority": 1e-6}), + online(TEXT_INPUT, ServiceTier::Priority, Band::over_prompt_tokens(272_000)) + )] + #[case::batch( + json!({"input_cost_per_token_batches": 1e-6}), + Price { meter: TEXT_INPUT, channel: Channel::Batch, when: When::default() } + )] + #[case::first_token_tier_is_the_base_band( + json!({"tiered_pricing": [{"range": [0.0, 256000.0], "input_cost_per_token": 1e-6}]}), + online(TEXT_INPUT, ServiceTier::Standard, Band::Base) + )] + #[case::result_tier_stays_in_its_meter( + json!({"tiered_pricing": [ + {"max_results_range": [0, 25], "input_cost_per_query": 2e-6}, + {"max_results_range": [26, 100], "input_cost_per_query": 1e-6} + ]}), + online( + Meter::Input(InputUnit::Query { over_results: NonZeroU64::new(25) }), + ServiceTier::Standard, + Band::Base + ) + )] + #[case::search_context( + json!({"search_context_cost_per_query": {"search_context_size_low": 1e-6}}), + online(Meter::WebSearch(SearchContextSize::Low), ServiceTier::Standard, Band::Base) + )] + #[case::character_band_is_a_prompt_token_band( + json!({"input_cost_per_character_above_128k_tokens": 1e-6}), + online(Meter::Input(InputUnit::Character), ServiceTier::Standard, Band::over_prompt_tokens(128_000)) + )] + #[case::guardrail( + json!({"guardrail_cost_per_unit": {"contentPolicyUnits": 1e-6}}), + online(Meter::Guardrail(GuardrailUnit::new("contentPolicyUnits")), ServiceTier::Standard, Band::Base) + )] + fn every_spelling_lowers_to_one_price(#[case] entry: Value, #[case] expected: Price) { + assert_eq!(prices(entry).unwrap().get(&expected), UnitPrice::new(1e-6)); + } + + #[rstest] + fn two_spellings_that_agree_are_one_price() { + let prices = prices(json!({ + "input_cost_per_token": 1e-6, + "tiered_pricing": [ + {"range": [0, 10], "input_cost_per_token": 1e-6}, + {"range": [10, 20], "input_cost_per_token": 2e-6} + ] + })) + .unwrap(); + assert_eq!( + prices.get(&Price::online(TEXT_INPUT, When::default())), + UnitPrice::new(1e-6) + ); + } + + #[rstest] + fn token_tiers_lower_to_prompt_bands_at_each_start() { + let prices = prices(json!({"tiered_pricing": [ + {"range": [0, 256000], "input_cost_per_token": 1e-6}, + {"range": [256000, 1000000], "input_cost_per_token": 2e-6} + ]})) + .unwrap(); + assert_eq!( + prices.band(ServiceTier::Standard, 256_001), + Band::over_prompt_tokens(256_000) + ); + assert_eq!( + prices + .rate( + &TEXT_INPUT, + ServiceTier::Standard, + Band::over_prompt_tokens(256_000) + ) + .map(UnitPrice::get), + Some(2e-6) + ); + } + + #[rstest] + #[case::two_spellings_of_one_price( + json!({ + "input_cost_per_token": 1e-6, + "tiered_pricing": [{"range": [0, 10], "input_cost_per_token": 2e-6}] + }), + "/tiered_pricing/0/input_cost_per_token", + "duplicate_price" + )] + #[case::zero_band_is_the_base_price( + json!({"input_cost_per_token": 1e-6, "input_cost_per_token_above_0k_tokens": 2e-6}), + "/input_cost_per_token_above_0k_tokens", + "duplicate_price" + )] + #[case::tier_without_range( + json!({"tiered_pricing": [{"input_cost_per_token": 1e-6}]}), + "/tiered_pricing/0", + "tier_range" + )] + #[case::tier_with_suffix( + json!({"tiered_pricing": [{"range": [0, 10], "input_cost_per_token_priority": 1e-6}]}), + "/tiered_pricing/0/input_cost_per_token_priority", + "suffix_inside_block" + )] + #[case::reversed_range( + json!({"tiered_pricing": [{"range": [0, 10], "input_cost_per_token": 1e-6}, {"range": [10, 5], "input_cost_per_token": 1e-6}]}), + "/tiered_pricing/1", + "tier_reversed" + )] + #[case::gap_between_tiers( + json!({"tiered_pricing": [ + {"range": [0, 10], "input_cost_per_token": 1e-6}, + {"range": [20, 30], "input_cost_per_token": 2e-6} + ]}), + "/tiered_pricing/1", + "tier_gap" + )] + #[case::ladder_not_from_zero( + json!({"tiered_pricing": [{"range": [5, 10], "input_cost_per_token": 1e-6}]}), + "/tiered_pricing/0", + "tier_gap" + )] + #[case::result_ladder_overlaps( + json!({"tiered_pricing": [ + {"max_results_range": [1, 10], "input_cost_per_query": 1e-6}, + {"max_results_range": [10, 20], "input_cost_per_query": 2e-6} + ]}), + "/tiered_pricing/1", + "tier_gap" + )] + #[case::mixed_basis( + json!({"tiered_pricing": [ + {"range": [0, 10], "input_cost_per_token": 1e-6}, + {"max_results_range": [11, 20], "input_cost_per_query": 2e-6} + ]}), + "/tiered_pricing", + "mixed_tier_basis" + )] + #[case::result_tier_prices_a_token( + json!({"tiered_pricing": [{"max_results_range": [0, 10], "input_cost_per_token": 1e-6}]}), + "/tiered_pricing/0/input_cost_per_token", + "not_priced_by_results" + )] + #[case::two_band_sources( + json!({ + "output_cost_per_token_above_200k_tokens": 1e-6, + "tiered_pricing": [{"range": [0, 10], "input_cost_per_token": 1e-6}] + }), + "/output_cost_per_token_above_200k_tokens", + "bands_from_two_sources" + )] + #[case::off_peak_without_hours( + json!({"off_peak_pricing": {"input_cost_per_token": 1e-6}}), + "/off_peak_pricing", + "no_off_peak_hours" + )] + #[case::off_peak_without_prices( + json!({"off_peak_pricing": {"hours_utc": "00:00-01:00"}}), + "/off_peak_pricing", + "no_off_peak_prices" + )] + #[case::off_peak_empty_windows( + json!({"off_peak_pricing": {"windows": [], "input_cost_per_token": 1e-6}}), + "/off_peak_pricing/windows", + "empty_list" + )] + #[case::off_peak_empty_hours( + json!({"off_peak_pricing": {"hours_utc": [], "input_cost_per_token": 1e-6}}), + "/off_peak_pricing/hours_utc", + "empty_list" + )] + #[case::off_peak_empty_weekdays( + json!({"off_peak_pricing": { + "windows": [{"hours_utc": "00:00-01:00", "weekdays": []}], + "input_cost_per_token": 1e-6 + }}), + "/off_peak_pricing/windows/0/weekdays", + "empty_list" + )] + #[case::off_peak_image_price( + json!({"off_peak_pricing": {"hours_utc": "00:00-01:00", "output_cost_per_image": 1}}), + "/off_peak_pricing/output_cost_per_image", + "not_an_off_peak_rate" + )] + #[case::off_peak_band( + json!({"off_peak_pricing": {"hours_utc": "00:00-01:00", "input_cost_per_token_above_200k_tokens": 1}}), + "/off_peak_pricing/input_cost_per_token_above_200k_tokens", + "suffix_inside_block" + )] + fn rules_across_keys_name_where_each_price_came_from( + #[case] entry: Value, + #[case] expected_path: &str, + #[case] expected_variant: &str, + ) { + let problems = prices(entry).unwrap_err(); + assert!( + problems + .iter() + .any(|problem| problem.path() == expected_path + && <&str>::from(problem.kind()) == expected_variant), + "{problems:?}" + ); + } + + #[rstest] + fn off_peak_hours_and_windows_form_one_schedule_with_its_rates() { + let prices = prices(json!({"off_peak_pricing": { + "hours_utc": ["00:00-01:00"], + "windows": [{"hours_utc": "10:00-12:00", "weekdays": [6, " sun "]}], + "weekday_timezone": "Asia/Shanghai", + "output_cost_per_token": 1e-6, + "explicit_cache_read_input_token_cost": 2e-6 + }})) + .unwrap(); + let off_peak = prices.off_peak().unwrap(); + let window = |text: &str| text.parse::().unwrap(); + + assert_eq!( + off_peak.schedule(), + &Schedule { + windows: vec![ + Window { + hours: vec![window("00:00-01:00")], + weekdays: None, + }, + Window { + hours: vec![window("10:00-12:00")], + weekdays: Some(vec![Weekday::Saturday, Weekday::Sunday]), + }, + ], + weekday_timezone: Some("Asia/Shanghai".into()), + } + ); + assert_eq!( + off_peak.rates(), + &OffPeakRates { + output: UnitPrice::new(1e-6), + explicit_cache_read: UnitPrice::new(2e-6), + ..OffPeakRates::default() + } + ); + assert_eq!(prices.iter().count(), 0); + } + + #[rstest] + fn uplifts_are_adjustments_not_prices() { + let wire = wire(json!({ + "regional_endpoint_uplift_multiplier": 1.1, + "regional_processing_uplift_multiplier_eu": "1.2" + })); + let adjustments = Adjustments::from(&wire); + + assert_eq!(Prices::try_from(&wire).unwrap().iter().count(), 0); + assert_eq!( + adjustments + .get(&Uplift::NonGlobalEndpoint) + .map(Multiplier::get), + Some(1.1) + ); + assert_eq!( + adjustments + .get(&Uplift::Residency(Residency::Eu)) + .map(Multiplier::get), + Some(1.2) + ); + assert_eq!(adjustments.get(&Uplift::Residency(Residency::Us)), None); + } + + #[rstest] + fn a_model_reports_field_provider_and_price_problems_together() { + let Value::Object(fields) = json!({ + "mode": "chatt", + "tiered_pricing": [{"input_cost_per_token": 1e-6}] + }) else { + unreachable!() + }; + let problems: Vec = Model::from_fields(&fields) + .unwrap_err() + .iter() + .map(|problem| litellm_error::Report::new(problem).to_string()) + .collect(); + + assert_eq!(problems.len(), 3, "{problems:?}"); + assert!(problems[0].starts_with("mode: invalid value: unknown variant `chatt`")); + assert_eq!(problems[1], "litellm_provider: required"); + assert_eq!( + problems[2], + "tiered_pricing[0]: a tier needs exactly one of range and max_results_range" + ); + } + + #[rstest] + fn a_null_value_is_absent() { + let Value::Object(fields) = json!({ + "litellm_provider": "openai", + "input_cost_per_token": null, + "mode": null + }) else { + unreachable!() + }; + let model = Model::from_fields(&fields).unwrap(); + + assert_eq!(model.prices.iter().count(), 0); + assert_eq!(model.attributes.mode, None); + assert!(model.unrecognized.is_empty()); + } + + #[rstest] + #[case::tokens_win_over_characters( + json!({"litellm_provider": "vertex_ai-language-models", "input_cost_per_character": 1e-6, "input_cost_per_token": 1e-6}), + Billing::default() + )] + #[case::characters_only( + json!({"litellm_provider": "vertex_ai-language-models", "input_cost_per_character": 1e-6}), + Billing { text_unit: TextUnit::Character, ..Billing::default() } + )] + #[case::xai_band_is_inclusive( + json!({"litellm_provider": "xai", "input_cost_per_token": 1e-6}), + Billing { band_edge: BandEdge::Inclusive, ..Billing::default() } + )] + #[case::fireworks_without_cache_read_borrows_half( + json!({"litellm_provider": "fireworks_ai", "input_cost_per_token": 1e-6}), + Billing { cache_read_fallback: CacheReadFallback::HalfInputRate, ..Billing::default() } + )] + #[case::fireworks_with_cache_read_keeps_it( + json!({"litellm_provider": "fireworks_ai", "input_cost_per_token": 1e-6, "cache_read_input_token_cost": 1e-7}), + Billing::default() + )] + #[case::transcription_prices_the_audio_file( + json!({"litellm_provider": "openai", "mode": "audio_transcription", "input_cost_per_second": 1e-4}), + Billing { wall_clock: WallClock::AudioFile, ..Billing::default() } + )] + #[case::provisioned_chat_prices_response_time( + json!({"litellm_provider": "bedrock", "mode": "chat", "cost_per_second": 1e-2}), + Billing::default() + )] + #[case::image_generation_is_never_wall_clock( + json!({"litellm_provider": "openai", "mode": "image_generation"}), + Billing { wall_clock: WallClock::Never, ..Billing::default() } + )] + #[case::gemini_per_query_search( + json!({"litellm_provider": "gemini", "web_search_billing_unit": "per_query"}), + Billing { web_search: WebSearchUnit::PerQuery, ..Billing::default() } + )] + #[case::dashscope_tiers_bill_the_whole_call( + json!({"litellm_provider": "dashscope", "tiered_pricing": [{"range": [0, 32000], "input_cost_per_token": 1e-6}]}), + Billing { tier_scope: TierScope::WholeCall, ..Billing::default() } + )] + fn billing_is_lowered_from_the_row_and_its_provider( + #[case] entry: Value, + #[case] expected: Billing, + ) { + assert_eq!(prices(entry).unwrap().billing(), expected); + } +} diff --git a/litellm-rust/crates/model-catalog/src/model.rs b/litellm-rust/crates/model-catalog/src/model.rs new file mode 100644 index 00000000000..c1acd1c14cb --- /dev/null +++ b/litellm-rust/crates/model-catalog/src/model.rs @@ -0,0 +1,22 @@ +use std::collections::BTreeMap; + +use litellm_cost::price::{Adjustments, Prices}; + +use crate::error::Error; +use crate::wire::{Attributes, EntryWire, Fields}; + +/// One catalog entry read into types. +#[derive(Clone, Debug, PartialEq)] +pub struct Model { + pub attributes: Attributes, + pub prices: Prices, + pub adjustments: Adjustments, + pub capabilities: BTreeMap, + pub unrecognized: Vec, +} + +impl Model { + pub fn from_fields(fields: &Fields) -> Result> { + Self::try_from(EntryWire::read(fields)) + } +} diff --git a/litellm-rust/crates/model-catalog/src/model_info.rs b/litellm-rust/crates/model-catalog/src/model_info.rs index ebee2c64140..8f7549c1c6d 100644 --- a/litellm-rust/crates/model-catalog/src/model_info.rs +++ b/litellm-rust/crates/model-catalog/src/model_info.rs @@ -1,582 +1,194 @@ -use crate::capabilities::{AudioFormat, InputModality, Mode, OutputModality, VertexAiAudioApi}; -use crate::pricing::{OffPeakPricing, SearchContextCostPerQuery, TieredRate, WebSearchBillingUnit}; -use litellm_llms_types::formats::chat_completions::ReasoningEffort; -use serde::{Deserialize, Serialize}; +use indexmap::IndexMap; use serde_json::Value; -use std::collections::BTreeMap; -/// Typed mirror of one catalog model entry. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -pub struct ModelInfo { - #[serde(skip_serializing_if = "Option::is_none")] - pub annotation_cost_per_page: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub annotation_cost_per_page_batches: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub audio_transcription_config: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub bedrock_converse_supports_strict_tools: Option, - /// Highest reasoning effort the Bedrock output_config accepts for this model. - #[serde(skip_serializing_if = "Option::is_none")] - pub bedrock_output_config_effort_ceiling: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_creation_input_audio_token_cost: Option, - /// USD per token written to the provider's prompt cache. - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_creation_input_token_cost: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_creation_input_token_cost_above_32k_tokens: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_creation_input_token_cost_above_100k_tokens: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_creation_input_token_cost_above_100k_tokens_batches: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_creation_input_token_cost_above_128k_tokens: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_creation_input_token_cost_above_1hr: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_creation_input_token_cost_above_1hr_above_100k_tokens: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_creation_input_token_cost_above_1hr_above_200k_tokens: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_creation_input_token_cost_above_200k_tokens: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_creation_input_token_cost_above_200k_tokens_batches: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_creation_input_token_cost_above_256k_tokens: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_creation_input_token_cost_above_272k_tokens: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_creation_input_token_cost_above_272k_tokens_batches: Option, - /// Flex service-tier rate for the same-named base field. - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_creation_input_token_cost_above_272k_tokens_flex: Option, - /// Priority service-tier rate for the same-named base field. - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_creation_input_token_cost_above_272k_tokens_priority: Option, - /// Ultrafast service-tier rate for the same-named base field. - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_creation_input_token_cost_above_272k_tokens_ultrafast: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_creation_input_token_cost_batches: Option, - /// Flex service-tier rate for the same-named base field. - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_creation_input_token_cost_flex: Option, - /// Priority service-tier rate for the same-named base field. - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_creation_input_token_cost_priority: Option, - /// Ultrafast service-tier rate for the same-named base field. - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_creation_input_token_cost_ultrafast: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_read_input_audio_token_cost: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_read_input_image_token_cost: Option, - /// USD per prompt token served from the provider's prompt cache. - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_read_input_token_cost: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_read_input_token_cost_above_32k_tokens: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_read_input_token_cost_above_100k_tokens: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_read_input_token_cost_above_100k_tokens_batches: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_read_input_token_cost_above_128k_tokens: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_read_input_token_cost_above_200k_tokens: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_read_input_token_cost_above_200k_tokens_batches: Option, - /// Priority service-tier rate for the same-named base field. - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_read_input_token_cost_above_200k_tokens_priority: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_read_input_token_cost_above_256k_tokens: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_read_input_token_cost_above_272k_tokens: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_read_input_token_cost_above_272k_tokens_batches: Option, - /// Flex service-tier rate for the same-named base field. - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_read_input_token_cost_above_272k_tokens_flex: Option, - /// Priority service-tier rate for the same-named base field. - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_read_input_token_cost_above_272k_tokens_priority: Option, - /// Ultrafast service-tier rate for the same-named base field. - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_read_input_token_cost_above_272k_tokens_ultrafast: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_read_input_token_cost_above_512k_tokens: Option, - /// Balanced service-tier rate for the same-named base field. - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_read_input_token_cost_balanced: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_read_input_token_cost_batches: Option, - /// Flex service-tier rate for the same-named base field. - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_read_input_token_cost_flex: Option, - /// Priority service-tier rate for the same-named base field. - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_read_input_token_cost_priority: Option, - /// Ultrafast service-tier rate for the same-named base field. - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_read_input_token_cost_ultrafast: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub citation_cost_per_token: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub code_interpreter_cost_per_session: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub comment: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub computer_use_input_cost_per_1k_tokens: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub computer_use_output_cost_per_1k_tokens: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cost_per_second: Option, - /// Reasoning effort the provider applies when the request omits reasoning_effort. Gates whether a non-default temperature or the top_p/logprobs sampling params are accepted, which hold only when the effort resolves to 'none'. - #[serde(skip_serializing_if = "Option::is_none")] - pub default_reasoning_effort: Option, - /// Date the provider deprecates the model, YYYY-MM-DD. - #[serde(skip_serializing_if = "Option::is_none")] - pub deprecation_date: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub file_search_cost_per_1k_calls: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub file_search_cost_per_gb_per_day: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub gemini_audio_only_live: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub gemini_native_audio: Option, - /// USD per Grounding with Google Maps request; billed per query or per prompt per web_search_billing_unit. - #[serde(skip_serializing_if = "Option::is_none")] - pub google_maps_grounding_cost_per_query: Option, - /// USD cost per billable guardrail unit, keyed by the provider's usage counter name (e.g. Bedrock's contentPolicyUnits). - #[serde(skip_serializing_if = "Option::is_none")] - pub guardrail_cost_per_unit: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_audio_per_second: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_audio_per_second_above_128k_tokens: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_audio_token: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_audio_token_batches: Option, - /// Priority service-tier rate for the same-named base field. - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_audio_token_priority: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_character: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_character_above_128k_tokens: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_image: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_image_above_128k_tokens: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_image_token: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_image_token_batches: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_pixel: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_query: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_request: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_second: Option, - /// USD per prompt token. - #[serde(skip_serializing_if = "Option::is_none")] +/// The fields main's `ModelInfoBase(...)` copies (`litellm/utils.py` at c7c895db45), one typed field each, +/// read from the raw row without lowering it, so building one never fails +#[derive(Clone, Debug, Default, PartialEq)] +pub struct ModelInfoFields { + pub key: String, + pub max_tokens: Option, + pub max_input_tokens: Option, + pub max_output_tokens: Option, pub input_cost_per_token: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_token_above_32k_tokens: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_token_above_100k_tokens: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_token_above_100k_tokens_batches: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_token_above_128k_tokens: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_token_above_200k_tokens: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_token_above_200k_tokens_batches: Option, - /// Priority service-tier rate for the same-named base field. - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_token_above_200k_tokens_priority: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_token_above_256k_tokens: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_token_above_272k_tokens: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_token_above_272k_tokens_batches: Option, - /// Flex service-tier rate for the same-named base field. - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_token_above_272k_tokens_flex: Option, - /// Priority service-tier rate for the same-named base field. - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_token_above_272k_tokens_priority: Option, - /// Ultrafast service-tier rate for the same-named base field. - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_token_above_272k_tokens_ultrafast: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_token_above_512k_tokens: Option, - /// Balanced service-tier rate for the same-named base field. - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_token_balanced: Option, - /// USD per prompt token via the provider's batch API. - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_token_batches: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_token_cache_hit: Option, - /// Flex service-tier rate for the same-named base field. - #[serde(skip_serializing_if = "Option::is_none")] pub input_cost_per_token_flex: Option, - /// Priority service-tier rate for the same-named base field. - #[serde(skip_serializing_if = "Option::is_none")] pub input_cost_per_token_priority: Option, - /// Ultrafast service-tier rate for the same-named base field. - #[serde(skip_serializing_if = "Option::is_none")] + pub input_cost_per_token_balanced: Option, pub input_cost_per_token_ultrafast: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_video_per_second: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_video_per_second_above_128k_tokens: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_video_per_second_above_15s_interval: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_video_per_second_above_8s_interval: Option, - #[serde(skip_serializing_if = "Option::is_none")] + pub cache_creation_input_token_cost: Option, + pub cache_creation_input_token_cost_above_200k_tokens: Option, + pub cache_creation_input_token_cost_above_272k_tokens: Option, + pub cache_creation_input_token_cost_above_272k_tokens_priority: Option, + pub cache_creation_input_token_cost_above_272k_tokens_flex: Option, + pub cache_creation_input_token_cost_above_272k_tokens_ultrafast: Option, + pub cache_creation_input_token_cost_flex: Option, + pub cache_creation_input_token_cost_priority: Option, + pub cache_creation_input_token_cost_ultrafast: Option, + pub cache_read_input_token_cost: Option, + pub cache_read_input_audio_token_cost: Option, + pub cache_read_input_audio_token_cost_flex: Option, + pub cache_read_input_audio_token_cost_priority: Option, + pub cache_read_input_audio_token_cost_batches: Option, + pub explicit_cache_read_input_token_cost: Option, + pub prompt_cache_min_tokens: Option, + pub cache_read_input_token_cost_above_200k_tokens: Option, + pub cache_read_input_token_cost_above_200k_tokens_priority: Option, + pub cache_read_input_token_cost_above_272k_tokens: Option, + pub cache_read_input_token_cost_above_272k_tokens_priority: Option, + pub cache_read_input_token_cost_above_272k_tokens_flex: Option, + pub cache_read_input_token_cost_above_272k_tokens_ultrafast: Option, + pub cache_read_input_token_cost_above_512k_tokens: Option, + pub cache_read_input_token_cost_flex: Option, + pub cache_read_input_token_cost_priority: Option, + pub cache_read_input_token_cost_balanced: Option, + pub cache_read_input_token_cost_ultrafast: Option, + pub cache_read_input_token_cost_batches: Option, + pub cache_read_input_token_cost_above_200k_tokens_batches: Option, + pub cache_read_input_token_cost_above_272k_tokens_batches: Option, + pub cache_creation_input_token_cost_batches: Option, + pub cache_creation_input_token_cost_above_200k_tokens_batches: Option, + pub cache_creation_input_token_cost_above_272k_tokens_batches: Option, + pub cache_creation_input_token_cost_above_1hr: Option, + pub cache_creation_input_token_cost_above_12hr: Option, + pub cache_creation_input_token_cost_above_12hr_flex: Option, + pub cache_creation_input_token_cost_above_12hr_priority: Option, + pub cache_creation_input_token_cost_above_12hr_above_272k_tokens: Option, + pub cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex: Option, + pub cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority: Option, + pub cache_creation_input_token_cost_above_12hr_batches: Option, + pub cache_creation_input_token_cost_above_12hr_ultrafast: Option, + pub cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches: Option, + pub cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast: Option, + pub off_peak_pricing: Option, + pub input_cost_per_character: Option, + pub input_cost_per_token_above_128k_tokens: Option, + pub input_cost_per_token_above_200k_tokens: Option, + pub input_cost_per_token_above_200k_tokens_priority: Option, + pub input_cost_per_token_above_272k_tokens: Option, + pub input_cost_per_token_above_272k_tokens_priority: Option, + pub input_cost_per_token_above_272k_tokens_flex: Option, + pub input_cost_per_token_above_272k_tokens_ultrafast: Option, + pub input_cost_per_token_above_512k_tokens: Option, + pub input_cost_per_query: Option, + pub cost_per_second: Option, + pub input_cost_per_second: Option, + pub input_cost_per_text_message: Option, + pub input_cost_per_audio_token: Option, + pub input_cost_per_audio_token_flex: Option, + pub input_cost_per_audio_token_priority: Option, + pub input_cost_per_image_token: Option, pub input_cost_per_video_token: Option, - #[serde(skip_serializing_if = "Option::is_none")] + pub input_cost_per_audio_token_batches: Option, + pub input_cost_per_image_token_batches: Option, + pub input_cost_per_image: Option, + pub input_cost_per_audio_per_second: Option, + pub input_cost_per_video_per_second: Option, + pub input_cost_per_token_batches: Option, pub input_cost_per_video_token_batches: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub input_dbu_cost_per_token: Option, - /// LiteLLM provider slug; one of https://docs.litellm.ai/docs/providers. - #[serde(skip_serializing_if = "Option::is_none")] - pub litellm_provider: Option, - /// Maximum prompt/context tokens the model accepts. - #[serde(skip_serializing_if = "Option::is_none")] - pub max_input_tokens: Option, - /// Maximum tokens the model can generate in one response. - #[serde(skip_serializing_if = "Option::is_none")] - pub max_output_tokens: Option, - /// Legacy field: max output tokens if the provider specifies it, else max input tokens. - #[serde(skip_serializing_if = "Option::is_none")] - pub max_tokens: Option, - /// Free-form notes about the entry (e.g. pricing derivation). - #[serde(skip_serializing_if = "Option::is_none")] - pub metadata: Option>, - /// Primary API surface / task type of the model. - #[serde(skip_serializing_if = "Option::is_none")] - pub mode: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub ocr_cost_per_credit: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub ocr_cost_per_page: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub ocr_cost_per_page_batches: Option, - /// Rates that replace the same-named base fields while the request falls inside the stated UTC windows. - #[serde(skip_serializing_if = "Option::is_none")] - pub off_peak_pricing: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_audio_token: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_character: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_character_above_128k_tokens: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_image: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_image_1024: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_image_1536: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_image_512: Option, - #[serde( - rename = "output_cost_per_image_0.5K", - skip_serializing_if = "Option::is_none" - )] - pub output_cost_per_image_0_5k: Option, - #[serde( - rename = "output_cost_per_image_1K", - skip_serializing_if = "Option::is_none" - )] - pub output_cost_per_image_1k: Option, - #[serde( - rename = "output_cost_per_image_2K", - skip_serializing_if = "Option::is_none" - )] - pub output_cost_per_image_2k: Option, - #[serde( - rename = "output_cost_per_image_4K", - skip_serializing_if = "Option::is_none" - )] - pub output_cost_per_image_4k: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_image_token: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_pixel: Option, - /// USD per reasoning/thinking token, when billed separately. - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_reasoning_token: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_second: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_second_1080p: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_second_2k: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_second_480p: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_second_4k: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_second_720p: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_second_768p: Option, - /// USD per generated token. - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_token: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_token_above_32k_tokens: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_token_above_100k_tokens: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_token_above_100k_tokens_batches: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_token_above_128k_tokens: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_token_above_200k_tokens: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_token_above_200k_tokens_batches: Option, - /// Priority service-tier rate for the same-named base field. - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_token_above_200k_tokens_priority: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_token_above_256k_tokens: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_token_above_272k_tokens: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_token_above_272k_tokens_batches: Option, - /// Flex service-tier rate for the same-named base field. - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_token_above_272k_tokens_flex: Option, - /// Priority service-tier rate for the same-named base field. - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_token_above_272k_tokens_priority: Option, - /// Ultrafast service-tier rate for the same-named base field. - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_token_above_272k_tokens_ultrafast: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_token_above_512k_tokens: Option, - /// Balanced service-tier rate for the same-named base field. - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_token_balanced: Option, - /// USD per generated token via the provider's batch API. - #[serde(skip_serializing_if = "Option::is_none")] + pub input_cost_per_token_above_200k_tokens_batches: Option, + pub input_cost_per_token_above_272k_tokens_batches: Option, pub output_cost_per_token_batches: Option, - /// Flex service-tier rate for the same-named base field. - #[serde(skip_serializing_if = "Option::is_none")] + pub output_cost_per_token_above_200k_tokens_batches: Option, + pub output_cost_per_token_above_272k_tokens_batches: Option, + pub output_cost_per_token: Option, pub output_cost_per_token_flex: Option, - /// Priority service-tier rate for the same-named base field. - #[serde(skip_serializing_if = "Option::is_none")] pub output_cost_per_token_priority: Option, - /// Ultrafast service-tier rate for the same-named base field. - #[serde(skip_serializing_if = "Option::is_none")] + pub output_cost_per_token_balanced: Option, pub output_cost_per_token_ultrafast: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_video_per_second: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_video_token: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub output_dbu_cost_per_token: Option, - /// Embedding dimension for embedding models. - #[serde(skip_serializing_if = "Option::is_none")] - pub output_vector_size: Option, - /// Smallest prefix the provider will actually cache; absent means the provider default applies. - #[serde(skip_serializing_if = "Option::is_none")] - pub prompt_cache_min_tokens: Option, - /// Provider-internal routing hints (e.g. bedrock_invocation_schema). - #[serde(skip_serializing_if = "Option::is_none")] - pub provider_specific_entry: Option>, - /// Exact reasoning_effort levels this deployment accepts; wins over supports_* flags. - #[serde(skip_serializing_if = "Option::is_none")] - pub reasoning_effort_levels: Option>, - /// Multiplier applied to all token costs when served from a non-global Vertex AI endpoint (e.g. 1.10 = +10%). - #[serde(skip_serializing_if = "Option::is_none")] - pub regional_endpoint_uplift_multiplier: Option, - /// Multiplier applied to all token costs for EU data residency (e.g. 1.10 = +10%). - #[serde(skip_serializing_if = "Option::is_none")] pub regional_processing_uplift_multiplier_eu: Option, - /// Multiplier applied to all token costs for US data residency (e.g. 1.10 = +10%). - #[serde(skip_serializing_if = "Option::is_none")] pub regional_processing_uplift_multiplier_us: Option, - /// Provider default requests-per-minute limit. - #[serde(skip_serializing_if = "Option::is_none")] - pub rpm: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub rules: Option>, - /// USD cost per web search query, keyed by search context size. - #[serde(skip_serializing_if = "Option::is_none")] - pub search_context_cost_per_query: Option, - /// URL of the provider pricing/model page this entry was taken from. - #[serde(skip_serializing_if = "Option::is_none")] - pub source: Option, - /// Audio container formats the model can return. - #[serde(skip_serializing_if = "Option::is_none")] - pub supported_audio_formats: Option>, - /// OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions. - #[serde(skip_serializing_if = "Option::is_none")] + pub regional_endpoint_uplift_multiplier: Option, + pub output_cost_per_audio_token: Option, + pub output_cost_per_character: Option, + pub output_cost_per_reasoning_token: Option, + pub output_cost_per_reasoning_token_flex: Option, + pub output_cost_per_reasoning_token_priority: Option, + pub output_cost_per_token_above_128k_tokens: Option, + pub output_cost_per_character_above_128k_tokens: Option, + pub output_cost_per_token_above_200k_tokens: Option, + pub output_cost_per_token_above_200k_tokens_priority: Option, + pub output_cost_per_token_above_272k_tokens: Option, + pub output_cost_per_token_above_272k_tokens_priority: Option, + pub output_cost_per_token_above_272k_tokens_flex: Option, + pub output_cost_per_token_above_272k_tokens_ultrafast: Option, + pub output_cost_per_token_above_512k_tokens: Option, + pub output_cost_per_second: Option, + pub output_cost_per_second_1080p: Option, + pub output_cost_per_second_480p: Option, + pub output_cost_per_second_720p: Option, + pub output_cost_per_second_768p: Option, + pub output_cost_per_second_2k: Option, + pub output_cost_per_second_4k: Option, + pub output_cost_per_video_per_second: Option, + pub output_cost_per_image: Option, + pub output_cost_per_pixel: Option, + pub output_cost_per_image_token: Option, + pub output_cost_per_image_token_batches: Option, + pub output_cost_per_video_token: Option, + pub output_vector_size: Option, + pub citation_cost_per_token: Option, + pub tiered_pricing: Option, + pub litellm_provider: Option, + pub mode: Option, pub supported_endpoints: Option>, - /// Input modalities the model accepts. - #[serde(skip_serializing_if = "Option::is_none")] - pub supported_modalities: Option>, - /// Output modalities the model can produce. - #[serde(skip_serializing_if = "Option::is_none")] - pub supported_output_modalities: Option>, - /// Cloud regions the model is available in ('global' or region ids). - #[serde(skip_serializing_if = "Option::is_none")] - pub supported_regions: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_adaptive_thinking: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_anthropic_compaction: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_anthropic_thinking_payload: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_assistant_prefill: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_audio_input: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_audio_output: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_bedrock_runtime_chat_completions_response_format: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_bedrock_runtime_chat_completions_tools_with_reasoning: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_computer_use: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_embedding_image_input: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_fast_mode: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_forced_tool_use: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_function_calling: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_image_input: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_image_size: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_legacy_thinking: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_low_reasoning_effort: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_max_reasoning_effort: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_mid_conversation_system: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_minimal_reasoning_effort: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_multimodal: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_native_streaming: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_native_structured_output: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_none_reasoning_effort: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_nova_canvas_image_edit: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_output_config: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_parallel_function_calling: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_parallel_tool_use_config: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_pdf_input: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_prompt_cache_breakpoint: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_prompt_caching: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_reasoning: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_response_schema: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_sampling_params: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_speed: Option, - #[serde(skip_serializing_if = "Option::is_none")] pub supports_system_messages: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_thinking_cache_preservation: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_tool_choice: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_tool_search: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_url_context: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_video_input: Option, - #[serde(skip_serializing_if = "Option::is_none")] + pub supports_response_schema: Option, pub supports_vision: Option, - #[serde(skip_serializing_if = "Option::is_none")] + pub supports_function_calling: Option, + pub supports_parallel_function_calling: Option, + pub supports_tool_choice: Option, + pub supports_assistant_prefill: Option, + pub supports_prompt_caching: Option, + pub supports_prompt_cache_breakpoint: Option, + pub supports_thinking_cache_preservation: Option, + pub supports_audio_input: Option, + pub supports_video_input: Option, + pub supports_box_questions: Option, + pub supports_audio_output: Option, + pub supports_pdf_input: Option, + pub supports_embedding_image_input: Option, + pub supports_native_streaming: Option, + pub supports_native_structured_output: Option, + pub supports_bedrock_native_structured_output: Option, pub supports_web_search: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub supports_xhigh_reasoning_effort: Option, - #[serde(skip_serializing_if = "Option::is_none")] + pub supports_url_context: Option, + pub supports_reasoning: Option, + pub supports_adaptive_thinking: Option, + pub supports_legacy_thinking: Option, pub thinking_always_on: Option, - /// Context-length or result-count tiered rates; each tier's costs apply within its range. - #[serde(skip_serializing_if = "Option::is_none")] - pub tiered_pricing: Option>, - /// Provider default tokens-per-minute limit. - #[serde(skip_serializing_if = "Option::is_none")] - pub tpm: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub use_openai_responses_path: Option, - #[serde(skip_serializing_if = "Option::is_none")] + pub supports_tool_search: Option, + pub supports_mid_conversation_system: Option, + pub supports_anthropic_thinking_payload: Option, + pub supports_anthropic_compaction: Option, + pub supports_none_reasoning_effort: Option, + pub supports_minimal_reasoning_effort: Option, + pub supports_low_reasoning_effort: Option, + pub supports_xhigh_reasoning_effort: Option, + pub supports_max_reasoning_effort: Option, + pub reasoning_effort_levels: Option, + pub default_reasoning_effort: Option, + pub bedrock_output_config_effort_ceiling: Option, + pub bedrock_converse_supports_strict_tools: Option, + pub supports_regex_lookaround: Option, + pub supports_computer_use: Option, + pub search_context_cost_per_query: Option, + pub web_search_billing_unit: Option, + pub search_free_requests: Option, + pub google_maps_grounding_cost_per_query: Option, + pub tpm: Option, + pub rpm: Option, + pub ocr_cost_per_page: Option, + pub ocr_cost_per_page_batches: Option, + pub ocr_cost_per_credit: Option, + pub annotation_cost_per_page: Option, + pub annotation_cost_per_page_batches: Option, + pub provider_specific_entry: Option, pub uses_embed_content: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub vector_store_cost_per_gb_per_day: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub vertex_ai_audio_api: Option, - /// Whether web search is billed per query or per prompt. - #[serde(skip_serializing_if = "Option::is_none")] - pub web_search_billing_unit: Option, + pub supports_image_size: Option, + pub supported_audio_formats: Option, + pub token_count_framing: Option, + pub token_count_model: Option, + pub token_count_tokenizer: Option, + pub vertex_ai_audio_api: Option, + /// Every key matching `ABOVE_THRESHOLD_COST_KEY_PATTERN`, copied as written + pub above_threshold_costs: IndexMap, } diff --git a/litellm-rust/crates/model-catalog/src/pricing.rs b/litellm-rust/crates/model-catalog/src/pricing.rs deleted file mode 100644 index b8ed652451c..00000000000 --- a/litellm-rust/crates/model-catalog/src/pricing.rs +++ /dev/null @@ -1,97 +0,0 @@ -use serde::{Deserialize, Serialize}; - -/// Whether web search is billed per query or per prompt. -#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(rename_all = "snake_case")] -pub enum WebSearchBillingUnit { - PerQuery, - PerPrompt, -} - -/// UTC "HH:MM-HH:MM" window, or a list of them; a window may wrap past midnight. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(untagged)] -pub enum UtcHours { - Single(String), - Multiple(Vec), -} - -/// ISO-8601 weekday number (1 = Monday .. 7 = Sunday) or English day name. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(untagged)] -pub enum Weekday { - Number(u8), - Name(String), -} - -/// One off-peak window entry inside `windows`. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(deny_unknown_fields)] -pub struct OffPeakWindow { - pub hours_utc: UtcHours, - #[serde(skip_serializing_if = "Option::is_none")] - pub weekdays: Option>, -} - -/// Rates that replace the same-named base fields inside the stated UTC windows. -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(deny_unknown_fields)] -pub struct OffPeakPricing { - #[serde(skip_serializing_if = "Option::is_none")] - pub hours_utc: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub windows: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub weekday_timezone: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_token: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_token: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_reasoning_token: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_read_input_token_cost: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_creation_input_token_cost: Option, -} - -/// USD cost per web search query, keyed by search context size. -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(deny_unknown_fields)] -pub struct SearchContextCostPerQuery { - #[serde(skip_serializing_if = "Option::is_none")] - pub search_context_size_low: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub search_context_size_medium: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub search_context_size_high: Option, -} - -/// One tier of a context-length or result-count tiered rate. -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(deny_unknown_fields)] -pub struct TieredRate { - #[serde(skip_serializing_if = "Option::is_none")] - pub range: Option<[f64; 2]>, - #[serde(skip_serializing_if = "Option::is_none")] - pub max_results_range: Option<[f64; 2]>, - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_token: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_token: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub output_cost_per_reasoning_token: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_read_input_token_cost: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_creation_input_token_cost: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub input_cost_per_query: Option, -} diff --git a/litellm-rust/crates/model-catalog/src/register.rs b/litellm-rust/crates/model-catalog/src/register.rs new file mode 100644 index 00000000000..ed34082ae57 --- /dev/null +++ b/litellm-rust/crates/model-catalog/src/register.rs @@ -0,0 +1,26 @@ +use std::collections::{BTreeSet, HashMap}; + +use indexmap::IndexMap; + +use crate::wire::Fields; + +/// Where one `register` call writes and which row it inherits from +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RegistrationTarget { + /// Always the caller's key, never a case variant of it + pub write: String, + /// The first candidate that is an exact key whose row passes `provider_matches` + pub inherit_from: Option, +} + +/// Python's inheritance candidates in main's order, `get_model_info`'s then `_resolve_builtin_model_cost_entry`'s +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RegistrationCandidates(pub Vec); + +/// Layer 3, and the fields `register` last wrote, which litellm owns again +#[derive(Clone, Debug, Default)] +#[expect(dead_code, reason = "07 adds the functions that read registrations")] +pub struct Registrations { + persisted: IndexMap, + registered_fields: HashMap>, +} diff --git a/litellm-rust/crates/model-catalog/src/resolve/image.rs b/litellm-rust/crates/model-catalog/src/resolve/image.rs new file mode 100644 index 00000000000..d24e170b32c --- /dev/null +++ b/litellm-rust/crates/model-catalog/src/resolve/image.rs @@ -0,0 +1,18 @@ +/// What an image call asks for. `resolve_image` builds main's eight key spellings from it +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ImageCandidates { + pub model: String, + pub provider: Option, + /// Width and height, `1024 x 1024` when the request names none, as on main + pub size: Option<(u32, u32)>, + pub quality: Option, +} + +/// A size and quality keyed row, `{quality}/{W}-x-{H}/{model}`, parsed once per index build for fal's nearest match +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ImageKey { + pub quality: Option, + pub width: u32, + pub height: u32, + pub model: String, +} diff --git a/litellm-rust/crates/model-catalog/src/resolve/mod.rs b/litellm-rust/crates/model-catalog/src/resolve/mod.rs new file mode 100644 index 00000000000..fbaead622aa --- /dev/null +++ b/litellm-rust/crates/model-catalog/src/resolve/mod.rs @@ -0,0 +1,46 @@ +mod image; + +pub use image::*; + +/// The seven names main's `get_model_info` tries, built by Python, walked in this order by Rust +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Candidates { + pub combined_model_name: Option, + pub model: String, + pub region_free_combined_model_name: Option, + pub split_model: Option, + pub combined_stripped_model_name: Option, + pub stripped_model_name: Option, + pub provider_prefixed_model_name: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CandidateSlot { + CombinedModelName, + Model, + RegionFreeCombinedModelName, + SplitModel, + CombinedStrippedModelName, + StrippedModelName, + ProviderPrefixedModelName, +} + +/// Which step matched, so a test can name the step that changed +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Via { + pub slot: CandidateSlot, + /// Matched through the case-insensitive index rather than an exact key + pub case_insensitive: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Resolution { + Found { + key: String, + via: Via, + }, + /// Every candidate tried, in order + Missing { + tried: Vec, + }, +} diff --git a/litellm-rust/crates/model-catalog/src/schema.rs b/litellm-rust/crates/model-catalog/src/schema.rs index 7de988b3ff3..66f9933a3e3 100644 --- a/litellm-rust/crates/model-catalog/src/schema.rs +++ b/litellm-rust/crates/model-catalog/src/schema.rs @@ -1,9 +1,80 @@ -use schemars::Schema; +use std::collections::BTreeMap; + +use litellm_cost::money::{Multiplier, UnitPrice}; +use schemars::{JsonSchema, Schema}; use serde_json::{Map, Value, json}; +use crate::fallback::FallbackGeneralizations; +use crate::wire::{Attributes, OffPeakWindow, SearchContextCostPerQuery, UtcHours}; + +#[derive(JsonSchema)] +#[schemars(rename = "ModelEntry")] +struct ModelEntry { + #[serde(flatten)] + _attributes: Attributes, + #[serde(flatten)] + _blocks: PriceBlocks, +} + +#[derive(JsonSchema)] +#[allow(dead_code)] +struct PriceBlocks { + /// USD cost per billable guardrail unit, keyed by the provider's usage counter name (e.g. Bedrock's contentPolicyUnits). + guardrail_cost_per_unit: Option>, + /// Rates that replace the same-named base fields while the request falls inside the stated UTC windows. + off_peak_pricing: Option, + /// Multiplier applied to all token costs when served from a non-global Vertex AI endpoint (e.g. 1.10 = +10%). + regional_endpoint_uplift_multiplier: Option, + /// Multiplier applied to all token costs for EU data residency (e.g. 1.10 = +10%). + regional_processing_uplift_multiplier_eu: Option, + /// Multiplier applied to all token costs for US data residency (e.g. 1.10 = +10%). + regional_processing_uplift_multiplier_us: Option, + /// USD cost per web search query, keyed by search context size. + search_context_cost_per_query: Option, + /// Context-length or result-count tiered rates; each tier's costs apply within its range. + tiered_pricing: Option>, +} + +/// Rates that replace the same-named base fields inside the stated UTC windows. +#[derive(JsonSchema)] +#[allow(dead_code)] +struct OffPeakPricing { + hours_utc: Option, + windows: Option>, + weekday_timezone: Option, +} + +/// One tier of a context-length or result-count tiered rate. +#[derive(JsonSchema)] +#[allow(dead_code)] +struct TieredRate { + range: Option<[u64; 2]>, + max_results_range: Option<[u64; 2]>, +} + +#[derive(JsonSchema)] +#[allow(dead_code)] +struct RegistryDocument { + sample_spec: Option>, + fallback_generalizations: Option, +} + +pub(crate) const FLAT_KEY_PATTERN: &str = concat!( + r"^(", + r"input_cost_per_((audio_|image_|video_)?token|character|image|pixel|query|request|second|text_message|audio_per_second|video_per_second(_above_\d+s_interval)?)", + r"|output_cost_per_((audio_|image_|video_|reasoning_)?token|character|image(_\d+(\.\d+)?K?)?|pixel|second(_\d+[pk])?|video_per_second)", + r"|cache_read_input_(audio_|image_|video_)?token_cost", + r"|explicit_cache_read_input_token_cost", + r"|cache_creation_input_(audio_|image_|video_)?token_cost(_above_1hr|_above_12hr)?", + r"|annotation_cost_per_page|citation_cost_per_token|code_interpreter_cost_per_session", + r"|google_maps_grounding_cost_per_query|ocr_cost_per_credit|ocr_cost_per_page|cost_per_second", + r"|browser_open_cost_per_request|cost_per_additional_result", + r")(_above_\d+k?_tokens)?(_(priority|flex|ultrafast|balanced|batches|decisions))?$", +); + /// JSON Schema for one model entry, including registry validation constraints. pub fn model_entry_json_schema() -> Schema { - let mut schema = serde_json::to_value(schemars::schema_for!(crate::ModelInfo)) + let mut schema = serde_json::to_value(schemars::schema_for!(ModelEntry)) .expect("derived model schema serializes"); remove_nullable_optional_fields(&mut schema); decorate_model_entry(&mut schema); @@ -22,24 +93,21 @@ pub fn registry_json_schema() -> Schema { entry.as_object_mut().unwrap().remove("$schema"); definitions.insert("modelEntry".into(), entry); - let mut fallback = serde_json::to_value(schemars::schema_for!(crate::FallbackGeneralizations)) - .expect("derived fallback schema serializes"); - remove_nullable_optional_fields(&mut fallback); - definitions.extend(take_definitions(&mut fallback)); - fallback.as_object_mut().unwrap().remove("$schema"); - - let root = json!({ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "LiteLLM model prices and context window registry", - "type": "object", - "properties": { - "sample_spec": {"type": "object"}, - "fallback_generalizations": fallback - }, - "additionalProperties": {"$ref": "#/$defs/modelEntry"}, - "$defs": definitions - }); - Schema::from(root.as_object().unwrap().clone()) + let mut registry = serde_json::to_value(schemars::schema_for!(RegistryDocument)) + .expect("derived registry schema serializes"); + remove_nullable_optional_fields(&mut registry); + definitions.extend(take_definitions(&mut registry)); + let root = registry.as_object_mut().unwrap(); + root.insert( + "title".into(), + "LiteLLM model prices and context window registry".into(), + ); + root.insert( + "additionalProperties".into(), + json!({"$ref": "#/$defs/modelEntry"}), + ); + root.insert("$defs".into(), Value::Object(definitions)); + Schema::from(root.clone()) } fn take_definitions(schema: &mut Value) -> Map { @@ -80,42 +148,21 @@ fn remove_nullable_optional_fields(value: &mut Value) { } fn decorate_model_entry(schema: &mut Value) { + let rates = json!({FLAT_KEY_PATTERN: {"type": "number", "minimum": 0}}); let object = schema.as_object_mut().unwrap(); object.insert("required".into(), json!(["litellm_provider"])); object.insert("additionalProperties".into(), Value::Bool(true)); - let properties = object - .get_mut("properties") - .unwrap() - .as_object_mut() - .unwrap(); - properties.insert( - "aliases".into(), - json!({"type": "array", "items": {"type": "string"}}), + object.insert( + "patternProperties".into(), + json!({ + FLAT_KEY_PATTERN: {"type": "number", "minimum": 0}, + "^supports_": {"type": "boolean"}, + }), ); - properties.get_mut("deprecation_date").unwrap()["format"] = json!("date"); - properties.get_mut("deprecation_date").unwrap()["pattern"] = - json!(r"^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])$"); - - properties.iter_mut().for_each(|(name, property)| { - if name.contains("cost") { - property["minimum"] = json!(0); - } else if name.contains("uplift_multiplier") { - property["minimum"] = json!(1); - } - }); - properties.get_mut("guardrail_cost_per_unit").unwrap()["additionalProperties"]["minimum"] = - json!(0); - let definitions = object.get_mut("$defs").unwrap().as_object_mut().unwrap(); - for definition in ["OffPeakPricing", "TieredRate", "SearchContextCostPerQuery"] { - let properties = definitions[definition]["properties"] - .as_object_mut() - .unwrap(); - properties.iter_mut().for_each(|(name, property)| { - if name.contains("cost") || definition == "SearchContextCostPerQuery" { - property["minimum"] = json!(0); - } - }); + for definition in ["OffPeakPricing", "TieredRate"] { + definitions[definition]["patternProperties"] = rates.clone(); + definitions[definition]["additionalProperties"] = Value::Bool(false); } definitions["OffPeakPricing"]["anyOf"] = json!([ {"required": ["hours_utc"]}, @@ -123,15 +170,43 @@ fn decorate_model_entry(schema: &mut Value) { ]); definitions["OffPeakPricing"]["properties"]["windows"]["minItems"] = json!(1); definitions["OffPeakWindow"]["properties"]["weekdays"]["minItems"] = json!(1); - definitions["TieredRate"]["properties"]["range"]["items"]["minimum"] = json!(0); - definitions["TieredRate"]["properties"]["max_results_range"]["items"]["minimum"] = json!(0); - definitions["Weekday"]["anyOf"][0]["minimum"] = json!(1); - definitions["Weekday"]["anyOf"][0]["maximum"] = json!(7); - definitions["Weekday"]["anyOf"][1]["pattern"] = json!( - r"(?i)^(mon|monday|tue|tues|tuesday|wed|wednesday|thu|thur|thurs|thursday|fri|friday|sat|saturday|sun|sunday)$" - ); - let window_pattern = json!(r"^([01]\d|2[0-3]):[0-5]\d-([01]\d|2[0-3]):[0-5]\d$"); - definitions["UtcHours"]["anyOf"][0]["pattern"] = window_pattern.clone(); - definitions["UtcHours"]["anyOf"][1]["items"]["pattern"] = window_pattern; definitions["UtcHours"]["anyOf"][1]["minItems"] = json!(1); } + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::FLAT_KEY_PATTERN; + use crate::wire::FlatKey; + + #[rstest] + #[case::input_token("input_cost_per_token")] + #[case::cache_write_one_hour_band( + "cache_creation_input_token_cost_above_1hr_above_200k_tokens" + )] + #[case::image_label("output_cost_per_image_0.5K")] + #[case::resolution_tier("output_cost_per_second_4k_flex")] + #[case::feature_batch("annotation_cost_per_page_batches")] + #[case::video_interval("input_cost_per_video_per_second_above_8s_interval")] + #[case::explicit_cache_read("explicit_cache_read_input_token_cost")] + #[case::wall_clock("cost_per_second")] + #[case::unused_meter("cost_per_token")] + #[case::other_currency("input_dbu_cost_per_token")] + #[case::typo("input_cost_per_tokne")] + #[case::read_has_no_ttl("cache_read_input_token_cost_above_1hr")] + #[case::interval_on_a_token_meter("input_cost_per_token_above_8s_interval")] + fn flat_key_pattern_agrees_with_the_parser(#[case] key: &str) { + let schema = serde_json::json!({ + "type": "object", + "patternProperties": {FLAT_KEY_PATTERN: {}}, + "additionalProperties": false, + }); + let validator = jsonschema::validator_for(&schema).unwrap(); + + assert_eq!( + validator.is_valid(&serde_json::json!({key: 1})), + FlatKey::parse(key).is_some() + ); + } +} diff --git a/litellm-rust/crates/model-catalog/src/store/AGENTS.md b/litellm-rust/crates/model-catalog/src/store/AGENTS.md new file mode 100644 index 00000000000..f1dacd2e217 --- /dev/null +++ b/litellm-rust/crates/model-catalog/src/store/AGENTS.md @@ -0,0 +1,35 @@ +# Store + +`Store` is an in-memory store, a small database built for exactly three queries. It knows nothing about the catalog. A source picks its own `Rows::Row` type, how names fold for matching, and the group each key is listed under. The catalog's sources fold to lowercase and group by raw `litellm_provider` + +| Query | Method | Real caller | +|---|---|---| +| row by exact key | `Store::get` | reading `litellm.model_cost["gpt-4o"]` | +| actual key for a name that folds to the same form | `Store::find_key` | `get_model_info("GPT-4o")` finding `gpt-4o` | +| keys in a group | `Store::keys_in` | the provider sets, `litellm.open_ai_chat_completion_models` | + +## Rules + +- nothing in `store/` imports from the rest of the crate. Wire types, case rules and field names such as `litellm_provider` belong to the source that implements `Rows` +- the index in `name_index.rs` is private to the store. Never build a folded-name map or a provider grouping anywhere else in the workspace. A new lookup pattern is a new `Store` method backed by `NameIndex` +- the index holds keys, never rows. Rows are always read from `Rows`, so a price edit is seen on the next read and never rebuilds the index +- `find_key` and `keys_in` never return a key the rows lack. A hit on a removed key rebuilds the index once and asks again +- `keys_in` matches the group exactly as the source wrote it. Provider aliases such as `vertex_ai` over `vertex_ai-language-models` are Python's job +- when two keys fold to the same name the later key in row order wins, like main's `_model_cost_lowercase_map` + +## Stamps + +`Rows::stamp` is how the store learns that the index may be stale. The store keeps the stamp it built the index at and rebuilds when the source returns a different one. A source must change its stamp whenever a key or a key's group changes, since those are the two things the index holds + +| Source | Stamp | Blind spot | +|---|---|---| +| `Catalog`'s `CatalogRows` | `()`, every change goes through `Store::edit`, which drops the index | none | +| python-bridge's bound `litellm.model_cost` | the dict object and its `len` | a delete plus an add between two reads, or a `litellm_provider` rewrite on an existing key | + +A source with a blind spot calls `Store::invalidate` when it knows keys changed. Until then a key added in the blind spot is not found, and a key moved to another provider stays listed under the old one. Removed keys are still never returned + +## Tests + +Tests live in `mod.rs`'s `tests` module and drive the store through `ModelCost`, a fake `litellm.model_cost` with real bundled keys whose row is just its `litellm_provider`. Each Python write is a method (`set`, `del`, `rebind`) and its stamp is what python-bridge can see. Add a scenario as a named test that reads like the Python user code it stands for + +The two property tests compare every answer with a linear scan of the same dict. With a stamp that sees every key change, answers must match exactly. With the dict stamp, every answer must be sound (present, same folded name), and `invalidate` must make them match again. A store change that fails either is a bug in the store, not in the test diff --git a/litellm-rust/crates/model-catalog/src/store/mod.rs b/litellm-rust/crates/model-catalog/src/store/mod.rs new file mode 100644 index 00000000000..2220526f8b6 --- /dev/null +++ b/litellm-rust/crates/model-catalog/src/store/mod.rs @@ -0,0 +1,460 @@ +mod name_index; + +use std::borrow::Cow; +use std::fmt; +use std::sync::{PoisonError, RwLock}; + +use name_index::NameIndex; + +/// Where a `Store`'s rows live. The store rebuilds its index whenever `stamp` changes, so a source must change its +/// stamp whenever a key or a key's group changes, or call `Store::invalidate` when it cannot +pub trait Rows { + type Row: ToOwned + ?Sized; + type Stamp: PartialEq; + + /// The name two spellings of one key share, which `Store::find_key` matches on + fn fold(name: &str) -> String; + fn stamp(&self) -> Self::Stamp; + fn contains(&self, key: &str) -> bool; + fn row(&self, key: &str) -> Option>; + /// Every key in row order with the group it is listed under + fn scan(&self, visit: &mut dyn FnMut(&str, Option<&str>)); +} + +/// An in-memory store for three queries (row by exact key, actual key for a name by its folded form, keys in a group) +/// that owns the only index behind them +pub struct Store { + rows: R, + index: RwLock>>, +} + +struct Built { + stamp: S, + names: NameIndex, +} + +impl Store { + pub fn new(rows: R) -> Self { + Self { + rows, + index: RwLock::new(None), + } + } + + pub fn rows(&self) -> &R { + &self.rows + } + + /// Changes the rows in place. The index is rebuilt on the next query + pub fn edit(&mut self, edit: impl FnOnce(&mut R) -> T) -> T { + let edited = edit(&mut self.rows); + *self.index.get_mut().unwrap_or_else(PoisonError::into_inner) = None; + edited + } + + pub fn get(&self, key: &str) -> Option> { + self.rows.row(key) + } + + /// The key itself when present, otherwise the actual key that folds to the same name. Never a key the rows lack + pub fn find_key(&self, name: &str) -> Option { + if self.rows.contains(name) { + return Some(name.to_owned()); + } + let folded = R::fold(name); + let hit = self.query(|names| names.key_for(&folded).map(str::to_owned))?; + if self.rows.contains(&hit) { + return Some(hit); + } + self.invalidate(); + self.query(|names| names.key_for(&folded).map(str::to_owned)) + .filter(|key| self.rows.contains(key)) + } + + /// Keys whose group is exactly `group`, in row order. Never a key the rows lack + pub fn keys_in(&self, group: &str) -> Vec { + self.query(|names| names.keys_in(group).to_vec()) + .into_iter() + .filter(|key| self.rows.contains(key)) + .collect() + } + + /// Drops the index, for a source whose stamp cannot see every key change + pub fn invalidate(&self) { + *self.index.write().unwrap_or_else(PoisonError::into_inner) = None; + } + + fn query(&self, read: impl Fn(&NameIndex) -> T) -> T { + let stamp = self.rows.stamp(); + if let Some(built) = self + .index + .read() + .unwrap_or_else(PoisonError::into_inner) + .as_ref() + .filter(|built| built.stamp == stamp) + { + return read(&built.names); + } + let names = NameIndex::scan(&self.rows); + let answer = read(&names); + *self.index.write().unwrap_or_else(PoisonError::into_inner) = Some(Built { stamp, names }); + answer + } +} + +impl Clone for Store { + fn clone(&self) -> Self { + Self::new(self.rows.clone()) + } +} + +impl fmt::Debug for Store { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Store") + .field("rows", &self.rows) + .finish_non_exhaustive() + } +} + +#[cfg(test)] +mod tests { + use std::cell::{Cell, RefCell}; + + use indexmap::IndexMap; + use rstest::{fixture, rstest}; + + use super::*; + + const BUNDLED: &[(&str, Option<&str>)] = &[ + ("gpt-4o", Some("openai")), + ("o3-pro", Some("openai")), + ("azure/gpt-4o", Some("azure")), + ("claude-sonnet-4-5", Some("anthropic")), + ("gemini-2.5-pro", Some("vertex_ai-language-models")), + ( + "fireworks_ai/accounts/fireworks/models/llama-v3p1-8b-instruct", + Some("fireworks_ai"), + ), + ( + "anyscale/meta-llama/Meta-Llama-3-8B-Instruct", + Some("anyscale"), + ), + ("sample_free_row", None), + ]; + + /// `litellm.model_cost` as Python sees it. The stamp is what python-bridge can observe for free, the dict's + /// identity and its length, so a write that keeps both hides a key change from the store + struct ModelCost { + dict: RefCell>>, + identity: Cell, + index_changes: Cell, + honest_stamp: bool, + scans: Cell, + } + + impl ModelCost { + fn new(rows: &[(&str, Option<&str>)], honest_stamp: bool) -> Self { + Self { + dict: RefCell::new( + rows.iter() + .map(|(key, provider)| ((*key).to_owned(), provider.map(str::to_owned))) + .collect(), + ), + identity: Cell::new(0), + index_changes: Cell::new(0), + honest_stamp, + scans: Cell::new(0), + } + } + + /// `model_cost[key] = {"litellm_provider": provider, ...}` + fn set(&self, key: &str, provider: Option<&str>) { + let previous = self + .dict + .borrow_mut() + .insert(key.to_owned(), provider.map(str::to_owned)); + if previous.as_ref().map(Option::as_deref) != Some(provider) { + self.index_changes.set(self.index_changes.get() + 1); + } + } + + /// `del model_cost[key]` + fn del(&self, key: &str) { + if self.dict.borrow_mut().shift_remove(key).is_some() { + self.index_changes.set(self.index_changes.get() + 1); + } + } + + /// `litellm.model_cost = {...}`, a new dict object + fn rebind(&self, rows: &[(&str, Option<&str>)]) { + *self.dict.borrow_mut() = rows + .iter() + .map(|(key, provider)| ((*key).to_owned(), provider.map(str::to_owned))) + .collect(); + self.identity.set(self.identity.get() + 1); + self.index_changes.set(self.index_changes.get() + 1); + } + } + + /// A row is its `litellm_provider`, which is also the group the store lists its key under + impl Rows for &ModelCost { + type Row = Option; + type Stamp = (u64, u64); + + fn fold(name: &str) -> String { + name.to_lowercase() + } + + fn stamp(&self) -> Self::Stamp { + if self.honest_stamp { + (self.identity.get(), self.index_changes.get()) + } else { + (self.identity.get(), self.dict.borrow().len() as u64) + } + } + + fn contains(&self, key: &str) -> bool { + self.dict.borrow().contains_key(key) + } + + fn row(&self, key: &str) -> Option>> { + Some(Cow::Owned(self.dict.borrow().get(key)?.clone())) + } + + fn scan(&self, visit: &mut dyn FnMut(&str, Option<&str>)) { + self.scans.set(self.scans.get() + 1); + for (key, provider) in self.dict.borrow().iter() { + visit(key, provider.as_deref()); + } + } + } + + #[fixture] + fn model_cost() -> ModelCost { + ModelCost::new(BUNDLED, false) + } + + /// Rows only `Store::edit` can change, so the stamp never moves + #[derive(Debug)] + struct Owned(IndexMap>); + + impl Rows for Owned { + type Row = Option; + type Stamp = (); + + fn fold(name: &str) -> String { + name.to_lowercase() + } + + fn stamp(&self) -> Self::Stamp {} + + fn contains(&self, key: &str) -> bool { + self.0.contains_key(key) + } + + fn row(&self, key: &str) -> Option>> { + self.0.get(key).map(Cow::Borrowed) + } + + fn scan(&self, visit: &mut dyn FnMut(&str, Option<&str>)) { + for (key, provider) in &self.0 { + visit(key, provider.as_deref()); + } + } + } + + fn owned(rows: &[(&str, Option<&str>)]) -> Store { + Store::new(Owned( + rows.iter() + .map(|(key, provider)| ((*key).to_owned(), provider.map(str::to_owned))) + .collect(), + )) + } + + #[rstest] + #[case::bundled_key_as_written("gpt-4o", Some("gpt-4o"))] + #[case::user_typed_upper_case("GPT-4o", Some("gpt-4o"))] + #[case::provider_prefixed("Azure/GPT-4o", Some("azure/gpt-4o"))] + #[case::fireworks_path_in_another_case( + "fireworks_ai/accounts/fireworks/models/Llama-v3p1-8B-Instruct", + Some("fireworks_ai/accounts/fireworks/models/llama-v3p1-8b-instruct") + )] + #[case::mixed_case_bundled_key( + "anyscale/meta-llama/meta-llama-3-8b-instruct", + Some("anyscale/meta-llama/Meta-Llama-3-8B-Instruct") + )] + #[case::unmapped_model("my-finetune", None)] + #[case::prefix_is_not_a_match("gpt-4", None)] + fn find_key_maps_a_requested_model_to_its_key( + model_cost: ModelCost, + #[case] model: &str, + #[case] expected: Option<&str>, + ) { + assert_eq!(Store::new(&model_cost).find_key(model).as_deref(), expected); + } + + #[rstest] + #[case::the_bundled_spelling("gpt-4o", "gpt-4o")] + #[case::the_registered_spelling("GPT-4o", "GPT-4o")] + #[case::another_spelling_takes_the_later_key("Gpt-4O", "GPT-4o")] + fn a_registered_case_variant_is_its_own_row( + model_cost: ModelCost, + #[case] model: &str, + #[case] expected: &str, + ) { + model_cost.set("GPT-4o", Some("openai")); + assert_eq!( + Store::new(&model_cost).find_key(model).as_deref(), + Some(expected) + ); + } + + #[rstest] + #[case::openai_in_map_order("openai", &["gpt-4o", "o3-pro"])] + #[case::azure("azure", &["azure/gpt-4o"])] + #[case::raw_vertex_family("vertex_ai-language-models", &["gemini-2.5-pro"])] + #[case::provider_alias_is_python_s_job("vertex_ai", &[])] + #[case::matched_as_written("OpenAI", &[])] + #[case::unknown_provider("acme", &[])] + fn keys_in_lists_the_keys_a_raw_provider_names( + model_cost: ModelCost, + #[case] provider: &str, + #[case] expected: &[&str], + ) { + assert_eq!(Store::new(&model_cost).keys_in(provider), expected); + } + + #[rstest] + fn a_model_written_straight_into_the_dict_is_found_in_any_case(model_cost: ModelCost) { + let store = Store::new(&model_cost); + assert_eq!(store.find_key("mycorp/custom-model"), None); + + model_cost.set("MyCorp/Custom-Model", Some("openai")); + + assert_eq!( + store.find_key("mycorp/custom-model").as_deref(), + Some("MyCorp/Custom-Model") + ); + assert_eq!( + store.keys_in("openai"), + ["gpt-4o", "o3-pro", "MyCorp/Custom-Model"] + ); + } + + #[rstest] + fn a_price_edit_reuses_the_index(model_cost: ModelCost) { + let store = Store::new(&model_cost); + assert_eq!(store.find_key("GPT-4o").as_deref(), Some("gpt-4o")); + + model_cost.set("gpt-4o", Some("openai")); + model_cost.set("o3-pro", Some("openai")); + model_cost.set("gpt-4o", Some("openai")); + + assert_eq!(store.find_key("O3-PRO").as_deref(), Some("o3-pro")); + assert_eq!(store.keys_in("openai"), ["gpt-4o", "o3-pro"]); + assert_eq!(model_cost.scans.get(), 1); + } + + #[rstest] + fn a_reload_that_rebinds_the_dict_is_seen_even_at_the_same_length(model_cost: ModelCost) { + let store = Store::new(&model_cost); + assert_eq!(store.find_key("GPT-4o").as_deref(), Some("gpt-4o")); + + let mut reloaded = BUNDLED.to_vec(); + reloaded[0] = ("gpt-4o-2024-11-20", Some("openai")); + model_cost.rebind(&reloaded); + + assert_eq!(store.find_key("GPT-4o"), None); + assert_eq!( + store.find_key("GPT-4o-2024-11-20").as_deref(), + Some("gpt-4o-2024-11-20") + ); + } + + #[rstest] + fn a_deleted_model_is_never_returned(model_cost: ModelCost) { + let store = Store::new(&model_cost); + assert_eq!(store.find_key("O3-PRO").as_deref(), Some("o3-pro")); + + model_cost.del("o3-pro"); + + assert_eq!(store.find_key("O3-PRO"), None); + assert_eq!(store.keys_in("openai"), ["gpt-4o"]); + } + + #[rstest] + fn a_respelled_key_at_the_same_length_is_found_after_one_rebuild(model_cost: ModelCost) { + let store = Store::new(&model_cost); + assert_eq!(store.find_key("o3-PRO").as_deref(), Some("o3-pro")); + + model_cost.del("o3-pro"); + model_cost.set("O3-Pro", Some("openai")); + + assert_eq!(store.find_key("o3-PRO").as_deref(), Some("O3-Pro")); + assert_eq!(model_cost.scans.get(), 2); + } + + #[rstest] + fn a_swap_at_the_same_length_hides_the_new_key_until_invalidate(model_cost: ModelCost) { + let store = Store::new(&model_cost); + assert_eq!(store.find_key("O3-PRO").as_deref(), Some("o3-pro")); + + model_cost.del("o3-pro"); + model_cost.set("MyCorp/Custom-Model", Some("openai")); + + assert_eq!(store.find_key("mycorp/custom-model"), None); + assert_eq!(store.keys_in("openai"), ["gpt-4o"]); + + store.invalidate(); + + assert_eq!( + store.find_key("mycorp/custom-model").as_deref(), + Some("MyCorp/Custom-Model") + ); + assert_eq!(store.keys_in("openai"), ["gpt-4o", "MyCorp/Custom-Model"]); + } + + #[rstest] + fn a_provider_rewrite_on_an_existing_key_needs_invalidate(model_cost: ModelCost) { + let store = Store::new(&model_cost); + assert_eq!(store.keys_in("openai"), ["gpt-4o", "o3-pro"]); + + model_cost.set("o3-pro", Some("azure")); + + assert_eq!(store.keys_in("openai"), ["gpt-4o", "o3-pro"]); + + store.invalidate(); + + assert_eq!(store.keys_in("openai"), ["gpt-4o"]); + assert_eq!(store.keys_in("azure"), ["o3-pro", "azure/gpt-4o"]); + } + + #[rstest] + fn an_edit_on_owned_rows_is_seen_by_the_next_query() { + let mut store = owned(BUNDLED); + assert_eq!(store.find_key("GPT-4o").as_deref(), Some("gpt-4o")); + + store.edit(|rows| { + rows.0.shift_remove("gpt-4o"); + rows.0 + .insert("MyCorp/Custom-Model".to_owned(), Some("openai".to_owned())); + }); + + assert_eq!(store.find_key("GPT-4o"), None); + assert_eq!( + store.find_key("mycorp/custom-model").as_deref(), + Some("MyCorp/Custom-Model") + ); + assert_eq!(store.keys_in("openai"), ["o3-pro", "MyCorp/Custom-Model"]); + } + + #[rstest] + fn get_reads_the_row_by_exact_key_only() { + let store = owned(BUNDLED); + assert_eq!( + store.get("azure/gpt-4o").as_deref(), + Some(&Some("azure".to_owned())) + ); + assert_eq!(store.get("Azure/GPT-4o"), None); + } +} diff --git a/litellm-rust/crates/model-catalog/src/store/name_index.rs b/litellm-rust/crates/model-catalog/src/store/name_index.rs new file mode 100644 index 00000000000..86f05cb31af --- /dev/null +++ b/litellm-rust/crates/model-catalog/src/store/name_index.rs @@ -0,0 +1,35 @@ +use std::collections::HashMap; + +use super::Rows; + +/// Keys only, never rows, so a value write never makes it stale. A later key wins when two keys fold to one name +#[derive(Debug, Default)] +pub(super) struct NameIndex { + by_folded: HashMap, + by_group: HashMap>, +} + +impl NameIndex { + pub(super) fn scan(rows: &R) -> Self { + let mut index = Self::default(); + rows.scan(&mut |key, group| { + index.by_folded.insert(R::fold(key), key.to_owned()); + if let Some(group) = group { + index + .by_group + .entry(group.to_owned()) + .or_default() + .push(key.to_owned()); + } + }); + index + } + + pub(super) fn key_for(&self, folded: &str) -> Option<&str> { + self.by_folded.get(folded).map(String::as_str) + } + + pub(super) fn keys_in(&self, group: &str) -> &[String] { + self.by_group.get(group).map_or(&[], Vec::as_slice) + } +} diff --git a/litellm-rust/crates/model-catalog/src/validation.rs b/litellm-rust/crates/model-catalog/src/validation.rs deleted file mode 100644 index 6dd4174148c..00000000000 --- a/litellm-rust/crates/model-catalog/src/validation.rs +++ /dev/null @@ -1,252 +0,0 @@ -use std::collections::BTreeSet; - -use serde_json::{Map, Value}; -use thiserror::Error; - -use crate::{AliasIssue, Catalog, ModelInfo, UtcHours, Weekday}; - -#[derive(Debug, Error)] -pub enum RegistryValidationError { - #[error("invalid registry entry {model:?}")] - Entry { - model: String, - #[source] - reason: EntryViolation, - }, - #[error(transparent)] - Alias(AliasIssue), -} - -#[derive(Debug, Error)] -pub enum EntryViolation { - #[error("entry must be an object")] - NotObject, - #[error("aliases must be an array")] - AliasesNotArray, - #[error("aliases must contain strings")] - AliasNotString, - #[error("entry does not match the model schema")] - Schema(#[source] serde_json::Error), - #[error("litellm_provider is required")] - MissingProvider, - #[error("invalid deprecation_date")] - DeprecationDate(#[source] time::error::Parse), - #[error("off_peak_pricing needs hours or windows")] - OffPeakWithoutHours, - #[error("off_peak_pricing.hours_utc is invalid")] - InvalidHours, - #[error("off_peak_pricing.windows is empty")] - EmptyWindows, - #[error("off_peak_pricing.weekdays is invalid")] - InvalidWeekdays, - #[error("typed entry could not be serialized")] - Serialize(#[source] serde_json::Error), - #[error("entry has an unknown field, null, or changed value: {}", fields.join(", "))] - ChangedFields { fields: Vec }, - #[error("{field} must be nonnegative")] - Negative { field: String }, - #[error("{field} must be at least one")] - UpliftBelowOne { field: String }, -} - -pub fn validate_model_entry(model: &str, value: &Value) -> Result<(), RegistryValidationError> { - validate_entry_inner(value).map_err(|reason| RegistryValidationError::Entry { - model: model.to_owned(), - reason, - }) -} - -pub fn validate_registry(catalog: &Catalog) -> Result<(), RegistryValidationError> { - if let Some(issue) = catalog.alias_issues().first() { - return Err(RegistryValidationError::Alias(issue.clone())); - } - catalog.model_names().try_for_each(|name| { - let entry = catalog.lookup(name).expect("catalog name must resolve"); - validate_model_entry(name, &Value::Object(entry.entry.fields().clone())) - }) -} - -fn json_eq(left: &Value, right: &Value) -> bool { - match (left, right) { - (Value::Number(left), Value::Number(right)) => left.as_f64() == right.as_f64(), - (Value::Array(left), Value::Array(right)) => { - left.len() == right.len() && left.iter().zip(right).all(|(a, b)| json_eq(a, b)) - } - (Value::Object(left), Value::Object(right)) => { - left.len() == right.len() - && left - .iter() - .all(|(key, value)| right.get(key).is_some_and(|other| json_eq(value, other))) - } - _ => left == right, - } -} - -fn keys(value: &Map) -> BTreeSet { - value.keys().cloned().collect() -} - -fn symmetric_difference(left: &BTreeSet, right: &BTreeSet) -> Vec { - left.symmetric_difference(right).cloned().collect() -} - -fn validate_entry_inner(value: &Value) -> Result<(), EntryViolation> { - let object = value.as_object().ok_or(EntryViolation::NotObject)?; - if let Some(aliases) = object.get("aliases") { - let names = aliases.as_array().ok_or(EntryViolation::AliasesNotArray)?; - if names.iter().any(|name| !name.is_string()) { - return Err(EntryViolation::AliasNotString); - } - } - let info: ModelInfo = serde_json::from_value(value.clone()).map_err(EntryViolation::Schema)?; - if info.litellm_provider.is_none() { - return Err(EntryViolation::MissingProvider); - } - validate_dates_and_windows(&info)?; - let serialized = serde_json::to_value(info).map_err(EntryViolation::Serialize)?; - let mut expected = object.clone(); - expected.remove("aliases"); - if !json_eq(&Value::Object(expected.clone()), &serialized) { - let actual = serialized - .as_object() - .expect("ModelInfo serializes as an object"); - return Err(EntryViolation::ChangedFields { - fields: symmetric_difference(&keys(&expected), &keys(actual)), - }); - } - check_prices("", value) -} - -fn validate_dates_and_windows(info: &ModelInfo) -> Result<(), EntryViolation> { - if let Some(date) = &info.deprecation_date { - let format = time::format_description::parse_borrowed::<2>("[year]-[month]-[day]").unwrap(); - time::Date::parse(date, &format).map_err(EntryViolation::DeprecationDate)?; - } - let Some(pricing) = &info.off_peak_pricing else { - return Ok(()); - }; - if pricing.hours_utc.is_none() && pricing.windows.is_none() { - return Err(EntryViolation::OffPeakWithoutHours); - } - if let Some(hours) = &pricing.hours_utc { - validate_hours(hours)?; - } - if let Some(windows) = &pricing.windows { - if windows.is_empty() { - return Err(EntryViolation::EmptyWindows); - } - windows.iter().try_for_each(|window| { - validate_hours(&window.hours_utc)?; - if let Some(days) = &window.weekdays - && (days.is_empty() || days.iter().any(|day| !valid_weekday(day))) - { - return Err(EntryViolation::InvalidWeekdays); - } - Ok(()) - })?; - } - Ok(()) -} - -fn validate_hours(hours: &UtcHours) -> Result<(), EntryViolation> { - let values = match hours { - UtcHours::Single(value) => std::slice::from_ref(value), - UtcHours::Multiple(values) => values.as_slice(), - }; - if values.is_empty() || values.iter().any(|value| !valid_utc_window(value)) { - return Err(EntryViolation::InvalidHours); - } - Ok(()) -} - -fn valid_utc_window(value: &str) -> bool { - let Some((start, end)) = value.split_once('-') else { - return false; - }; - [start, end].into_iter().all(|clock| { - let Some((hour, minute)) = clock.split_once(':') else { - return false; - }; - hour.len() == 2 - && minute.len() == 2 - && hour.parse::().is_ok_and(|hour| hour < 24) - && minute.parse::().is_ok_and(|minute| minute < 60) - }) -} - -fn valid_weekday(day: &Weekday) -> bool { - match day { - Weekday::Number(number) => (1..=7).contains(number), - Weekday::Name(name) => matches!( - name.to_ascii_lowercase().as_str(), - "mon" - | "monday" - | "tue" - | "tues" - | "tuesday" - | "wed" - | "wednesday" - | "thu" - | "thur" - | "thurs" - | "thursday" - | "fri" - | "friday" - | "sat" - | "saturday" - | "sun" - | "sunday" - ), - } -} - -fn check_prices(path: &str, value: &Value) -> Result<(), EntryViolation> { - let Some(object) = value.as_object() else { - return Ok(()); - }; - object.iter().try_for_each(|(key, field)| { - let field_path = format!("{path}.{key}"); - if (key.contains("cost") - || path.ends_with(".guardrail_cost_per_unit") - || path.ends_with(".search_context_cost_per_query")) - && let Some(number) = field.as_f64() - && number < 0.0 - { - return Err(EntryViolation::Negative { - field: relative(&field_path), - }); - } - if key.contains("uplift_multiplier") - && let Some(number) = field.as_f64() - && number < 1.0 - { - return Err(EntryViolation::UpliftBelowOne { - field: relative(&field_path), - }); - } - if matches!(key.as_str(), "range" | "max_results_range") - && field.as_array().is_some_and(|values| { - values - .iter() - .any(|value| value.as_f64().is_some_and(|n| n < 0.0)) - }) - { - return Err(EntryViolation::Negative { - field: relative(&field_path), - }); - } - if matches!(key.as_str(), "metadata" | "provider_specific_entry") { - return Ok(()); - } - match field.as_array() { - Some(items) => items.iter().enumerate().try_for_each(|(index, item)| { - check_prices(&format!("{field_path}[{index}]"), item) - }), - None => check_prices(&field_path, field), - } - }) -} - -fn relative(field_path: &str) -> String { - field_path.trim_start_matches('.').to_owned() -} diff --git a/litellm-rust/crates/model-catalog/src/vocabulary.rs b/litellm-rust/crates/model-catalog/src/vocabulary.rs new file mode 100644 index 00000000000..2b7ac8bd1ab --- /dev/null +++ b/litellm-rust/crates/model-catalog/src/vocabulary.rs @@ -0,0 +1,126 @@ +/// Primary API surface / task type of the model. +#[macro_rules_attribute::apply(crate::wire_type)] +#[derive(Copy, Eq, Hash)] +#[serde(rename_all = "snake_case")] +pub enum Mode { + AudioSpeech, + AudioTranscription, + Chat, + Completion, + Decisions, + Embedding, + Evaluation, + Extraction, + Guardrail, + ImageEdit, + ImageGeneration, + Moderation, + Ocr, + Realtime, + Rerank, + Responses, + Search, + VectorStore, + VideoGeneration, +} + +/// Gemini audio generation API the model is served through. +#[macro_rules_attribute::apply(crate::wire_type)] +#[derive(Copy, Eq, Hash)] +#[serde(rename_all = "snake_case")] +pub enum VertexAiAudioApi { + LyriaPredict, + LyriaInteractions, +} + +/// Audio container format the model can return. +#[macro_rules_attribute::apply(crate::wire_type)] +#[derive(Copy, Eq, Hash)] +#[serde(rename_all = "snake_case")] +pub enum AudioFormat { + Mp3, + Wav, +} + +/// Input modality the model accepts. +#[macro_rules_attribute::apply(crate::wire_type)] +#[derive(Copy, Eq, Hash)] +#[serde(rename_all = "snake_case")] +pub enum InputModality { + Text, + Image, + Audio, + Video, +} + +/// Output modality the model can produce. +#[macro_rules_attribute::apply(crate::wire_type)] +#[derive(Copy, Eq, Hash)] +#[serde(rename_all = "snake_case")] +pub enum OutputModality { + Text, + Image, + Audio, + Video, + Code, +} + +/// API route a model can be called through. +#[macro_rules_attribute::apply(crate::wire_type)] +#[derive(Copy, Eq, Hash)] +pub enum Endpoint { + #[serde(rename = "/v1/chat/completions")] + ChatCompletions, + #[serde(rename = "/v1/completions")] + Completions, + #[serde(rename = "/v1/responses")] + Responses, + #[serde(rename = "/v1/messages")] + Messages, + #[serde(rename = "/v1/batch")] + Batch, + #[serde(rename = "/v1/embeddings")] + Embeddings, + #[serde(rename = "/v1/images/generations")] + ImageGenerations, + #[serde(rename = "/v1/images/edits")] + ImageEdits, + #[serde(rename = "/v1/audio/speech")] + AudioSpeech, + #[serde(rename = "/v1/audio/transcriptions")] + AudioTranscriptions, + #[serde(rename = "/v1/realtime")] + Realtime, + #[serde(rename = "/v1/realtime/transcription_sessions")] + RealtimeTranscriptionSessions, + #[serde(rename = "/v1/videos")] + Videos, + #[serde(rename = "/v1/ocr")] + Ocr, + #[serde(rename = "/v1/listen")] + Listen, + #[serde(rename = "/v1/systemone")] + SystemOne, + #[serde(rename = "/v1/decisions")] + Decisions, + #[serde(rename = "/v1beta/interactions")] + Interactions, + #[serde(rename = "/v1beta/models")] + Models, + #[serde(rename = "/vertex_ai/live")] + VertexAiLive, +} + +/// Reasoning effort level a model accepts or applies. +#[macro_rules_attribute::apply(crate::wire_type)] +#[derive(Copy, Eq, Hash)] +#[serde(rename_all = "snake_case")] +pub enum EffortLevel { + None, + Minimal, + Low, + Medium, + High, + Xhigh, + Max, +} diff --git a/litellm-rust/crates/model-catalog/src/wire/attributes.rs b/litellm-rust/crates/model-catalog/src/wire/attributes.rs new file mode 100644 index 00000000000..93c10a81524 --- /dev/null +++ b/litellm-rust/crates/model-catalog/src/wire/attributes.rs @@ -0,0 +1,344 @@ +use std::collections::BTreeMap; +use std::fmt; +use std::str::FromStr; + +use litellm_cost::money::UnitPrice; +use serde::de::{self, Deserializer}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::error::{DateCause, Error, Repr}; +use crate::vocabulary::{ + AudioFormat, EffortLevel, Endpoint, InputModality, Mode, OutputModality, VertexAiAudioApi, +}; +use crate::wire::lax::Count; + +macro_rules! attributes { + ($($(#[$doc:meta])* $name:ident: $ty:ty $(as $via:ty)?),* $(,)?) => { + /// The named, non-price fields of a catalog entry. + #[derive(Clone, Debug, Default, PartialEq)] + #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] + pub struct Attributes { + $($(#[$doc])* pub $name: Option<$ty>,)* + } + + impl Attributes { + pub(crate) fn read( + &mut self, + key: &str, + value: &Value, + ) -> Option> { + match key { + $(stringify!($name) => Some( + attributes!(@read value, $ty $(, $via)?).map(|read| self.$name = Some(read)), + ),)* + _ => None, + } + } + } + }; + (@read $value:ident, $ty:ty) => { + <$ty as Deserialize>::deserialize($value) + }; + (@read $value:ident, $ty:ty, $via:ty) => { + serde_with::de::DeserializeAsWrap::<$ty, $via>::deserialize($value) + .map(serde_with::de::DeserializeAsWrap::into_inner) + }; +} + +attributes! { + /// Other names this entry is served under. + aliases: Vec, + audio_transcription_config: String, + bedrock_converse_supports_strict_tools: bool, + /// Highest reasoning effort the Bedrock output_config accepts for this model. + bedrock_output_config_effort_ceiling: EffortLevel, + comment: String, + /// Reasoning effort the provider applies when the request omits reasoning_effort. Gates whether a non-default temperature or the top_p/logprobs sampling params are accepted, which hold only when the effort resolves to 'none'. + default_reasoning_effort: EffortLevel, + /// Date the provider deprecates the model, YYYY-MM-DD. + deprecation_date: IsoDate, + gemini_audio_only_live: bool, + gemini_native_audio: bool, + input_cost_per_token_cache_hit: UnitPrice, + /// Databricks units, not USD. + input_dbu_cost_per_token: Dbu, + /// LiteLLM provider slug; one of https://docs.litellm.ai/docs/providers. + litellm_provider: String, + /// Maximum prompt/context tokens the model accepts. + max_input_tokens: u64 as Count, + /// Maximum tokens the model can generate in one response. + max_output_tokens: u64 as Count, + /// Legacy field: max output tokens if the provider specifies it, else max input tokens. + max_tokens: u64 as Count, + /// Free-form notes about the entry (e.g. pricing derivation). + metadata: BTreeMap, + /// Primary API surface / task type of the model. + mode: Mode, + /// Databricks units, not USD. + output_dbu_cost_per_token: Dbu, + /// Embedding dimension for embedding models. + output_vector_size: u64 as Count, + /// Smallest prefix the provider will actually cache; absent means the provider default applies. + prompt_cache_min_tokens: u64 as Count, + /// Provider-internal routing hints (e.g. bedrock_invocation_schema). + provider_specific_entry: BTreeMap, + reasoning_contexts: Vec, + /// Exact reasoning_effort levels this deployment accepts; wins over supports_* flags. + reasoning_effort_levels: Vec, + reasoning_modes: Vec, + /// Provider default requests-per-minute limit. + rpm: u64 as Count, + /// Grounded search requests the provider includes for free before search is billed. + search_free_requests: SearchFreeRequests, + /// URL of the provider pricing/model page this entry was taken from. + source: String, + /// Audio container formats the model can return. + supported_audio_formats: Vec, + /// OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions. + supported_endpoints: Vec, + /// Input modalities the model accepts. + supported_modalities: Vec, + /// Output modalities the model can produce. + supported_output_modalities: Vec, + /// Cloud regions the model is available in ('global' or region ids). + supported_regions: Vec, + thinking_always_on: bool, + token_count_framing: u64 as Count, + token_count_model: TokenCountModel, + token_count_tokenizer: TokenCountTokenizer, + /// Provider default tokens-per-minute limit. + tpm: u64 as Count, + use_openai_responses_path: bool, + uses_embed_content: bool, + vertex_ai_audio_api: VertexAiAudioApi, + /// Whether web search is billed per query or per prompt. + web_search_billing_unit: WebSearchBillingUnit, +} + +/// The model whose provider token counter counts tokens for this entry. +#[macro_rules_attribute::apply(crate::wire_type)] +#[derive(Eq, Hash)] +#[serde(deny_unknown_fields)] +pub struct TokenCountModel { + pub model: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub litellm_provider: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub region: Option, +} + +/// Free grounded search requests per period, shared across the models of one group. +#[macro_rules_attribute::apply(crate::wire_type)] +#[derive(Eq, Hash)] +#[serde(deny_unknown_fields)] +pub struct SearchFreeRequests { + pub count: u64, + pub period: FreeRequestPeriod, + pub shared_group: String, +} + +#[macro_rules_attribute::apply(crate::wire_type)] +#[derive(Copy, Eq, Hash)] +#[serde(rename_all = "snake_case")] +pub enum FreeRequestPeriod { + Day, + Month, +} + +/// Local tokenizer that counts tokens for this entry. +#[macro_rules_attribute::apply(crate::wire_type)] +#[derive(Copy, Eq, Hash)] +#[serde(rename_all = "snake_case")] +pub enum TokenCountTokenizer { + KimiK3, +} + +/// Whether web search is billed per query or per prompt. +#[macro_rules_attribute::apply(crate::wire_type)] +#[derive(Copy, Eq, Hash)] +#[serde(rename_all = "snake_case")] +pub enum WebSearchBillingUnit { + PerQuery, + PerPrompt, +} + +/// A non-negative amount of Databricks units. +#[derive(Clone, Copy, Debug, PartialEq, PartialOrd, Serialize)] +#[serde(transparent)] +pub struct Dbu(f64); + +impl Dbu { + pub fn new(units: f64) -> Option { + (units.is_finite() && units >= 0.0).then_some(Self(units)) + } + pub fn get(self) -> f64 { + self.0 + } +} + +impl<'de> Deserialize<'de> for Dbu { + fn deserialize>(deserializer: D) -> Result { + let units = f64::deserialize(deserializer)?; + Self::new(units).ok_or_else(|| { + de::Error::invalid_value( + de::Unexpected::Float(units), + &"a non-negative number of Databricks units", + ) + }) + } +} + +/// A calendar date written YYYY-MM-DD. +#[derive( + Clone, + Copy, + Debug, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + serde_with::DeserializeFromStr, + serde_with::SerializeDisplay, +)] +pub struct IsoDate(pub time::Date); + +impl FromStr for IsoDate { + type Err = Error; + + fn from_str(date: &str) -> Result { + let mut parts = date.split('-'); + let (Some(year), Some(month), Some(day), None) = + (parts.next(), parts.next(), parts.next(), parts.next()) + else { + return Err(Repr::InvalidDate { cause: None }.into()); + }; + if year.len() != 4 || month.len() != 2 || day.len() != 2 { + return Err(Repr::InvalidDate { cause: None }.into()); + } + let number = |part: &str| part.parse::().map_err(DateCause::from); + let month = time::Month::try_from(number(month)?).map_err(DateCause::from)?; + time::Date::from_calendar_date(year.parse().map_err(DateCause::from)?, month, number(day)?) + .map(Self) + .map_err(DateCause::from) + .map_err(Error::from) + } +} + +impl fmt::Display for IsoDate { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{:04}-{:02}-{:02}", + self.0.year(), + u8::from(self.0.month()), + self.0.day() + ) + } +} + +#[cfg(feature = "schema")] +mod schema { + use std::borrow::Cow; + + use schemars::{JsonSchema, Schema, SchemaGenerator, json_schema}; + + use super::{Dbu, IsoDate}; + + impl JsonSchema for IsoDate { + fn inline_schema() -> bool { + true + } + fn schema_name() -> Cow<'static, str> { + "IsoDate".into() + } + fn json_schema(_: &mut SchemaGenerator) -> Schema { + json_schema!({ + "type": "string", + "format": "date", + "pattern": r"^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])$" + }) + } + } + + impl JsonSchema for Dbu { + fn inline_schema() -> bool { + true + } + fn schema_name() -> Cow<'static, str> { + "Dbu".into() + } + fn json_schema(_: &mut SchemaGenerator) -> Schema { + json_schema!({"type": "number", "minimum": 0}) + } + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + #[rstest] + #[case::leap_day("2028-02-29", true)] + #[case::not_a_leap_year("2026-02-29", false)] + #[case::impossible_day("2026-02-31", false)] + #[case::one_digit_month("2026-2-01", false)] + #[case::extra_part("2026-02-01-01", false)] + fn iso_dates_are_real_calendar_dates(#[case] text: &str, #[case] valid: bool) { + assert_eq!(text.parse::().is_ok(), valid); + } + + #[rstest] + #[case::count_written_as_text("max_tokens", json!("8192"), true)] + #[case::negative_count("max_tokens", json!(-1), false)] + #[case::known_mode("mode", json!("chat"), true)] + #[case::unknown_mode("mode", json!("chatt"), false)] + #[case::negative_dbu("input_dbu_cost_per_token", json!(-1), false)] + #[case::reasoning_contexts("reasoning_contexts", json!(["reasoning", "non_reasoning"]), true)] + #[case::reasoning_modes("reasoning_modes", json!(["none", "adaptive"]), true)] + #[case::reasoning_context_not_a_list("reasoning_contexts", json!("reasoning"), false)] + #[case::reasoning_mode_not_a_string("reasoning_modes", json!([1]), false)] + fn each_attribute_reads_its_own_type( + #[case] key: &str, + #[case] value: Value, + #[case] valid: bool, + ) { + assert_eq!( + Attributes::default() + .read(key, &value) + .map(|read| read.is_ok()), + Some(valid) + ); + } + + #[rstest] + fn keys_that_are_not_attributes_are_left_to_the_caller() { + assert!( + Attributes::default() + .read("input_cost_per_token", &json!(1)) + .is_none() + ); + } + + #[rstest] + fn reasoning_metadata_is_retained() { + let mut attributes = Attributes::default(); + attributes + .read("reasoning_contexts", &json!(["reasoning", "non_reasoning"])) + .unwrap() + .unwrap(); + attributes + .read("reasoning_modes", &json!(["none", "adaptive"])) + .unwrap() + .unwrap(); + assert_eq!( + attributes.reasoning_contexts.unwrap(), + ["reasoning", "non_reasoning"] + ); + assert_eq!(attributes.reasoning_modes.unwrap(), ["none", "adaptive"]); + } +} diff --git a/litellm-rust/crates/model-catalog/src/wire/blocks.rs b/litellm-rust/crates/model-catalog/src/wire/blocks.rs new file mode 100644 index 00000000000..3a355686b7c --- /dev/null +++ b/litellm-rust/crates/model-catalog/src/wire/blocks.rs @@ -0,0 +1,182 @@ +use litellm_cost::money::UnitPrice; +use litellm_cost::schedule::{UtcWindow, Weekday}; +use serde::Deserialize; +use serde::de::DeserializeOwned; +use serde_json::{Map, Value}; +use serde_with::de::DeserializeAsWrap; + +use crate::error::{Error, Repr, pointer_child, pointer_for_key}; +use crate::wire::flat_key::FlatKey; +use crate::wire::lax::Count; + +/// A flat price key with its spelling, kept so problems can name it. +#[derive(Clone, Debug, PartialEq)] +pub struct FlatPrice { + pub key: String, + pub path: String, + pub flat: FlatKey, + pub price: UnitPrice, +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct TierRow { + pub path: String, + pub range: Option<[u64; 2]>, + pub max_results_range: Option<[u64; 2]>, + pub prices: Vec, +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct OffPeakBlock { + pub hours_utc: Option, + pub windows: Option>, + pub weekday_timezone: Option, + pub prices: Vec, +} + +/// One UTC window or a list of them. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(untagged)] +pub enum UtcHours { + Single(UtcWindow), + Multiple(Vec), +} + +/// One off-peak window entry inside `windows`. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields)] +pub struct OffPeakWindow { + pub hours_utc: UtcHours, + pub weekdays: Option>, +} + +/// USD cost per web search query, keyed by search context size. +#[derive(Clone, Debug, Default, PartialEq, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields)] +pub struct SearchContextCostPerQuery { + pub search_context_size_low: Option, + pub search_context_size_medium: Option, + pub search_context_size_high: Option, +} + +impl UtcHours { + pub fn windows(&self) -> &[UtcWindow] { + match self { + Self::Single(window) => std::slice::from_ref(window), + Self::Multiple(windows) => windows, + } + } +} + +pub(crate) struct Reader<'a> { + pub(crate) problems: &'a mut Vec, +} + +impl Reader<'_> { + pub(crate) fn value(&mut self, path: String, value: &Value) -> Option { + T::deserialize(value) + .map_err(|error| self.problems.push(Error::new(path, Repr::Invalid(error)))) + .ok() + } + + pub(crate) fn price( + &mut self, + key: String, + path: String, + flat: FlatKey, + value: &Value, + ) -> Option { + let price = self.value(path.clone(), value)?; + Some(FlatPrice { + key, + path, + flat, + price, + }) + } + + fn object<'v>(&mut self, path: &str, value: &'v Value) -> Option<&'v Map> { + match value { + Value::Object(fields) => Some(fields), + other => self + .value::>(path.to_owned(), other) + .and(None), + } + } + + fn block_price( + &mut self, + name: &str, + key: String, + path: String, + value: &Value, + ) -> Option { + let Some(flat) = FlatKey::parse(name) else { + self.problems.push(Error::new(path, Repr::UnknownBlockKey)); + return None; + }; + self.price(key, path, flat, value) + } + + pub(crate) fn tier_rows(&mut self, value: &Value) -> Option> { + let path = pointer_for_key("tiered_pricing"); + let rows: Vec = self.value(path.clone(), value)?; + Some( + rows.iter() + .enumerate() + .filter_map(|(index, row)| self.tier_row(path.clone(), index, row)) + .collect(), + ) + } + + fn tier_row(&mut self, parent: String, index: usize, value: &Value) -> Option { + let path = format!("{parent}/{index}"); + let display_path = format!("tiered_pricing[{index}]"); + let fields = self.object(&path, value)?; + let mut row = TierRow { + path: path.clone(), + ..TierRow::default() + }; + for (name, value) in fields.iter().filter(|(_, value)| !value.is_null()) { + let key = format!("{display_path}.{name}"); + let child_path = pointer_child(&path, name); + match name.as_str() { + "range" => row.range = self.bounds(child_path, value), + "max_results_range" => row.max_results_range = self.bounds(child_path, value), + _ => row + .prices + .extend(self.block_price(name, key, child_path, value)), + } + } + Some(row) + } + + fn bounds(&mut self, path: String, value: &Value) -> Option<[u64; 2]> { + DeserializeAsWrap::<[u64; 2], [Count; 2]>::deserialize(value) + .map(DeserializeAsWrap::into_inner) + .map_err(|error| self.problems.push(Error::new(path, Repr::Invalid(error)))) + .ok() + } + + pub(crate) fn off_peak(&mut self, value: &Value) -> Option { + let path = pointer_for_key("off_peak_pricing"); + let fields = self.object(&path, value)?; + let mut block = OffPeakBlock::default(); + for (name, value) in fields.iter().filter(|(_, value)| !value.is_null()) { + let key = format!("off_peak_pricing.{name}"); + let child_path = pointer_child(&path, name); + match name.as_str() { + "hours_utc" => block.hours_utc = self.value(child_path, value), + "windows" => block.windows = self.value(child_path, value), + "weekday_timezone" => block.weekday_timezone = self.value(child_path, value), + _ => block + .prices + .extend(self.block_price(name, key, child_path, value)), + } + } + Some(block) + } +} diff --git a/litellm-rust/crates/model-catalog/src/wire/entry.rs b/litellm-rust/crates/model-catalog/src/wire/entry.rs new file mode 100644 index 00000000000..9c155c6496a --- /dev/null +++ b/litellm-rust/crates/model-catalog/src/wire/entry.rs @@ -0,0 +1,193 @@ +use std::collections::BTreeMap; + +use litellm_cost::money::{Multiplier, UnitPrice}; +use serde_json::{Map, Value}; + +use crate::error::{Error, Repr, pointer_for_key}; +use crate::wire::attributes::Attributes; +use crate::wire::blocks::{FlatPrice, OffPeakBlock, Reader, SearchContextCostPerQuery, TierRow}; +use crate::wire::flat_key::FlatKey; + +pub type Fields = Map; + +/// One catalog entry as written. Every non-null key lands in exactly one field, and a known key +/// with a bad value is kept out and listed in `problems`. +#[derive(Debug, Default)] +pub struct EntryWire { + pub attributes: Attributes, + pub prices: Vec, + pub capabilities: BTreeMap, + pub tier_rows: Option>, + pub off_peak: Option, + pub search_context: Option, + pub guardrail: Option>, + pub regional_endpoint_uplift: Option, + pub residency_uplift_eu: Option, + pub residency_uplift_us: Option, + pub unknown: Fields, + pub problems: Vec, +} + +impl EntryWire { + pub fn read(fields: &Fields) -> Self { + let mut problems = Vec::new(); + let mut wire = Self::default(); + let mut reader = Reader { + problems: &mut problems, + }; + for (key, value) in fields.iter().filter(|(_, value)| !value.is_null()) { + wire.read_key(&mut reader, key, value); + } + Self { problems, ..wire } + } + + fn read_key(&mut self, reader: &mut Reader<'_>, key: &str, value: &Value) { + let path = || pointer_for_key(key); + match key { + "tiered_pricing" => self.tier_rows = reader.tier_rows(value), + "off_peak_pricing" => self.off_peak = reader.off_peak(value), + "search_context_cost_per_query" => self.search_context = reader.value(path(), value), + "guardrail_cost_per_unit" => self.guardrail = reader.value(path(), value), + "regional_endpoint_uplift_multiplier" => { + self.regional_endpoint_uplift = reader.value(path(), value) + } + "regional_processing_uplift_multiplier_eu" => { + self.residency_uplift_eu = reader.value(path(), value) + } + "regional_processing_uplift_multiplier_us" => { + self.residency_uplift_us = reader.value(path(), value) + } + _ => self.read_open_key(reader, key, value), + } + } + + fn read_open_key(&mut self, reader: &mut Reader<'_>, key: &str, value: &Value) { + if let Some(read) = self.attributes.read(key, value) { + if let Err(error) = read { + reader + .problems + .push(Error::from_key(key, Repr::Invalid(error))); + } + return; + } + if let Some(flat) = FlatKey::parse(key) { + self.prices + .extend(reader.price(key.to_owned(), pointer_for_key(key), flat, value)); + return; + } + if key.starts_with("supports_") { + if let Some(flag) = reader.value(pointer_for_key(key), value) { + self.capabilities.insert(key.to_owned(), flag); + } + return; + } + self.unknown.insert(key.to_owned(), value.clone()); + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + fn read(entry: Value) -> EntryWire { + let Value::Object(fields) = entry else { + panic!("entry must be an object") + }; + EntryWire::read(&fields) + } + + fn problem_keys(wire: &EntryWire) -> Vec { + wire.problems + .iter() + .map(|problem| problem.path().to_owned()) + .collect() + } + + #[rstest] + fn every_key_lands_in_exactly_one_place() { + let wire = read(json!({ + "litellm_provider": "openai", + "max_tokens": "8192", + "input_cost_per_token": "0.000001", + "tiered_pricing": [{"range": [0, 10], "output_cost_per_token": 2e-6}], + "supports_vision": true, + "id": "deployment-1", + "mode": null + })); + + assert_eq!(wire.attributes.litellm_provider.as_deref(), Some("openai")); + assert_eq!(wire.attributes.max_tokens, Some(8192)); + assert_eq!(wire.attributes.mode, None); + assert_eq!( + wire.prices + .iter() + .map(|price| price.key.as_str()) + .collect::>(), + ["input_cost_per_token"] + ); + assert_eq!(wire.tier_rows.unwrap()[0].prices.len(), 1); + assert_eq!( + wire.capabilities, + BTreeMap::from([("supports_vision".to_owned(), true)]) + ); + assert_eq!(wire.unknown.keys().collect::>(), ["id"]); + assert!(wire.problems.is_empty()); + } + + #[rstest] + fn every_bad_value_is_reported_by_its_key_at_once() { + let wire = read(json!({ + "mode": "chatt", + "max_tokens": -1, + "input_cost_per_token": "x", + "output_cost_per_image_0.5K": "x", + "supports_vision": "yes", + "regional_endpoint_uplift_multiplier": 0.5, + "tiered_pricing": [ + {"range": [0, 10], "input_cost_per_token": 1e-6}, + {"range": [10.5, 20], "typo_cost": 1, "output_cost_per_token": -1} + ], + "off_peak_pricing": {"hours_utc": "25:00-01:00", "input_cost_per_token": 1e-6} + })); + + assert_eq!( + problem_keys(&wire) + .into_iter() + .collect::>(), + [ + "/input_cost_per_token", + "/max_tokens", + "/mode", + "/off_peak_pricing/hours_utc", + "/output_cost_per_image_0.5K", + "/regional_endpoint_uplift_multiplier", + "/supports_vision", + "/tiered_pricing/1/output_cost_per_token", + "/tiered_pricing/1/range", + "/tiered_pricing/1/typo_cost", + ] + .map(str::to_owned) + .into_iter() + .collect() + ); + } + + #[rstest] + #[case::tiers_not_a_list(json!({"tiered_pricing": {"range": [0, 1]}}), "/tiered_pricing")] + #[case::row_not_an_object(json!({"tiered_pricing": [3]}), "/tiered_pricing/0")] + #[case::off_peak_not_an_object(json!({"off_peak_pricing": true}), "/off_peak_pricing")] + #[case::unknown_search_size( + json!({"search_context_cost_per_query": {"search_context_size_huge": 1}}), + "/search_context_cost_per_query" + )] + #[case::negative_guardrail( + json!({"guardrail_cost_per_unit": {"contentPolicyUnits": -1}}), + "/guardrail_cost_per_unit" + )] + fn a_bad_block_is_reported_at_the_block(#[case] entry: Value, #[case] expected: &str) { + assert_eq!(problem_keys(&read(entry)), [expected]); + } +} diff --git a/litellm-rust/crates/model-catalog/src/wire/flat_key.rs b/litellm-rust/crates/model-catalog/src/wire/flat_key.rs new file mode 100644 index 00000000000..906d917b487 --- /dev/null +++ b/litellm-rust/crates/model-catalog/src/wire/flat_key.rs @@ -0,0 +1,359 @@ +use std::num::NonZeroU64; + +use litellm_cost::price::{ + Band, CacheReadMode, CacheTtl, Channel, Feature, ImageSize, InputUnit, Meter, OutputUnit, + Resolution, ServiceTier, TokenKind, When, +}; + +/// A flat price key as the grammar reads it: `{meter}[_above_{N}k_tokens][_{tier}]`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FlatKey { + pub meter: Meter, + pub band: Band, + pub tier: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TierSuffix { + Service(ServiceTier), + Batches, + Decisions, +} + +impl FlatKey { + pub fn parse(key: &str) -> Option { + let (rest, tier) = split_tier(key); + let (base, suffix) = split_above(rest)?; + let (meter, band) = match suffix { + Above::None => (meter(base)?, Band::Base), + Above::PromptTokens(tokens) => (meter(base)?, Band::over_prompt_tokens(tokens)), + Above::Seconds(seconds) => match meter(base)? { + Meter::Input(InputUnit::VideoSecond { over_seconds: None }) => ( + Meter::Input(InputUnit::VideoSecond { + over_seconds: NonZeroU64::new(seconds), + }), + Band::Base, + ), + _ => return None, + }, + }; + Some(Self { meter, band, tier }) + } + + pub fn has_suffix(&self) -> bool { + self.band != Band::Base || self.tier.is_some() + } + + pub fn channel_and_when(&self) -> (Channel, When) { + let (channel, tier) = match self.tier { + None => (Channel::Online, ServiceTier::Standard), + Some(TierSuffix::Service(tier)) => (Channel::Online, tier), + Some(TierSuffix::Batches) => (Channel::Batch, ServiceTier::Standard), + Some(TierSuffix::Decisions) => (Channel::Decisions, ServiceTier::Standard), + }; + ( + channel, + When { + tier, + band: self.band, + }, + ) + } +} + +enum Above { + None, + PromptTokens(u64), + Seconds(u64), +} + +fn split_tier(key: &str) -> (&str, Option) { + let Some((head, tail)) = key.rsplit_once('_') else { + return (key, None); + }; + let tier = match tail { + "priority" => TierSuffix::Service(ServiceTier::Priority), + "flex" => TierSuffix::Service(ServiceTier::Flex), + "ultrafast" => TierSuffix::Service(ServiceTier::Ultrafast), + "balanced" => TierSuffix::Service(ServiceTier::Balanced), + "batches" => TierSuffix::Batches, + "decisions" => TierSuffix::Decisions, + _ => return (key, None), + }; + (head, Some(tier)) +} + +fn split_above(key: &str) -> Option<(&str, Above)> { + let Some((head, tail)) = key.rsplit_once("_above_") else { + return Some((key, Above::None)); + }; + if tail == "1hr" || tail == "12hr" { + return Some((key, Above::None)); + } + if let Some(thousands) = tail.strip_suffix("k_tokens") { + return Some(( + head, + Above::PromptTokens(number(thousands)?.checked_mul(1000)?), + )); + } + if let Some(tokens) = tail.strip_suffix("_tokens") { + return Some((head, Above::PromptTokens(number(tokens)?))); + } + let seconds = tail.strip_suffix("s_interval")?; + Some((head, Above::Seconds(number(seconds)?))) +} + +fn number(digits: &str) -> Option { + (!digits.is_empty() && digits.bytes().all(|byte| byte.is_ascii_digit())) + .then(|| digits.parse().ok()) + .flatten() +} + +fn meter(base: &str) -> Option { + let fixed = match base { + "cost_per_second" => Some(Meter::WallClockSecond), + "explicit_cache_read_input_token_cost" => Some(Meter::CacheRead { + kind: TokenKind::Text, + mode: CacheReadMode::Explicit, + }), + "annotation_cost_per_page" => Some(Meter::Feature(Feature::AnnotationPage)), + "citation_cost_per_token" => Some(Meter::Feature(Feature::CitationToken)), + "code_interpreter_cost_per_session" => { + Some(Meter::Feature(Feature::CodeInterpreterSession)) + } + "google_maps_grounding_cost_per_query" => { + Some(Meter::Feature(Feature::GoogleMapsGroundingQuery)) + } + "ocr_cost_per_credit" => Some(Meter::Feature(Feature::OcrCredit)), + "ocr_cost_per_page" => Some(Meter::Feature(Feature::OcrPage)), + "browser_open_cost_per_request" => Some(Meter::Feature(Feature::BrowserOpen)), + "cost_per_additional_result" => Some(Meter::Feature(Feature::AdditionalResult)), + _ => None, + }; + if fixed.is_some() { + return fixed; + } + if let Some(unit) = base.strip_prefix("input_cost_per_") { + return input_unit(unit).map(Meter::Input); + } + if let Some(unit) = base.strip_prefix("output_cost_per_") { + return output_unit(unit).map(Meter::Output); + } + if let Some(kind) = base + .strip_prefix("cache_read_input_") + .and_then(|rest| rest.strip_suffix("_cost")) + { + return Some(Meter::CacheRead { + kind: token_kind(kind)?, + mode: CacheReadMode::Implicit, + }); + } + let rest = base.strip_prefix("cache_creation_input_")?; + let (kind, ttl) = if let Some(kind) = rest.strip_suffix("_cost_above_1hr") { + (kind, CacheTtl::OneHour) + } else if let Some(kind) = rest.strip_suffix("_cost_above_12hr") { + (kind, CacheTtl::TwelveHours) + } else { + (rest.strip_suffix("_cost")?, CacheTtl::Default) + }; + Some(Meter::CacheWrite { + kind: token_kind(kind)?, + ttl, + }) +} + +fn token_kind(unit: &str) -> Option { + match unit { + "token" => Some(TokenKind::Text), + "audio_token" => Some(TokenKind::Audio), + "image_token" => Some(TokenKind::Image), + "video_token" => Some(TokenKind::Video), + _ => None, + } +} + +fn input_unit(unit: &str) -> Option { + if let Some(kind) = token_kind(unit) { + return Some(InputUnit::Token(kind)); + } + match unit { + "character" => Some(InputUnit::Character), + "image" => Some(InputUnit::Image), + "pixel" => Some(InputUnit::Pixel), + "query" => Some(InputUnit::Query { over_results: None }), + "request" => Some(InputUnit::Request), + "second" => Some(InputUnit::Second), + "text_message" => Some(InputUnit::TextMessage), + "audio_per_second" => Some(InputUnit::AudioSecond), + "video_per_second" => Some(InputUnit::VideoSecond { over_seconds: None }), + _ => None, + } +} + +fn output_unit(unit: &str) -> Option { + if let Some(kind) = token_kind(unit) { + return Some(OutputUnit::Token(kind)); + } + match unit.split_once('_') { + Some(("image", label)) => { + return ImageSize::new(label).map(|size| OutputUnit::Image(Some(size))); + } + Some(("second", label)) => { + return Resolution::new(label).map(|resolution| OutputUnit::Second(Some(resolution))); + } + _ => {} + } + match unit { + "reasoning_token" => Some(OutputUnit::ReasoningToken), + "character" => Some(OutputUnit::Character), + "image" => Some(OutputUnit::Image(None)), + "pixel" => Some(OutputUnit::Pixel), + "second" => Some(OutputUnit::Second(None)), + "video_per_second" => Some(OutputUnit::VideoSecond), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + const TEXT_INPUT: Meter = Meter::Input(InputUnit::Token(TokenKind::Text)); + const TEXT_OUTPUT: Meter = Meter::Output(OutputUnit::Token(TokenKind::Text)); + + fn key(meter: Meter, band: Band, tier: Option) -> FlatKey { + FlatKey { meter, band, tier } + } + + #[rstest] + #[case::input_token("input_cost_per_token", key(TEXT_INPUT, Band::Base, None))] + #[case::band_and_tier( + "output_cost_per_token_above_272k_tokens_priority", + key( + TEXT_OUTPUT, + Band::over_prompt_tokens(272_000), + Some(TierSuffix::Service(ServiceTier::Priority)) + ) + )] + #[case::band_without_k( + "input_cost_per_token_above_128_tokens", + key(TEXT_INPUT, Band::over_prompt_tokens(128), None) + )] + #[case::zero_band_is_base( + "input_cost_per_token_above_0k_tokens", + key(TEXT_INPUT, Band::Base, None) + )] + #[case::batch( + "input_cost_per_token_batches", + key(TEXT_INPUT, Band::Base, Some(TierSuffix::Batches)) + )] + #[case::reasoning( + "output_cost_per_reasoning_token", + key(Meter::Output(OutputUnit::ReasoningToken), Band::Base, None) + )] + #[case::cache_read_audio( + "cache_read_input_audio_token_cost", + key(Meter::CacheRead { kind: TokenKind::Audio, mode: CacheReadMode::Implicit }, Band::Base, None) + )] + #[case::explicit_cache_read( + "explicit_cache_read_input_token_cost", + key(Meter::CacheRead { kind: TokenKind::Text, mode: CacheReadMode::Explicit }, Band::Base, None) + )] + #[case::cache_write_one_hour_band( + "cache_creation_input_token_cost_above_1hr_above_200k_tokens", + key( + Meter::CacheWrite { kind: TokenKind::Text, ttl: CacheTtl::OneHour }, + Band::over_prompt_tokens(200_000), + None + ) + )] + #[case::cache_write_twelve_hours_priority( + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority", + key( + Meter::CacheWrite { kind: TokenKind::Text, ttl: CacheTtl::TwelveHours }, + Band::over_prompt_tokens(272_000), + Some(TierSuffix::Service(ServiceTier::Priority)) + ) + )] + #[case::image_label( + "output_cost_per_image_0.5K", + key( + Meter::Output(OutputUnit::Image(ImageSize::new("0.5K"))), + Band::Base, + None + ) + )] + #[case::image_token_is_a_token( + "output_cost_per_image_token", + key(Meter::Output(OutputUnit::Token(TokenKind::Image)), Band::Base, None) + )] + #[case::resolution( + "output_cost_per_second_720p", + key( + Meter::Output(OutputUnit::Second(Resolution::new("720p"))), + Band::Base, + None + ) + )] + #[case::video_interval( + "input_cost_per_video_per_second_above_8s_interval", + key(Meter::Input(InputUnit::VideoSecond { over_seconds: NonZeroU64::new(8) }), Band::Base, None) + )] + #[case::feature_batch( + "ocr_cost_per_page_batches", + key( + Meter::Feature(Feature::OcrPage), + Band::Base, + Some(TierSuffix::Batches) + ) + )] + #[case::wall_clock("cost_per_second", key(Meter::WallClockSecond, Band::Base, None))] + #[case::browser_open( + "browser_open_cost_per_request", + key(Meter::Feature(Feature::BrowserOpen), Band::Base, None) + )] + #[case::additional_result( + "cost_per_additional_result", + key(Meter::Feature(Feature::AdditionalResult), Band::Base, None) + )] + fn parses_every_key_shape(#[case] text: &str, #[case] expected: FlatKey) { + assert_eq!(FlatKey::parse(text), Some(expected)); + } + + #[rstest] + #[case::typo("input_cost_per_tokne")] + #[case::unused_meter("cost_per_token")] + #[case::other_currency("input_dbu_cost_per_token")] + #[case::cache_hit_alias("input_cost_per_token_cache_hit")] + #[case::non_numeric_band("input_cost_per_token_above_xk_tokens")] + #[case::interval_on_a_token_meter("input_cost_per_token_above_8s_interval")] + #[case::read_has_no_ttl("cache_read_input_token_cost_above_1hr")] + #[case::cache_unit_must_be_token("cache_read_input_character_cost")] + #[case::bad_image_size("output_cost_per_image_big")] + #[case::bad_resolution("output_cost_per_second_hd")] + #[case::unknown_feature("telepathy_cost_per_page")] + #[case::empty("")] + fn rejects_keys_outside_the_grammar(#[case] text: &str) { + assert_eq!(FlatKey::parse(text), None); + } + + #[rstest] + #[case::plain("input_cost_per_token", (Channel::Online, When::default()))] + #[case::service_tier( + "input_cost_per_token_flex", + (Channel::Online, When { tier: ServiceTier::Flex, band: Band::Base }) + )] + #[case::batch( + "input_cost_per_token_above_200k_tokens_batches", + (Channel::Batch, When { tier: ServiceTier::Standard, band: Band::over_prompt_tokens(200_000) }) + )] + #[case::decisions("input_cost_per_token_decisions", (Channel::Decisions, When::default()))] + fn batch_and_decisions_pick_a_channel_not_a_tier( + #[case] text: &str, + #[case] expected: (Channel, When), + ) { + assert_eq!(FlatKey::parse(text).unwrap().channel_and_when(), expected); + } +} diff --git a/litellm-rust/crates/model-catalog/src/wire/lax.rs b/litellm-rust/crates/model-catalog/src/wire/lax.rs new file mode 100644 index 00000000000..d514a6a82c1 --- /dev/null +++ b/litellm-rust/crates/model-catalog/src/wire/lax.rs @@ -0,0 +1,128 @@ +use std::fmt; + +use serde::Deserializer; +use serde::de::{self, Visitor}; +use serde_with::DeserializeAs; + +pub(crate) struct LaxI64; + +pub(crate) struct Count; + +impl<'de> DeserializeAs<'de, i64> for LaxI64 { + fn deserialize_as>(deserializer: D) -> Result { + deserializer.deserialize_any(Self) + } +} + +impl Visitor<'_> for LaxI64 { + type Value = i64; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("an integer in the i64 range") + } + + fn visit_i64(self, value: i64) -> Result { + Ok(value) + } + + fn visit_u64(self, value: u64) -> Result { + i64::try_from(value).map_err(E::custom) + } + + fn visit_f64(self, value: f64) -> Result { + integral_float(value).ok_or_else(|| E::custom("expected an integer in the i64 range")) + } + + fn visit_str(self, value: &str) -> Result { + integer_string(value.trim()) + .ok_or_else(|| E::custom("expected an integer in the i64 range")) + } + + fn visit_bool(self, value: bool) -> Result { + Ok(i64::from(value)) + } +} + +impl<'de> DeserializeAs<'de, u64> for Count { + fn deserialize_as>(deserializer: D) -> Result { + let count = LaxI64::deserialize_as(deserializer)?; + u64::try_from(count).map_err(|_| { + de::Error::invalid_value(de::Unexpected::Signed(count), &"a count of zero or more") + }) + } +} + +fn integer_string(value: &str) -> Option { + let integer = match value.split_once('.') { + Some((integer, fraction)) => { + if fraction.is_empty() || !fraction.bytes().all(|byte| byte == b'0') { + return None; + } + integer + } + None => value, + }; + if integer.starts_with('_') || integer.ends_with('_') || integer.contains("__") { + return None; + } + let digits = integer.strip_prefix(['+', '-']).unwrap_or(integer); + if digits.is_empty() + || digits.starts_with('_') + || !digits + .bytes() + .all(|byte| byte.is_ascii_digit() || byte == b'_') + { + return None; + } + integer.replace('_', "").parse().ok() +} + +fn integral_float(value: f64) -> Option { + (value.is_finite() + && value.fract() == 0.0 + && value >= i64::MIN as f64 + && value < -(i64::MIN as f64)) + .then_some(value as i64) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::{Value, json}; + use serde_with::de::DeserializeAsWrap; + + use super::*; + + fn count(value: Value) -> Option { + DeserializeAsWrap::::deserialize(value) + .ok() + .map(DeserializeAsWrap::into_inner) + } + + use serde::Deserialize; + + #[rstest] + #[case::integer(json!(128000), Some(128_000))] + #[case::whole_float(json!(128000.0), Some(128_000))] + #[case::string(json!(" 8192 "), Some(8192))] + #[case::separated(json!("1_000"), Some(1000))] + #[case::trailing_zeros(json!("2.000"), Some(2))] + #[case::boolean(json!(true), Some(1))] + #[case::maximum(json!(i64::MAX.to_string()), Some(i64::MAX as u64))] + #[case::negative(json!(-5), None)] + #[case::fraction(json!(2.5), None)] + #[case::precise_fraction(json!("1.0000000000000001"), None)] + #[case::exponent(json!("1e3"), None)] + #[case::missing_fraction(json!("2."), None)] + #[case::repeated_separator(json!("2__0"), None)] + #[case::above_i64(json!(u64::MAX), None)] + #[case::float_above_i64(json!(9_223_372_036_854_775_808.0), None)] + #[case::word(json!("8k"), None)] + #[case::object(json!({}), None)] + fn counts_accept_whole_numbers_however_they_are_written( + #[case] written: Value, + #[case] expected: Option, + ) { + assert_eq!(count(written), expected); + } +} diff --git a/litellm-rust/crates/model-catalog/src/wire/mod.rs b/litellm-rust/crates/model-catalog/src/wire/mod.rs new file mode 100644 index 00000000000..e8f23b7bd0d --- /dev/null +++ b/litellm-rust/crates/model-catalog/src/wire/mod.rs @@ -0,0 +1,10 @@ +mod attributes; +mod blocks; +mod entry; +mod flat_key; +mod lax; + +pub use attributes::*; +pub use blocks::*; +pub use entry::*; +pub use flat_key::*; diff --git a/litellm-rust/crates/model-catalog/tests/catalog.rs b/litellm-rust/crates/model-catalog/tests/catalog.rs index d384dfea9a8..30c070cfb52 100644 --- a/litellm-rust/crates/model-catalog/tests/catalog.rs +++ b/litellm-rust/crates/model-catalog/tests/catalog.rs @@ -1,303 +1,405 @@ -use std::path::{Path, PathBuf}; +use std::collections::BTreeSet; -use litellm_model_catalog::{AliasIssue, Catalog, Error, IntegrityLimits, Mode, Provenance}; +use litellm_model_catalog::{ + BUNDLED_REGISTRY, Catalog, Error, ErrorKind, IntegrityLimits, Mode, Rejects, Written, +}; use rstest::{fixture, rstest}; -use serde_json::json; +use serde_json::{Map, Value, json}; const ALPHA_FIXTURE: &[u8] = br#"{ - "sample_spec":{"explanation":"example"}, + "sample_spec":{"explanation":"example","input_cost_per_token":"not checked"}, "fallback_generalizations":{"rules":[{"name":"family","pattern":"^new-","model_info":{"mode":"chat"}}]}, "Alpha":{"litellm_provider":"test","aliases":["short"],"price":0,"enabled":false, "optional":null,"unknown":{"nested":[1,{"x":true}]}} }"#; -#[fixture] -fn repo_root() -> PathBuf { - Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..") -} - -#[fixture] -fn current_catalog(repo_root: PathBuf) -> Catalog { - let body = std::fs::read(repo_root.join("model_prices_and_context_window.json")).unwrap(); - Catalog::parse(&body, Provenance::default()).unwrap() -} - -#[fixture] -fn backup_catalog(repo_root: PathBuf) -> Catalog { - let body = std::fs::read(repo_root.join("litellm/model_prices_and_context_window_backup.json")) - .unwrap(); - Catalog::parse(&body, Provenance::default()).unwrap() -} - #[fixture] fn fixture_catalog() -> Catalog { - Catalog::parse( - ALPHA_FIXTURE, - Provenance { - source: Some("fixture".into()), - revision: Some("rev".into()), - etag: None, - }, - ) - .unwrap() + Catalog::parse(ALPHA_FIXTURE).unwrap().value +} + +fn loaded(document: Value) -> litellm_model_catalog::Loaded { + Catalog::parse(document.to_string().as_bytes()).unwrap() +} + +fn catalog(document: Value) -> Catalog { + loaded(document).value +} + +fn written(catalog: &Catalog) -> String { + let document: Map = catalog + .written() + .map(|(key, value)| { + let value = match value { + Written::Entry(fields) => serde_json::to_value(fields).unwrap(), + Written::Section(value) => value.clone(), + }; + (key.to_owned(), value) + }) + .collect(); + serde_json::to_string(&document).unwrap() +} + +fn report(error: &Error) -> String { + litellm_error::Report::new(error).to_string() +} + +fn error_text(document: Value) -> Vec { + loaded(document).errors.iter().map(report).collect() } #[rstest] #[case::decisions("decisions", Mode::Decisions)] #[case::legacy_evaluation("evaluation", Mode::Evaluation)] fn parses_decision_modes(#[case] mode: &str, #[case] expected: Mode) { - let source = json!({"Test":{"litellm_provider":"test","mode":mode}}).to_string(); - let catalog = Catalog::parse(source.as_bytes(), Provenance::default()).unwrap(); - + let catalog = catalog(json!({"Test": {"litellm_provider": "test", "mode": mode}})); assert_eq!( - catalog.lookup("Test").unwrap().entry.info().mode, + catalog + .lookup("Test") + .unwrap() + .entry + .model() + .unwrap() + .attributes + .mode, Some(expected) ); - assert_eq!(serde_json::to_value(expected).unwrap(), json!(mode)); } #[rstest] -#[ignore] -fn preserves_fields_and_metadata(fixture_catalog: Catalog) { - let catalog = fixture_catalog; - let entry = catalog.lookup("SHORT").unwrap(); - assert_eq!(entry.canonical_key, "Alpha"); - assert_eq!(entry.matched_key, "short"); - assert_eq!(entry.entry.field("price"), Some(&json!(0))); - assert_eq!(entry.entry.field("enabled"), Some(&json!(false))); - assert_eq!(entry.entry.field("optional"), Some(&json!(null))); - assert_eq!(entry.entry.field("missing"), None); +fn unknown_keys_are_kept_raw_and_listed_as_unrecognized(fixture_catalog: Catalog) { + let entry = fixture_catalog.lookup("Alpha").unwrap().entry; assert_eq!( - entry.entry.field("unknown"), + entry.field("unknown"), Some(&json!({"nested":[1,{"x":true}]})) ); - assert_eq!(entry.entry.field("aliases"), None); - assert_eq!(entry.entry.info().litellm_provider.as_deref(), Some("test")); assert_eq!( - catalog.sample_spec(), - Some(&json!({"explanation":"example"})) - ); - assert_eq!(catalog.fallback_rules().unwrap().len(), 1); - assert_eq!(catalog.provenance().revision.as_deref(), Some("rev")); - assert_eq!(catalog.model_count(), 1); -} - -#[rstest] -#[ignore] -fn snapshot_does_not_borrow_source() { - let mut source = ALPHA_FIXTURE.to_vec(); - let catalog = Catalog::parse(&source, Provenance::default()).unwrap(); - source.fill(b' '); - - let entry = catalog.lookup("short").unwrap(); - assert_eq!(entry.canonical_key, "Alpha"); - assert_eq!(entry.entry.field("price"), Some(&json!(0))); -} - -#[rstest] -#[case("Shared", "First")] -#[case("Second", "Second")] -#[case("shared", "Second")] -#[case("FIRST", "First")] -#[case("sHaReD", "Second")] -#[ignore] -fn alias_collisions_and_case_fallback_follow_entry_order( - #[case] lookup: &str, - #[case] expected: &str, -) { - let catalog = Catalog::parse( - br#"{ - "First":{"aliases":["Shared","Second","first"],"value":1}, - "Second":{"aliases":["Shared","sHaReD"],"value":2}, - "SHARED":{"value":3} - }"#, - Provenance::default(), - ) - .unwrap(); - assert_eq!(catalog.lookup(lookup).unwrap().canonical_key, expected); - assert_eq!(catalog.alias_count(), 3); - assert!( - catalog - .alias_issues() - .contains(&AliasIssue::CanonicalCollision { - model: "First".into(), - alias: "Second".into(), - }) - ); - assert!( - catalog - .alias_issues() - .contains(&AliasIssue::AliasCollision { - model: "Second".into(), - alias: "Shared".into(), - }) + fixture_catalog.unrecognized().collect::>(), + BTreeSet::from([ + ("Alpha", "enabled"), + ("Alpha", "price"), + ("Alpha", "unknown") + ]) ); } -#[test] -#[ignore] -fn json_entry_order_controls_alias_ownership_and_case_fallback() { - let forward = Catalog::parse( - br#"{ - "Alpha":{"aliases":["shared"]}, - "Beta":{"aliases":["shared"]}, - "Foo":{}, - "fOO":{} - }"#, - Provenance::default(), - ) - .unwrap(); - let reversed = Catalog::parse( - br#"{ - "fOO":{}, - "Foo":{}, - "Beta":{"aliases":["shared"]}, - "Alpha":{"aliases":["shared"]} - }"#, - Provenance::default(), - ) - .unwrap(); - - assert_eq!(forward.lookup("shared").unwrap().canonical_key, "Alpha"); - assert_eq!(reversed.lookup("shared").unwrap().canonical_key, "Beta"); - assert_eq!(forward.lookup("foo").unwrap().canonical_key, "fOO"); - assert_eq!(reversed.lookup("foo").unwrap().canonical_key, "Foo"); -} - -#[derive(Debug)] -enum ValidationOutcome { - Ok, - Shrunk, - BelowMinimum, - InvalidRatio, -} - #[rstest] -#[case( - IntegrityLimits { - reference_model_count: 2, - min_model_count: 1, - min_reference_ratio: 0.5, - }, - ValidationOutcome::Ok -)] -#[case( - IntegrityLimits { - reference_model_count: 3, - min_model_count: 1, - min_reference_ratio: 0.5, - }, - ValidationOutcome::Shrunk -)] -#[case( - IntegrityLimits { - reference_model_count: 0, - min_model_count: 2, - min_reference_ratio: 0.5, - }, - ValidationOutcome::BelowMinimum -)] -#[case( - IntegrityLimits { - reference_model_count: 0, - min_model_count: 0, - min_reference_ratio: f64::NAN, - }, - ValidationOutcome::InvalidRatio -)] -#[ignore] -fn integrity_uses_canonical_count_and_strict_shrink_boundary( - #[case] limits: IntegrityLimits, - #[case] expected: ValidationOutcome, -) { - let catalog = Catalog::parse( - br#"{"sample_spec":{},"fallback_generalizations":{"rules":[]},"a":{"aliases":["b","c"]}}"#, - Provenance::default(), - ) - .unwrap(); - let actual = catalog.validate(limits); - match expected { - ValidationOutcome::Ok => assert!(actual.is_ok()), - ValidationOutcome::Shrunk => { - assert!(matches!(actual, Err(Error::Shrunk { actual: 1, .. }))) - } - ValidationOutcome::BelowMinimum => { - assert!(matches!(actual, Err(Error::BelowMinimum { actual: 1, .. }))) - } - ValidationOutcome::InvalidRatio => assert!(matches!(actual, Err(Error::InvalidRatio))), - } -} - -#[derive(Debug)] -enum MalformedOutcome { - Empty, - Json, - EntryNotObject, -} - -#[rstest] -#[case::empty(b"{}", MalformedOutcome::Empty)] -#[case::invalid_json(b"{", MalformedOutcome::Json)] -#[case::entry_not_object(br#"{"a":1}"#, MalformedOutcome::EntryNotObject)] -#[case::fallback_rules_missing( - br#"{"fallback_generalizations":{},"a":{}}"#, - MalformedOutcome::Json -)] -#[ignore] -fn malformed_input_and_aliases_have_typed_outcomes( - #[case] body: &[u8], - #[case] expected: MalformedOutcome, -) { - let actual = Catalog::parse(body, Provenance::default()); - match expected { - MalformedOutcome::Empty => assert!(matches!(actual, Err(Error::Empty))), - MalformedOutcome::Json => assert!(matches!(actual, Err(Error::Json(_)))), - MalformedOutcome::EntryNotObject => { - assert!(matches!(actual, Err(Error::EntryNotObject { .. }))) - } - } -} - -#[rstest] -#[ignore] -fn invalid_aliases_are_reported_not_fatal() { - let catalog = Catalog::parse( - br#"{"a":{"aliases":"bad"},"b":{"aliases":[9,"ok"]}}"#, - Provenance::default(), - ) - .unwrap(); +fn sections_aliases_and_raw_fields_are_read_in_their_document_views(fixture_catalog: Catalog) { + let found = fixture_catalog.lookup("SHORT").unwrap(); + assert_eq!(found.canonical_key, "Alpha"); + assert_eq!(found.matched_key, "short"); + assert_eq!(found.entry.field("optional"), Some(&json!(null))); + assert_eq!(found.entry.field("aliases"), Some(&json!(["short"]))); assert_eq!( - catalog.alias_issues(), - &[ - AliasIssue::InvalidList { model: "a".into() }, - AliasIssue::InvalidName { model: "b".into() }, - ] + fixture_catalog.sample_spec(), + Some(&json!({"explanation":"example","input_cost_per_token":"not checked"})) + ); + assert_eq!(fixture_catalog.fallback_rules().unwrap()[0].name, "family"); + assert_eq!(fixture_catalog.model_count(), 1); + assert_eq!( + fixture_catalog + .served() + .map(|(key, _)| key) + .collect::>(), + ["Alpha", "short"] ); - assert_eq!(catalog.lookup("ok").unwrap().canonical_key, "b"); - assert!(catalog.lookup("missing").is_none()); } #[rstest] -#[ignore] -fn parses_current_and_packaged_catalogs_against_independent_baseline( - current_catalog: Catalog, - backup_catalog: Catalog, -) { - assert!(current_catalog.model_count() > 0); - assert!(backup_catalog.model_count() > 0); - assert!(current_catalog.sample_spec().is_some()); - assert!(backup_catalog.sample_spec().is_some()); - // Snapshot from 2026-09-23; the backup file mirrors the current file and cannot detect shrinkage. - const REFERENCE_MODEL_COUNT: usize = 4303; - current_catalog - .validate(IntegrityLimits { - reference_model_count: REFERENCE_MODEL_COUNT, - min_model_count: 50, - min_reference_ratio: 0.9, - }) - .unwrap(); - assert!(current_catalog.model_names().all(|name| { - let entry = current_catalog.lookup(name).unwrap().entry; - entry.info().litellm_provider.is_some() == entry.field("litellm_provider").is_some() +fn a_bad_entry_is_reported_by_model_and_key_and_kept_raw() { + let parsed = loaded(json!({ + "Good": {"litellm_provider": "test"}, + "Bad": {"litellm_provider": "test", "mode": "chatt", "input_cost_per_token": "free"}, + "Worse": {"mode": "chat"} })); + let catalog = parsed.value; + let problems: Vec = parsed.errors.iter().map(report).collect(); + assert_eq!(problems.len(), 3, "{problems:?}"); + assert!(problems[0].starts_with("Bad.mode: invalid value: unknown variant `chatt`")); + assert!(problems[1].starts_with("Bad.input_cost_per_token: invalid value")); + assert_eq!(problems[2], "Worse.litellm_provider: required"); + assert!(catalog.lookup("Good").unwrap().entry.model().is_some()); + assert!(catalog.lookup("Bad").unwrap().entry.model().is_none()); + assert_eq!(catalog.model_count(), 3); + assert!( + parsed + .errors + .iter() + .find(|error| error.rejects() == Rejects::Document) + .is_none() + ); +} + +#[rstest] +#[case::alias_names_a_model( + json!({"First": {"litellm_provider": "t", "aliases": ["Second"]}, "Second": {"litellm_provider": "t"}}), + r#"First: alias "Second" is already a model name"# +)] +#[case::alias_claimed_twice( + json!({"First": {"litellm_provider": "t", "aliases": ["shared"]}, "Second": {"litellm_provider": "t", "aliases": ["shared"]}}), + r#"Second: alias "shared" already belongs to First"# +)] +#[case::aliases_not_a_list( + json!({"a": {"litellm_provider": "t", "aliases": "bad"}}), + "a.aliases: invalid value: invalid type: string \"bad\", expected a sequence" +)] +#[case::entry_not_object( + json!({"a": 1, "valid": {"litellm_provider": "test"}}), + "a: must be a JSON object" +)] +#[case::fallback_rules_missing( + json!({"fallback_generalizations": {}, "a": {"litellm_provider": "t"}}), + "fallback_generalizations: invalid value: missing field `rules`" +)] +#[case::unknown_timezone( + json!({"a": {"litellm_provider": "t", "off_peak_pricing": { + "windows": [{"hours_utc": "00:00-01:00"}], + "weekday_timezone": "Not/AZone", + "input_cost_per_token": 1e-6 + }}}), + r#"a.off_peak_pricing.weekday_timezone: "Not/AZone" is not an IANA time zone"# +)] +fn document_level_problems_are_reported(#[case] document: Value, #[case] expected: &str) { + assert_eq!(error_text(document), [expected]); +} + +#[rstest] +fn escaped_row_paths_and_row_validation_share_the_parse_rules() { + let document = json!({ + "open/router~v1": {"litellm_provider": "test", "input_cost_per_token": -1} + }); + let parsed = loaded(document); + let fields = parsed + .value + .lookup("open/router~v1") + .unwrap() + .entry + .fields(); + let row_errors = Catalog::row_errors("open/router~v1", fields); + assert_eq!( + row_errors + .iter() + .map(|error| (error.path(), report(error))) + .collect::>(), + parsed + .errors + .iter() + .map(|error| (error.path(), report(error))) + .collect::>() + ); + assert_eq!( + row_errors[0].path(), + "/open~1router~0v1/input_cost_per_token" + ); + assert!(row_errors[0].kind() == ErrorKind::Invalid); +} + +#[rstest] +fn malformed_fallback_is_collected_while_models_are_read() { + let parsed = loaded(json!({ + "fallback_generalizations": {"rules": "bad"}, + "a": {"litellm_provider": "test"} + })); + assert_eq!(parsed.value.model_count(), 1); + assert_eq!(parsed.errors.len(), 1); + assert_eq!(parsed.errors[0].path(), "/fallback_generalizations"); +} + +#[rstest] +fn rejected_rows_and_their_aliases_are_removed_without_disturbing_written_order() { + let body = r#"{"before":{"litellm_provider":"test","aliases":["before-alias"]},"bad":{"litellm_provider":"test","aliases":["bad-alias"],"mode":"chatt"},"not-object":4,"sample_spec":{"keep":true},"after":{"litellm_provider":"test"}}"#; + let parsed = Catalog::parse(body.as_bytes()).unwrap(); + let (catalog, removed) = parsed.value.without_rejected(&parsed.errors); + + assert_eq!(removed.len(), 2); + assert_eq!( + catalog.lookup("before-alias").unwrap().canonical_key, + "before" + ); + assert_eq!( + catalog.lookup("BEFORE-ALIAS").unwrap().matched_key, + "before-alias" + ); + assert!(catalog.lookup("bad").is_none()); + assert!(catalog.lookup("BAD").is_none()); + assert!(catalog.lookup("Bad-Alias").is_none()); + assert_eq!(catalog.model_count(), 2); + assert_eq!( + written(&catalog), + r#"{"before":{"litellm_provider":"test","aliases":["before-alias"]},"sample_spec":{"keep":true},"after":{"litellm_provider":"test"}}"# + ); +} + +#[rstest] +fn invalid_empty_key_rows_are_removed() { + let parsed = loaded(json!({ + "": {"litellm_provider": "test", "mode": "chatt"}, + "good": {"litellm_provider": "test"} + })); + assert_eq!(parsed.errors[0].path(), "//mode"); + + let (catalog, removed) = parsed.value.without_rejected(&parsed.errors); + assert_eq!(catalog.model_count(), 1); + assert!(catalog.lookup("").is_none()); + assert!(catalog.lookup("good").is_some()); + assert_eq!(removed.len(), 1); +} + +#[rstest] +fn timezone_errors_reject_providerless_rows_without_rejecting_other_providerless_rows() { + let parsed = loaded(json!({ + "good": {"mode": "chat"}, + "bad": {"off_peak_pricing": { + "windows": [{"hours_utc": "00:00-01:00"}], + "weekday_timezone": "Not/AZone", + "input_cost_per_token": 1e-6 + }} + })); + assert_eq!(parsed.errors.len(), 3); + assert!( + parsed + .errors + .iter() + .any(|error| error.kind() == ErrorKind::UnknownTimezone) + ); + + let (catalog, removed) = parsed.value.without_rejected(&parsed.errors); + assert_eq!(catalog.model_count(), 1); + assert!(catalog.lookup("good").is_some()); + assert!(catalog.lookup("bad").is_none()); + assert_eq!(removed.len(), 2); +} + +#[rstest] +fn bundled_per_row_errors_match_document_errors() { + let parsed = Catalog::parse(BUNDLED_REGISTRY).unwrap(); + let row_errors: Vec<_> = parsed + .value + .written() + .flat_map(|(key, written)| match written { + Written::Entry(fields) => Catalog::row_errors(key, fields), + Written::Section(_) => Vec::new(), + }) + .map(|error| (error.path().to_owned(), report(&error))) + .collect(); + let document_errors: Vec<_> = parsed + .errors + .iter() + .map(|error| (error.path().to_owned(), report(error))) + .collect(); + + assert_eq!(row_errors, document_errors); +} + +#[rstest] +fn three_hundred_rows_filter_one_row_with_two_invalid_prices() { + let valid_rows: Map = (0..299) + .map(|index| { + ( + format!("valid-{index:03}"), + json!({"litellm_provider": "test"}), + ) + }) + .collect(); + let mut document = valid_rows; + document.insert( + "bad".to_owned(), + json!({ + "litellm_provider": "test", + "input_cost_per_token": -1, + "output_cost_per_token": -1 + }), + ); + let parsed = Catalog::parse(serde_json::to_vec(&document).unwrap().as_slice()).unwrap(); + assert_eq!(parsed.value.model_count(), 300); + assert_eq!(parsed.errors.len(), 2); + + let (catalog, removed) = parsed.value.without_rejected(&parsed.errors); + assert_eq!(catalog.model_count(), 299); + assert_eq!(removed.len(), 2); +} + +#[rstest] +fn integrity_limits_count_canonical_models_not_aliases() { + let fetched = catalog(json!({"a": {"litellm_provider": "t", "aliases": ["b", "c"]}})); + assert!( + fetched + .check_size(2, IntegrityLimits::new(1, 0.5).unwrap()) + .is_ok() + ); + assert_eq!( + fetched + .check_size(3, IntegrityLimits::new(1, 0.5).unwrap()) + .unwrap_err() + .kind(), + ErrorKind::Shrunk + ); + assert_eq!( + fetched + .check_size(2, IntegrityLimits::new(2, 0.5).unwrap()) + .unwrap_err() + .kind(), + ErrorKind::BelowMinimum + ); +} + +#[rstest] +#[case::empty(b"{}", "empty")] +#[case::only_sections(br#"{"sample_spec":{}}"#, "empty")] +#[case::invalid_json(b"{", "json")] +#[case::array_root_is_json_error(b"[]", "json")] +fn malformed_documents_are_rejected_whole(#[case] body: &[u8], #[case] kind: &str) { + let error = Catalog::parse(body).unwrap_err(); + assert_eq!(error.rejects(), Rejects::Document); + assert_eq!(<&str>::from(error.kind()), kind); +} + +#[rstest] +fn written_is_the_document_in_order_including_sections_and_bad_entries() { + let body = concat!( + r#"{"b":{"litellm_provider":"t","z":1,"a":null},"sample_spec":{"x":"y"},"n":4,"#, + r#""fallback_generalizations":{"rules":"bad"},"a":{"mode":"chatt","price":1.0}}"# + ); + let catalog = Catalog::parse(body.as_bytes()).unwrap().value; + assert_eq!(written(&catalog), body); + assert_eq!(catalog.model_count(), 2); +} + +#[rstest] +fn the_bundled_registry_parses_with_no_problems_and_every_key_understood() { + let loaded = Catalog::parse(BUNDLED_REGISTRY).unwrap(); + let catalog = loaded.value; + assert!(loaded.errors.is_empty(), "{:#?}", loaded.errors); + let unrecognized: Vec<_> = catalog.unrecognized().collect(); + assert!(unrecognized.is_empty(), "{unrecognized:#?}"); + assert!( + catalog + .fallback_rules() + .is_some_and(|rules| !rules.is_empty()) + ); + const REFERENCE_MODEL_COUNT: usize = 4303; + assert!(catalog.model_count() as f64 >= REFERENCE_MODEL_COUNT as f64 * 0.9); +} + +#[rstest] +fn the_bundled_registry_is_written_back_as_it_was_read() { + let document: Value = serde_json::from_slice(BUNDLED_REGISTRY).unwrap(); + assert_eq!( + written(&Catalog::parse(BUNDLED_REGISTRY).unwrap().value), + serde_json::to_string(&document).unwrap() + ); +} + +#[rstest] +fn a_bad_value_keeps_its_json_error_as_the_source_and_names_its_key_and_field() { + let parsed = + loaded(json!({"open/router": {"litellm_provider": "t", "input_cost_per_token": "free"}})); + let error = &parsed.errors[0]; + let cause = litellm_error::find::(error) + .expect("the JSON error is in the chain") + .to_string(); + assert_eq!(report(error).matches(&cause).count(), 1); + assert!(!error.to_string().contains(&cause)); + assert_eq!(error.key().as_deref(), Some("open/router")); + assert_eq!(error.field().as_deref(), Some("input_cost_per_token")); } diff --git a/litellm-rust/crates/model-catalog/tests/generate_price_lookup_reference.py b/litellm-rust/crates/model-catalog/tests/generate_price_lookup_reference.py new file mode 100644 index 00000000000..e49b635ff4a --- /dev/null +++ b/litellm-rust/crates/model-catalog/tests/generate_price_lookup_reference.py @@ -0,0 +1,91 @@ +import itertools +import json +import re +import subprocess +from pathlib import Path +from typing import Final + +from litellm.litellm_core_utils.llm_cost_calc.utils import _get_token_base_cost +from litellm.types.utils import Usage + +REPO_ROOT: Final = Path(__file__).resolve().parents[4] +TOKEN_RATE_KEY: Final = re.compile( + r"^(input_cost_per_token|output_cost_per_token|cache_read_input_token_cost" + r"|cache_creation_input_token_cost(_above_1hr)?)" + r"(_above_(\d+)(k?)_tokens)?(_(priority|flex|ultrafast|balanced|batches|decisions))?$" +) +SERVICE_TIERS: Final = ("priority", "flex", "ultrafast", "balanced") +LARGEST_PROMPT: Final = 10_000_000 +INCLUSIVE_PROVIDERS: Final = frozenset({"xai"}) + + +def token_rates(entry: dict[str, object]) -> dict[str, object]: + return { + key: value + for key, value in entry.items() + if value is not None and (key == "tiered_pricing" or TOKEN_RATE_KEY.match(key)) + } + + +def band_floors(info: dict[str, object]) -> frozenset[int]: + matches: Final = (TOKEN_RATE_KEY.match(key) for key in info) + flat: Final = frozenset( + int(match.group(4)) * (1000 if match.group(5) else 1) + for match in matches + if match is not None and match.group(4) is not None + ) + rows: Final = info.get("tiered_pricing") + tiers: Final = ( + frozenset(int(row["range"][0]) for row in rows if isinstance(row, dict) and "range" in row) + if isinstance(rows, list) + else frozenset() + ) + return flat | tiers + + +def tiers_in(info: dict[str, object]) -> tuple[str | None, ...]: + return (None, *(tier for tier in SERVICE_TIERS if any(key.endswith(f"_{tier}") for key in info))) + + +def prompts(info: dict[str, object]) -> tuple[int, ...]: + floors: Final = band_floors(info) + return tuple(sorted(({1, LARGEST_PROMPT} | floors | {floor + 1 for floor in floors}) - {0})) + + +def check(info: dict[str, object], tier: str | None, prompt: int, inclusive: bool) -> list[object]: + usage: Final = Usage(prompt_tokens=prompt, completion_tokens=1, total_tokens=prompt + 1) + rates: Final = _get_token_base_cost(info, usage, service_tier=tier, threshold_is_inclusive=inclusive) + one_hour: Final = rates.cache_creation_rates_by_ttl.get("1hr", rates.cache_creation_rate) + return [tier, prompt, [rates.input_rate, rates.output_rate, rates.cache_creation_rate, one_hour, rates.cache_read_rate]] + + +def shape(info: dict[str, object], inclusive: bool) -> str: + rows: Final = info.get("tiered_pricing") + row_keys: Final = [sorted(row) for row in rows if isinstance(row, dict)] if isinstance(rows, list) else [] + return json.dumps([sorted(info), row_keys, inclusive]) + + +def checks(info: dict[str, object], inclusive: bool) -> list[list[object]]: + return [check(info, tier, prompt, inclusive) for tier, prompt in itertools.product(tiers_in(info), prompts(info))] + + +def main() -> None: + registry: Final = json.loads((REPO_ROOT / "model_prices_and_context_window.json").read_text()) + entries: Final = tuple( + (token_rates(entry), entry.get("litellm_provider") in INCLUSIVE_PROVIDERS) + for name, entry in registry.items() + if name not in ("sample_spec", "fallback_generalizations") and isinstance(entry, dict) + ) + by_shape: Final = {shape(info, inclusive): (info, inclusive) for info, inclusive in reversed(entries) if info} + models: Final = tuple( + json.dumps({"info": info, "inclusive": inclusive, "checks": checks(info, inclusive)}) + for info, inclusive in by_shape.values() + ) + revision: Final = subprocess.check_output(("git", "rev-parse", "HEAD"), text=True, cwd=REPO_ROOT).strip() + Path(__file__).with_name("price_lookup_reference.json").write_text( + f'{{"python_reference_commit": "{revision}", "models": [\n' + ",\n".join(models) + "\n]}\n" + ) + + +if __name__ == "__main__": + main() diff --git a/litellm-rust/crates/model-catalog/tests/price_lookup.rs b/litellm-rust/crates/model-catalog/tests/price_lookup.rs new file mode 100644 index 00000000000..cc16e9f23e9 --- /dev/null +++ b/litellm-rust/crates/model-catalog/tests/price_lookup.rs @@ -0,0 +1,89 @@ +use litellm_cost::money::UnitPrice; +use litellm_cost::price::{ + CacheReadMode, CacheTtl, InputUnit, Meter, OutputUnit, Prices, ServiceTier, TokenKind, +}; +use litellm_model_catalog::wire::{EntryWire, Fields}; +use rstest::rstest; +use serde::Deserialize; + +#[derive(Deserialize)] +struct Reference { + models: Vec, +} + +#[derive(Deserialize)] +struct ReferenceModel { + info: Fields, + inclusive: bool, + checks: Vec<(Option, u64, [f64; 5])>, +} + +const METERS: [Meter; 5] = [ + Meter::Input(InputUnit::Token(TokenKind::Text)), + Meter::Output(OutputUnit::Token(TokenKind::Text)), + Meter::CacheWrite { + kind: TokenKind::Text, + ttl: CacheTtl::Default, + }, + Meter::CacheWrite { + kind: TokenKind::Text, + ttl: CacheTtl::OneHour, + }, + Meter::CacheRead { + kind: TokenKind::Text, + mode: CacheReadMode::Implicit, + }, +]; + +fn service_tier(name: Option<&str>) -> ServiceTier { + match name { + None => ServiceTier::Standard, + Some("priority") => ServiceTier::Priority, + Some("flex") => ServiceTier::Flex, + Some("ultrafast") => ServiceTier::Ultrafast, + Some("balanced") => ServiceTier::Balanced, + Some(other) => panic!("the reference uses an unknown service tier {other}"), + } +} + +fn mismatch( + prices: &Prices, + (tier_name, prompt, python): &(Option, u64, [f64; 5]), +) -> Option { + let tier = service_tier(tier_name.as_deref()); + let band = prices.band(tier, *prompt); + let rust = METERS + .each_ref() + .map(|meter| prices.rate(meter, tier, band).map_or(0.0, UnitPrice::get)); + (rust != *python) + .then(|| format!("tier={tier_name:?} prompt={prompt}: rust {rust:?}, python {python:?}")) +} + +#[rstest] +fn band_and_rate_match_python_token_base_cost() { + let reference: Reference = + serde_json::from_str(include_str!("price_lookup_reference.json")).unwrap(); + + let mismatches: Vec = reference + .models + .iter() + .flat_map(|model| { + let provider = if model.inclusive { "xai" } else { "openai" }; + let fields: Fields = model + .info + .clone() + .into_iter() + .chain([("litellm_provider".to_owned(), provider.into())]) + .collect(); + let prices = Prices::try_from(&EntryWire::read(&fields)).unwrap(); + model + .checks + .iter() + .filter_map(move |check| mismatch(&prices, check)) + .map(move |found| format!("{:?} {found}", model.info)) + }) + .collect(); + + assert!(reference.models.len() > 50); + assert!(mismatches.is_empty(), "{mismatches:#?}"); +} diff --git a/litellm-rust/crates/model-catalog/tests/price_lookup_reference.json b/litellm-rust/crates/model-catalog/tests/price_lookup_reference.json new file mode 100644 index 00000000000..4b87b696723 --- /dev/null +++ b/litellm-rust/crates/model-catalog/tests/price_lookup_reference.json @@ -0,0 +1,89 @@ +{"python_reference_commit": "c7c895db456eda349a7be9661b4b21f578f9b589", "models": [ +{"info": {"cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05, "cache_read_input_token_cost": 1e-06, "cache_read_input_token_cost_above_272k_tokens": 2e-06, "input_cost_per_token": 1e-05, "input_cost_per_token_above_272k_tokens": 2e-05, "output_cost_per_token": 5e-05, "output_cost_per_token_above_272k_tokens": 7.5e-05}, "inclusive": false, "checks": [[null, 1, [1e-05, 5e-05, 1.25e-05, 1.25e-05, 1e-06]], [null, 272000, [1e-05, 5e-05, 1.25e-05, 1.25e-05, 1e-06]], [null, 272001, [2e-05, 7.5e-05, 2.5e-05, 2.5e-05, 2e-06]], [null, 10000000, [2e-05, 7.5e-05, 2.5e-05, 2.5e-05, 2e-06]]]}, +{"info": {"cache_read_input_token_cost": 1e-07, "input_cost_per_token": 1e-06, "output_cost_per_token": 3e-06}, "inclusive": false, "checks": [[null, 1, [1e-06, 3e-06, 1e-06, 1e-06, 1e-07]], [null, 10000000, [1e-06, 3e-06, 1e-06, 1e-06, 1e-07]]]}, +{"info": {"input_cost_per_token": 1.25e-05, "output_cost_per_token": 1.25e-05}, "inclusive": false, "checks": [[null, 1, [1.25e-05, 1.25e-05, 1.25e-05, 1.25e-05, 1.25e-05]], [null, 10000000, [1.25e-05, 1.25e-05, 1.25e-05, 1.25e-05, 1.25e-05]]]}, +{"info": {"cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05, "cache_creation_input_token_cost_flex": 1.25e-06, "cache_creation_input_token_cost_priority": 5e-06, "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_272k_tokens": 4e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 2e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 8e-07, "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_above_272k_tokens_batches": 2e-07, "cache_creation_input_token_cost_batches": 1.25e-06, "cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-06, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 4e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, "input_cost_per_token_above_272k_tokens_flex": 2e-06, "input_cost_per_token_above_272k_tokens_priority": 8e-06, "input_cost_per_token_batches": 1e-06, "input_cost_per_token_above_272k_tokens_batches": 2e-06, "input_cost_per_token_flex": 1e-06, "input_cost_per_token_priority": 4e-06, "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, "output_cost_per_token_above_272k_tokens_flex": 7.5e-06, "output_cost_per_token_above_272k_tokens_priority": 3e-05, "output_cost_per_token_batches": 5e-06, "output_cost_per_token_above_272k_tokens_batches": 7.5e-06, "output_cost_per_token_flex": 5e-06, "output_cost_per_token_priority": 2e-05}, "inclusive": false, "checks": [[null, 1, [2e-06, 1e-05, 2.5e-06, 2.5e-06, 2e-07]], [null, 272000, [2e-06, 1e-05, 2.5e-06, 2.5e-06, 2e-07]], [null, 272001, [4e-06, 1.5e-05, 5e-06, 5e-06, 4e-07]], [null, 10000000, [4e-06, 1.5e-05, 5e-06, 5e-06, 4e-07]], ["priority", 1, [4e-06, 2e-05, 5e-06, 5e-06, 4e-07]], ["priority", 272000, [4e-06, 2e-05, 5e-06, 5e-06, 4e-07]], ["priority", 272001, [8e-06, 3e-05, 1e-05, 1e-05, 8e-07]], ["priority", 10000000, [8e-06, 3e-05, 1e-05, 1e-05, 8e-07]], ["flex", 1, [1e-06, 5e-06, 1.25e-06, 1.25e-06, 1e-07]], ["flex", 272000, [1e-06, 5e-06, 1.25e-06, 1.25e-06, 1e-07]], ["flex", 272001, [2e-06, 7.5e-06, 2.5e-06, 2.5e-06, 2e-07]], ["flex", 10000000, [2e-06, 7.5e-06, 2.5e-06, 2.5e-06, 2e-07]]]}, +{"info": {"cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_creation_input_token_cost_batches": 6.25e-07, "cache_read_input_token_cost": 1e-07, "cache_read_input_token_cost_batches": 5e-08, "input_cost_per_token": 1e-06, "input_cost_per_token_batches": 5e-07, "output_cost_per_token": 5e-06, "output_cost_per_token_batches": 2.5e-06}, "inclusive": false, "checks": [[null, 1, [1e-06, 5e-06, 1.25e-06, 2e-06, 1e-07]], [null, 10000000, [1e-06, 5e-06, 1.25e-06, 2e-06, 1e-07]]]}, +{"info": {"input_cost_per_token_above_100k_tokens_batches": 2.5e-07, "output_cost_per_token_above_100k_tokens_batches": 1.25e-06, "cache_creation_input_token_cost_above_100k_tokens_batches": 3.125e-07, "cache_read_input_token_cost_above_100k_tokens_batches": 2.5e-08, "cache_creation_input_token_cost": 1.25e-07, "cache_creation_input_token_cost_above_1hr": 2e-07, "cache_creation_input_token_cost_batches": 6.25e-08, "cache_read_input_token_cost": 1e-08, "cache_read_input_token_cost_batches": 5e-09, "input_cost_per_token": 1e-07, "input_cost_per_token_batches": 5e-08, "output_cost_per_token": 5e-07, "output_cost_per_token_batches": 2.5e-07, "input_cost_per_token_above_100k_tokens": 5e-07, "output_cost_per_token_above_100k_tokens": 2.5e-06, "cache_creation_input_token_cost_above_100k_tokens": 6.25e-07, "cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06, "cache_read_input_token_cost_above_100k_tokens": 5e-08}, "inclusive": false, "checks": [[null, 1, [1e-07, 5e-07, 1.25e-07, 2e-07, 1e-08]], [null, 100000, [1e-07, 5e-07, 1.25e-07, 2e-07, 1e-08]], [null, 100001, [5e-07, 2.5e-06, 6.25e-07, 1e-06, 5e-08]], [null, 10000000, [5e-07, 2.5e-06, 6.25e-07, 1e-06, 5e-08]]]}, +{"info": {"input_cost_per_token": 1.0000000000000001e-07, "output_cost_per_token": 5e-07, "cache_read_input_token_cost": 1e-08, "cache_creation_input_token_cost": 1.25e-07, "tiered_pricing": [{"range": [0, 272000], "input_cost_per_token": 1.0000000000000001e-07, "output_cost_per_token": 5e-07, "cache_read_input_token_cost": 1e-08, "cache_creation_input_token_cost": 1.25e-07}, {"range": [272000, 1050000], "input_cost_per_token": 2.0000000000000002e-07, "output_cost_per_token": 7.5e-07, "cache_read_input_token_cost": 2e-08, "cache_creation_input_token_cost": 2.5e-07}]}, "inclusive": false, "checks": [[null, 1, [1.0000000000000001e-07, 5e-07, 1.25e-07, 1.25e-07, 1e-08]], [null, 272000, [1.0000000000000001e-07, 5e-07, 1.25e-07, 1.25e-07, 1e-08]], [null, 272001, [2.0000000000000002e-07, 7.5e-07, 2.5e-07, 2.5e-07, 2e-08]], [null, 10000000, [2.0000000000000002e-07, 7.5e-07, 2.5e-07, 2.5e-07, 2e-08]]]}, +{"info": {"cache_creation_input_token_cost": 1e-06, "cache_read_input_token_cost": 8e-08, "input_cost_per_token": 8e-07, "output_cost_per_token": 4e-06}, "inclusive": false, "checks": [[null, 1, [8e-07, 4e-06, 1e-06, 1e-06, 8e-08]], [null, 10000000, [8e-07, 4e-06, 1e-06, 1e-06, 8e-08]]]}, +{"info": {"input_cost_per_token": 2e-06, "output_cost_per_token": 6e-06, "cache_read_input_token_cost": 5e-07, "tiered_pricing": [{"range": [0, 200000], "input_cost_per_token": 2e-06, "output_cost_per_token": 6e-06, "cache_read_input_token_cost": 5e-07}, {"range": [200000, 500000], "input_cost_per_token": 4e-06, "output_cost_per_token": 1.2e-05, "cache_read_input_token_cost": 1e-06}]}, "inclusive": false, "checks": [[null, 1, [2e-06, 6e-06, 2e-06, 2e-06, 5e-07]], [null, 200000, [2e-06, 6e-06, 2e-06, 2e-06, 5e-07]], [null, 200001, [4e-06, 1.2e-05, 4e-06, 4e-06, 1e-06]], [null, 10000000, [4e-06, 1.2e-05, 4e-06, 4e-06, 1e-06]]]}, +{"info": {"cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_priority": 1.5e-07, "input_cost_per_token": 7.5e-07, "input_cost_per_token_priority": 1.5e-06, "output_cost_per_token": 4.5e-06, "output_cost_per_token_priority": 9e-06}, "inclusive": false, "checks": [[null, 1, [7.5e-07, 4.5e-06, 7.5e-07, 7.5e-07, 7.5e-08]], [null, 10000000, [7.5e-07, 4.5e-06, 7.5e-07, 7.5e-07, 7.5e-08]], ["priority", 1, [1.5e-06, 9e-06, 1.5e-06, 1.5e-06, 1.5e-07]], ["priority", 10000000, [1.5e-06, 9e-06, 1.5e-06, 1.5e-06, 1.5e-07]]]}, +{"info": {"cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06}, "inclusive": false, "checks": [[null, 1, [1.25e-06, 2.5e-06, 1.25e-06, 1.25e-06, 2e-07]], [null, 200000, [1.25e-06, 2.5e-06, 1.25e-06, 1.25e-06, 2e-07]], [null, 200001, [2.5e-06, 5e-06, 2.5e-06, 2.5e-06, 4e-07]], [null, 10000000, [2.5e-06, 5e-06, 2.5e-06, 2.5e-06, 4e-07]]]}, +{"info": {"cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 1e-06, "output_cost_per_token": 5e-06}, "inclusive": false, "checks": [[null, 1, [1e-06, 5e-06, 1.25e-06, 2e-06, 1e-07]], [null, 10000000, [1e-06, 5e-06, 1.25e-06, 2e-06, 1e-07]]]}, +{"info": {"cache_read_input_token_cost": 8.25e-08, "cache_read_input_token_cost_flex": 4.125e-08, "cache_read_input_token_cost_priority": 1.44375e-07, "input_cost_per_token": 3.3e-07, "input_cost_per_token_batches": 1.65e-07, "input_cost_per_token_flex": 1.65e-07, "input_cost_per_token_priority": 5.775e-07, "output_cost_per_token": 2.75e-06, "output_cost_per_token_batches": 1.375e-06, "output_cost_per_token_flex": 1.375e-06, "output_cost_per_token_priority": 4.8125e-06}, "inclusive": false, "checks": [[null, 1, [3.3e-07, 2.75e-06, 3.3e-07, 3.3e-07, 8.25e-08]], [null, 10000000, [3.3e-07, 2.75e-06, 3.3e-07, 3.3e-07, 8.25e-08]], ["priority", 1, [5.775e-07, 4.8125e-06, 5.775e-07, 5.775e-07, 1.44375e-07]], ["priority", 10000000, [5.775e-07, 4.8125e-06, 5.775e-07, 5.775e-07, 1.44375e-07]], ["flex", 1, [1.65e-07, 1.375e-06, 1.65e-07, 1.65e-07, 4.125e-08]], ["flex", 10000000, [1.65e-07, 1.375e-06, 1.65e-07, 1.65e-07, 4.125e-08]]]}, +{"info": {"cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "cache_read_input_token_cost_priority": 1.5e-07, "input_cost_per_token": 7.5e-07, "output_cost_per_token": 4.5e-06, "input_cost_per_token_batches": 3.75e-07, "input_cost_per_token_flex": 3.75e-07, "input_cost_per_token_priority": 1.5e-06, "output_cost_per_token_batches": 2.25e-06, "output_cost_per_token_flex": 2.25e-06, "output_cost_per_token_priority": 9e-06}, "inclusive": false, "checks": [[null, 1, [7.5e-07, 4.5e-06, 7.5e-07, 7.5e-07, 7.5e-08]], [null, 10000000, [7.5e-07, 4.5e-06, 7.5e-07, 7.5e-07, 7.5e-08]], ["priority", 1, [1.5e-06, 9e-06, 1.5e-06, 1.5e-06, 1.5e-07]], ["priority", 10000000, [1.5e-06, 9e-06, 1.5e-06, 1.5e-06, 1.5e-07]], ["flex", 1, [3.75e-07, 2.25e-06, 3.75e-07, 3.75e-07, 3.75e-08]], ["flex", 10000000, [3.75e-07, 2.25e-06, 3.75e-07, 3.75e-07, 3.75e-08]]]}, +{"info": {"cache_read_input_token_cost": 5.5e-09, "input_cost_per_token": 5.5e-08, "input_cost_per_token_batches": 2.75e-08, "output_cost_per_token": 4.4e-07, "output_cost_per_token_batches": 2.2e-07, "cache_read_input_token_cost_batches": 2.75e-09}, "inclusive": false, "checks": [[null, 1, [5.5e-08, 4.4e-07, 5.5e-08, 5.5e-08, 5.5e-09]], [null, 10000000, [5.5e-08, 4.4e-07, 5.5e-08, 5.5e-08, 5.5e-09]]]}, +{"info": {"output_cost_per_token": 0.0}, "inclusive": false, "checks": [[null, 1, [0.0, 0.0, 0.0, 0.0, 0.0]], [null, 10000000, [0.0, 0.0, 0.0, 0.0, 0.0]]]}, +{"info": {"input_cost_per_token_above_100k_tokens": 5e-07, "output_cost_per_token_above_100k_tokens": 2.5e-06, "cache_creation_input_token_cost_above_100k_tokens": 6.25e-07, "cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06, "cache_read_input_token_cost_above_100k_tokens": 5e-08, "cache_creation_input_token_cost": 1.25e-07, "cache_creation_input_token_cost_above_1hr": 2e-07, "cache_creation_input_token_cost_batches": 6.25e-08, "cache_read_input_token_cost": 1e-08, "cache_read_input_token_cost_batches": 5e-09, "input_cost_per_token": 1e-07, "input_cost_per_token_batches": 5e-08, "output_cost_per_token": 5e-07, "output_cost_per_token_batches": 2.5e-07}, "inclusive": false, "checks": [[null, 1, [1e-07, 5e-07, 1.25e-07, 2e-07, 1e-08]], [null, 100000, [1e-07, 5e-07, 1.25e-07, 2e-07, 1e-08]], [null, 100001, [5e-07, 2.5e-06, 6.25e-07, 1e-06, 5e-08]], [null, 10000000, [5e-07, 2.5e-06, 6.25e-07, 1e-06, 5e-08]]]}, +{"info": {"input_cost_per_token_above_100k_tokens": 5.5e-07, "output_cost_per_token_above_100k_tokens": 2.75e-06, "cache_creation_input_token_cost_above_100k_tokens": 6.875e-07, "cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1.1e-06, "cache_read_input_token_cost_above_100k_tokens": 5.5e-08, "cache_creation_input_token_cost": 1.375e-07, "cache_creation_input_token_cost_above_1hr": 2.2e-07, "cache_read_input_token_cost": 1.1e-08, "input_cost_per_token": 1.1e-07, "output_cost_per_token": 5.5e-07}, "inclusive": false, "checks": [[null, 1, [1.1e-07, 5.5e-07, 1.375e-07, 2.2e-07, 1.1e-08]], [null, 100000, [1.1e-07, 5.5e-07, 1.375e-07, 2.2e-07, 1.1e-08]], [null, 100001, [5.5e-07, 2.75e-06, 6.875e-07, 1.1e-06, 5.5e-08]], [null, 10000000, [5.5e-07, 2.75e-06, 6.875e-07, 1.1e-06, 5.5e-08]]]}, +{"info": {"cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_flex": 2.0625e-06, "cache_creation_input_token_cost_priority": 7.21875e-06, "cache_read_input_token_cost": 3.3e-07, "cache_read_input_token_cost_flex": 1.65e-07, "cache_read_input_token_cost_priority": 5.775e-07, "input_cost_per_token": 3.3e-06, "input_cost_per_token_flex": 1.65e-06, "input_cost_per_token_priority": 5.775e-06, "output_cost_per_token": 1.65e-05, "output_cost_per_token_flex": 8.25e-06, "output_cost_per_token_priority": 2.8875e-05}, "inclusive": false, "checks": [[null, 1, [3.3e-06, 1.65e-05, 4.125e-06, 4.125e-06, 3.3e-07]], [null, 10000000, [3.3e-06, 1.65e-05, 4.125e-06, 4.125e-06, 3.3e-07]], ["priority", 1, [5.775e-06, 2.8875e-05, 7.21875e-06, 7.21875e-06, 5.775e-07]], ["priority", 10000000, [5.775e-06, 2.8875e-05, 7.21875e-06, 7.21875e-06, 5.775e-07]], ["flex", 1, [1.65e-06, 8.25e-06, 2.0625e-06, 2.0625e-06, 1.65e-07]], ["flex", 10000000, [1.65e-06, 8.25e-06, 2.0625e-06, 2.0625e-06, 1.65e-07]]]}, +{"info": {"cache_read_input_token_cost": 1.375e-07, "cache_read_input_token_cost_priority": 2.75e-07, "input_cost_per_token": 1.375e-06, "input_cost_per_token_batches": 6.875e-07, "input_cost_per_token_priority": 2.75e-06, "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, "output_cost_per_token_priority": 2.2e-05, "cache_read_input_token_cost_batches": 6.875e-08}, "inclusive": false, "checks": [[null, 1, [1.375e-06, 1.1e-05, 1.375e-06, 1.375e-06, 1.375e-07]], [null, 10000000, [1.375e-06, 1.1e-05, 1.375e-06, 1.375e-06, 1.375e-07]], ["priority", 1, [2.75e-06, 2.2e-05, 2.75e-06, 2.75e-06, 2.75e-07]], ["priority", 10000000, [2.75e-06, 2.2e-05, 2.75e-06, 2.75e-06, 2.75e-07]]]}, +{"info": {"input_cost_per_token": 5e-06, "input_cost_per_token_batches": 2.5e-06, "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06}, "inclusive": false, "checks": [[null, 1, [5e-06, 1.5e-05, 5e-06, 5e-06, 5e-06]], [null, 10000000, [5e-06, 1.5e-05, 5e-06, 5e-06, 5e-06]]]}, +{"info": {"cache_creation_input_token_cost": 2.1e-06, "cache_creation_input_token_cost_batches": 1.05e-06, "cache_creation_input_token_cost_flex": 1.05e-06, "cache_creation_input_token_cost_priority": 3.675e-06, "cache_read_input_token_cost": 3.12e-07, "cache_read_input_token_cost_batches": 1.56e-07, "cache_read_input_token_cost_flex": 1.56e-07, "cache_read_input_token_cost_priority": 5.46e-07, "input_cost_per_token": 1.68e-06, "input_cost_per_token_batches": 8.4e-07, "input_cost_per_token_flex": 8.4e-07, "input_cost_per_token_priority": 2.94e-06, "output_cost_per_token": 5.28e-06, "output_cost_per_token_batches": 2.64e-06, "output_cost_per_token_flex": 2.64e-06, "output_cost_per_token_priority": 9.24e-06}, "inclusive": false, "checks": [[null, 1, [1.68e-06, 5.28e-06, 2.1e-06, 2.1e-06, 3.12e-07]], [null, 10000000, [1.68e-06, 5.28e-06, 2.1e-06, 2.1e-06, 3.12e-07]], ["priority", 1, [2.94e-06, 9.24e-06, 3.675e-06, 3.675e-06, 5.46e-07]], ["priority", 10000000, [2.94e-06, 9.24e-06, 3.675e-06, 3.675e-06, 5.46e-07]], ["flex", 1, [8.4e-07, 2.64e-06, 1.05e-06, 1.05e-06, 1.56e-07]], ["flex", 10000000, [8.4e-07, 2.64e-06, 1.05e-06, 1.05e-06, 1.56e-07]]]}, +{"info": {"input_cost_per_token": 1.1e-05, "input_cost_per_token_above_272k_tokens": 2.2e-05, "cache_creation_input_token_cost": 1.375e-05, "cache_creation_input_token_cost_above_272k_tokens": 2.75e-05, "cache_creation_input_token_cost_above_272k_tokens_ultrafast": 0.000165, "cache_creation_input_token_cost_ultrafast": 8.25e-05, "cache_read_input_token_cost": 1.1e-06, "cache_read_input_token_cost_above_272k_tokens": 2.2e-06, "cache_read_input_token_cost_above_272k_tokens_ultrafast": 1.32e-05, "cache_read_input_token_cost_ultrafast": 6.6e-06, "input_cost_per_token_above_272k_tokens_ultrafast": 0.000132, "input_cost_per_token_ultrafast": 6.6e-05, "output_cost_per_token": 5.5e-05, "output_cost_per_token_above_272k_tokens": 8.25e-05, "output_cost_per_token_above_272k_tokens_ultrafast": 0.000495, "output_cost_per_token_ultrafast": 0.00033}, "inclusive": false, "checks": [[null, 1, [1.1e-05, 5.5e-05, 1.375e-05, 1.375e-05, 1.1e-06]], [null, 272000, [1.1e-05, 5.5e-05, 1.375e-05, 1.375e-05, 1.1e-06]], [null, 272001, [2.2e-05, 8.25e-05, 2.75e-05, 2.75e-05, 2.2e-06]], [null, 10000000, [2.2e-05, 8.25e-05, 2.75e-05, 2.75e-05, 2.2e-06]], ["ultrafast", 1, [6.6e-05, 0.00033, 8.25e-05, 8.25e-05, 6.6e-06]], ["ultrafast", 272000, [6.6e-05, 0.00033, 8.25e-05, 8.25e-05, 6.6e-06]], ["ultrafast", 272001, [0.000132, 0.000495, 0.000165, 0.000165, 1.32e-05]], ["ultrafast", 10000000, [0.000132, 0.000495, 0.000165, 0.000165, 1.32e-05]]]}, +{"info": {"cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05, "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-05, "cache_creation_input_token_cost_flex": 6.25e-06, "cache_creation_input_token_cost_priority": 2.5e-05, "cache_read_input_token_cost": 1e-06, "cache_read_input_token_cost_above_272k_tokens": 2e-06, "cache_read_input_token_cost_above_272k_tokens_flex": 1e-06, "cache_read_input_token_cost_above_272k_tokens_priority": 4e-06, "cache_read_input_token_cost_batches": 5e-07, "cache_read_input_token_cost_above_272k_tokens_batches": 1e-06, "cache_creation_input_token_cost_batches": 6.25e-06, "cache_creation_input_token_cost_above_272k_tokens_batches": 1.25e-05, "cache_creation_input_token_cost_above_272k_tokens_ultrafast": 0.00015, "cache_creation_input_token_cost_ultrafast": 7.5e-05, "cache_read_input_token_cost_above_272k_tokens_ultrafast": 1.2e-05, "cache_read_input_token_cost_flex": 5e-07, "cache_read_input_token_cost_priority": 2e-06, "cache_read_input_token_cost_ultrafast": 6e-06, "input_cost_per_token": 1e-05, "input_cost_per_token_above_272k_tokens": 2e-05, "input_cost_per_token_above_272k_tokens_flex": 1e-05, "input_cost_per_token_above_272k_tokens_priority": 4e-05, "input_cost_per_token_batches": 5e-06, "input_cost_per_token_above_272k_tokens_batches": 1e-05, "input_cost_per_token_above_272k_tokens_ultrafast": 0.00012, "input_cost_per_token_flex": 5e-06, "input_cost_per_token_priority": 2e-05, "input_cost_per_token_ultrafast": 6e-05, "output_cost_per_token": 5e-05, "output_cost_per_token_above_272k_tokens": 7.5e-05, "output_cost_per_token_above_272k_tokens_flex": 3.75e-05, "output_cost_per_token_above_272k_tokens_priority": 0.00015, "output_cost_per_token_batches": 2.5e-05, "output_cost_per_token_above_272k_tokens_batches": 3.75e-05, "output_cost_per_token_above_272k_tokens_ultrafast": 0.00045, "output_cost_per_token_flex": 2.5e-05, "output_cost_per_token_priority": 0.0001, "output_cost_per_token_ultrafast": 0.0003}, "inclusive": false, "checks": [[null, 1, [1e-05, 5e-05, 1.25e-05, 1.25e-05, 1e-06]], [null, 272000, [1e-05, 5e-05, 1.25e-05, 1.25e-05, 1e-06]], [null, 272001, [2e-05, 7.5e-05, 2.5e-05, 2.5e-05, 2e-06]], [null, 10000000, [2e-05, 7.5e-05, 2.5e-05, 2.5e-05, 2e-06]], ["priority", 1, [2e-05, 0.0001, 2.5e-05, 2.5e-05, 2e-06]], ["priority", 272000, [2e-05, 0.0001, 2.5e-05, 2.5e-05, 2e-06]], ["priority", 272001, [4e-05, 0.00015, 5e-05, 5e-05, 4e-06]], ["priority", 10000000, [4e-05, 0.00015, 5e-05, 5e-05, 4e-06]], ["flex", 1, [5e-06, 2.5e-05, 6.25e-06, 6.25e-06, 5e-07]], ["flex", 272000, [5e-06, 2.5e-05, 6.25e-06, 6.25e-06, 5e-07]], ["flex", 272001, [1e-05, 3.75e-05, 1.25e-05, 1.25e-05, 1e-06]], ["flex", 10000000, [1e-05, 3.75e-05, 1.25e-05, 1.25e-05, 1e-06]], ["ultrafast", 1, [6e-05, 0.0003, 7.5e-05, 7.5e-05, 6e-06]], ["ultrafast", 272000, [6e-05, 0.0003, 7.5e-05, 7.5e-05, 6e-06]], ["ultrafast", 272001, [0.00012, 0.00045, 0.00015, 0.00015, 1.2e-05]], ["ultrafast", 10000000, [0.00012, 0.00045, 0.00015, 0.00015, 1.2e-05]]]}, +{"info": {"input_cost_per_token": 2.2e-06, "output_cost_per_token": 6.6e-06, "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_flex": 2.75e-07, "cache_read_input_token_cost_priority": 9.625e-07, "input_cost_per_token_flex": 1.1e-06, "input_cost_per_token_priority": 3.85e-06, "output_cost_per_token_flex": 3.3e-06, "output_cost_per_token_priority": 1.155e-05}, "inclusive": false, "checks": [[null, 1, [2.2e-06, 6.6e-06, 2.2e-06, 2.2e-06, 5.5e-07]], [null, 10000000, [2.2e-06, 6.6e-06, 2.2e-06, 2.2e-06, 5.5e-07]], ["priority", 1, [3.85e-06, 1.155e-05, 3.85e-06, 3.85e-06, 9.625e-07]], ["priority", 10000000, [3.85e-06, 1.155e-05, 3.85e-06, 3.85e-06, 9.625e-07]], ["flex", 1, [1.1e-06, 3.3e-06, 1.1e-06, 1.1e-06, 2.75e-07]], ["flex", 10000000, [1.1e-06, 3.3e-06, 1.1e-06, 1.1e-06, 2.75e-07]]]}, +{"info": {"cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "input_cost_per_token": 3e-05, "input_cost_per_token_above_272k_tokens": 6e-05, "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027}, "inclusive": false, "checks": [[null, 1, [3e-05, 0.00018, 3e-05, 3e-05, 3e-06]], [null, 272000, [3e-05, 0.00018, 3e-05, 3e-05, 3e-06]], [null, 272001, [6e-05, 0.00027, 6e-05, 6e-05, 6e-06]], [null, 10000000, [6e-05, 0.00027, 6e-05, 6e-05, 6e-06]]]}, +{"info": {"input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 1.2e-07, "output_cost_per_token": 0}, "inclusive": false, "checks": [[null, 1, [1.5e-07, 0.0, 1.5e-07, 1.5e-07, 1.5e-07]], [null, 10000000, [1.5e-07, 0.0, 1.5e-07, 1.5e-07, 1.5e-07]]]}, +{"info": {"cache_read_input_token_cost_priority": 6e-07, "input_cost_per_token_priority": 1.2e-06, "output_cost_per_token_priority": 1.2e-06}, "inclusive": false, "checks": [[null, 1, [0.0, 0.0, 0.0, 0.0, 0.0]], [null, 10000000, [0.0, 0.0, 0.0, 0.0, 0.0]], ["priority", 1, [1.2e-06, 1.2e-06, 1.2e-06, 1.2e-06, 6e-07]], ["priority", 10000000, [1.2e-06, 1.2e-06, 1.2e-06, 1.2e-06, 6e-07]]]}, +{"info": {"cache_read_input_token_cost": 2e-07, "input_cost_per_token": 1.25e-06, "output_cost_per_token": 2.5e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07}, "inclusive": true, "checks": [[null, 1, [1.25e-06, 2.5e-06, 1.25e-06, 1.25e-06, 2e-07]], [null, 200000, [2.5e-06, 5e-06, 2.5e-06, 2.5e-06, 4e-07]], [null, 200001, [2.5e-06, 5e-06, 2.5e-06, 2.5e-06, 4e-07]], [null, 10000000, [2.5e-06, 5e-06, 2.5e-06, 2.5e-06, 4e-07]]]}, +{"info": {"cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_batches": 1.6e-07, "input_cost_per_token": 1.25e-06, "input_cost_per_token_batches": 1e-06, "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, "input_cost_per_token_above_200k_tokens_batches": 2e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "output_cost_per_token_above_200k_tokens_batches": 4e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07}, "inclusive": true, "checks": [[null, 1, [1.25e-06, 2.5e-06, 1.25e-06, 1.25e-06, 2e-07]], [null, 200000, [2.5e-06, 5e-06, 2.5e-06, 2.5e-06, 4e-07]], [null, 200001, [2.5e-06, 5e-06, 2.5e-06, 2.5e-06, 4e-07]], [null, 10000000, [2.5e-06, 5e-06, 2.5e-06, 2.5e-06, 4e-07]]]}, +{"info": {"input_cost_per_token": 5e-06}, "inclusive": false, "checks": [[null, 1, [5e-06, 0.0, 5e-06, 5e-06, 5e-06]], [null, 10000000, [5e-06, 0.0, 5e-06, 5e-06, 5e-06]]]}, +{"info": {"cache_read_input_token_cost": 1.5e-08, "input_cost_per_token": 6e-08, "input_cost_per_token_batches": 3e-08, "output_cost_per_token": 2.4e-07, "output_cost_per_token_batches": 1.2e-07}, "inclusive": false, "checks": [[null, 1, [6e-08, 2.4e-07, 6e-08, 6e-08, 1.5e-08]], [null, 10000000, [6e-08, 2.4e-07, 6e-08, 6e-08, 1.5e-08]]]}, +{"info": {"input_cost_per_token": 3.3e-05, "input_cost_per_token_above_272k_tokens": 6.6e-05, "input_cost_per_token_above_272k_tokens_batches": 3.3e-05, "input_cost_per_token_batches": 1.65e-05, "output_cost_per_token": 0.000198, "output_cost_per_token_above_272k_tokens": 0.000297, "output_cost_per_token_above_272k_tokens_batches": 0.0001485, "output_cost_per_token_batches": 9.9e-05}, "inclusive": false, "checks": [[null, 1, [3.3e-05, 0.000198, 3.3e-05, 3.3e-05, 3.3e-05]], [null, 272000, [3.3e-05, 0.000198, 3.3e-05, 3.3e-05, 3.3e-05]], [null, 272001, [6.6e-05, 0.000297, 6.6e-05, 6.6e-05, 6.6e-05]], [null, 10000000, [6.6e-05, 0.000297, 6.6e-05, 6.6e-05, 6.6e-05]]]}, +{"info": {"cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_priority": 1.3125e-07, "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_priority": 5.25e-07, "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 1.25e-06, "output_cost_per_token_priority": 4.375e-06}, "inclusive": false, "checks": [[null, 1, [3e-07, 2.5e-06, 3e-07, 3e-07, 7.5e-08]], [null, 10000000, [3e-07, 2.5e-06, 3e-07, 3e-07, 7.5e-08]], ["priority", 1, [5.25e-07, 4.375e-06, 5.25e-07, 5.25e-07, 1.3125e-07]], ["priority", 10000000, [5.25e-07, 4.375e-06, 5.25e-07, 5.25e-07, 1.3125e-07]]]}, +{"info": {"input_cost_per_token": 1.25e-06, "output_cost_per_token": 2.5e-06, "cache_read_input_token_cost": 2e-07, "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06}, "inclusive": false, "checks": [[null, 1, [1.25e-06, 2.5e-06, 1.25e-06, 1.25e-06, 2e-07]], [null, 200000, [1.25e-06, 2.5e-06, 1.25e-06, 1.25e-06, 2e-07]], [null, 200001, [2.5e-06, 5e-06, 2.5e-06, 2.5e-06, 2e-07]], [null, 10000000, [2.5e-06, 5e-06, 2.5e-06, 2.5e-06, 2e-07]]]}, +{"info": {"input_cost_per_token": 5e-06, "output_cost_per_token": 3e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token_above_272k_tokens": 1e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05}, "inclusive": false, "checks": [[null, 1, [5e-06, 3e-05, 5e-06, 5e-06, 5e-07]], [null, 272000, [5e-06, 3e-05, 5e-06, 5e-06, 5e-07]], [null, 272001, [1e-05, 4.5e-05, 1e-05, 1e-05, 5e-07]], [null, 10000000, [1e-05, 4.5e-05, 1e-05, 1e-05, 5e-07]]]}, +{"info": {"input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 7.5e-08, "input_cost_per_token_flex": 7.5e-08, "input_cost_per_token_priority": 2.625e-07, "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, "output_cost_per_token_flex": 3e-07, "output_cost_per_token_priority": 1.05e-06}, "inclusive": false, "checks": [[null, 1, [1.5e-07, 6e-07, 1.5e-07, 1.5e-07, 1.5e-07]], [null, 10000000, [1.5e-07, 6e-07, 1.5e-07, 1.5e-07, 1.5e-07]], ["priority", 1, [2.625e-07, 1.05e-06, 2.625e-07, 2.625e-07, 2.625e-07]], ["priority", 10000000, [2.625e-07, 1.05e-06, 2.625e-07, 2.625e-07, 2.625e-07]], ["flex", 1, [7.5e-08, 3e-07, 7.5e-08, 7.5e-08, 7.5e-08]], ["flex", 10000000, [7.5e-08, 3e-07, 7.5e-08, 7.5e-08, 7.5e-08]]]}, +{"info": {"tiered_pricing": [{"input_cost_per_token": 1.2e-06, "output_cost_per_token": 6e-06, "range": [0, 32000.0]}, {"input_cost_per_token": 2.4e-06, "output_cost_per_token": 1.2e-05, "range": [32000.0, 128000.0]}, {"input_cost_per_token": 3e-06, "output_cost_per_token": 1.5e-05, "range": [128000.0, 252000.0]}]}, "inclusive": false, "checks": [[null, 1, [1.2e-06, 6e-06, 1.2e-06, 1.2e-06, 1.2e-06]], [null, 32000, [1.2e-06, 6e-06, 1.2e-06, 1.2e-06, 1.2e-06]], [null, 32001, [2.4e-06, 1.2e-05, 2.4e-06, 2.4e-06, 2.4e-06]], [null, 128000, [2.4e-06, 1.2e-05, 2.4e-06, 2.4e-06, 2.4e-06]], [null, 128001, [3e-06, 1.5e-05, 3e-06, 3e-06, 3e-06]], [null, 10000000, [3e-06, 1.5e-05, 3e-06, 3e-06, 3e-06]]]}, +{"info": {"cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 1e-06, "output_cost_per_token": 5e-06, "input_cost_per_token_batches": 5e-07, "output_cost_per_token_batches": 2.5e-06}, "inclusive": false, "checks": [[null, 1, [1e-06, 5e-06, 1.25e-06, 2e-06, 1e-07]], [null, 10000000, [1e-06, 5e-06, 1.25e-06, 2e-06, 1e-07]]]}, +{"info": {"cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, "cache_read_input_token_cost_priority": 3.6e-07, "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, "input_cost_per_token_batches": 1e-06, "input_cost_per_token_priority": 3.6e-06, "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "output_cost_per_token_batches": 6e-06, "output_cost_per_token_priority": 2.16e-05, "cache_read_input_token_cost_flex": 2e-07, "input_cost_per_token_flex": 1e-06, "output_cost_per_token_flex": 6e-06}, "inclusive": false, "checks": [[null, 1, [2e-06, 1.2e-05, 2e-06, 2e-06, 2e-07]], [null, 200000, [2e-06, 1.2e-05, 2e-06, 2e-06, 2e-07]], [null, 200001, [4e-06, 1.8e-05, 4e-06, 4e-06, 4e-07]], [null, 10000000, [4e-06, 1.8e-05, 4e-06, 4e-06, 4e-07]], ["priority", 1, [3.6e-06, 2.16e-05, 3.6e-06, 3.6e-06, 3.6e-07]], ["priority", 200000, [3.6e-06, 2.16e-05, 3.6e-06, 3.6e-06, 3.6e-07]], ["priority", 200001, [7.2e-06, 3.24e-05, 7.2e-06, 7.2e-06, 7.2e-07]], ["priority", 10000000, [7.2e-06, 3.24e-05, 7.2e-06, 7.2e-06, 7.2e-07]], ["flex", 1, [1e-06, 6e-06, 1e-06, 1e-06, 2e-07]], ["flex", 200000, [1e-06, 6e-06, 1e-06, 1e-06, 2e-07]], ["flex", 200001, [4e-06, 1.8e-05, 4e-06, 4e-06, 4e-07]], ["flex", 10000000, [4e-06, 1.8e-05, 4e-06, 4e-06, 4e-07]]]}, +{"info": {"input_cost_per_token": 5e-06, "input_cost_per_token_batches": 2.5e-06, "input_cost_per_token_priority": 8.75e-06, "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, "output_cost_per_token_priority": 2.625e-05}, "inclusive": false, "checks": [[null, 1, [5e-06, 1.5e-05, 5e-06, 5e-06, 5e-06]], [null, 10000000, [5e-06, 1.5e-05, 5e-06, 5e-06, 5e-06]], ["priority", 1, [8.75e-06, 2.625e-05, 8.75e-06, 8.75e-06, 8.75e-06]], ["priority", 10000000, [8.75e-06, 2.625e-05, 8.75e-06, 8.75e-06, 8.75e-06]]]}, +{"info": {"cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, "cache_read_input_token_cost_batches": 1e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_priority": 3.6e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, "input_cost_per_token_batches": 1e-06, "output_cost_per_token": 1.2e-05, "output_cost_per_token_priority": 2.16e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "output_cost_per_token_batches": 6e-06}, "inclusive": false, "checks": [[null, 1, [2e-06, 1.2e-05, 2e-06, 2e-06, 2e-07]], [null, 200000, [2e-06, 1.2e-05, 2e-06, 2e-06, 2e-07]], [null, 200001, [4e-06, 1.8e-05, 4e-06, 4e-06, 4e-07]], [null, 10000000, [4e-06, 1.8e-05, 4e-06, 4e-06, 4e-07]], ["priority", 1, [3.6e-06, 2.16e-05, 3.6e-06, 3.6e-06, 3.6e-07]], ["priority", 200000, [3.6e-06, 2.16e-05, 3.6e-06, 3.6e-06, 3.6e-07]], ["priority", 200001, [7.2e-06, 3.24e-05, 7.2e-06, 7.2e-06, 7.2e-07]], ["priority", 10000000, [7.2e-06, 3.24e-05, 7.2e-06, 7.2e-06, 7.2e-07]]]}, +{"info": {"cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 3.6e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, "input_cost_per_token_batches": 1e-06, "input_cost_per_token_flex": 1e-06, "input_cost_per_token_priority": 3.6e-06, "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "output_cost_per_token_batches": 6e-06, "output_cost_per_token_flex": 6e-06, "output_cost_per_token_priority": 2.16e-05}, "inclusive": false, "checks": [[null, 1, [2e-06, 1.2e-05, 2e-06, 2e-06, 2e-07]], [null, 200000, [2e-06, 1.2e-05, 2e-06, 2e-06, 2e-07]], [null, 200001, [4e-06, 1.8e-05, 4e-06, 4e-06, 4e-07]], [null, 10000000, [4e-06, 1.8e-05, 4e-06, 4e-06, 4e-07]], ["priority", 1, [3.6e-06, 2.16e-05, 3.6e-06, 3.6e-06, 3.6e-07]], ["priority", 200000, [3.6e-06, 2.16e-05, 3.6e-06, 3.6e-06, 3.6e-07]], ["priority", 200001, [7.2e-06, 3.24e-05, 7.2e-06, 7.2e-06, 7.2e-07]], ["priority", 10000000, [7.2e-06, 3.24e-05, 7.2e-06, 7.2e-06, 7.2e-07]], ["flex", 1, [1e-06, 6e-06, 1e-06, 1e-06, 1e-07]], ["flex", 200000, [1e-06, 6e-06, 1e-06, 1e-06, 1e-07]], ["flex", 200001, [4e-06, 1.8e-05, 4e-06, 4e-06, 4e-07]], ["flex", 10000000, [4e-06, 1.8e-05, 4e-06, 4e-06, 4e-07]]]}, +{"info": {"input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 1.25e-06, "output_cost_per_token_flex": 1.25e-06}, "inclusive": false, "checks": [[null, 1, [3e-07, 2.5e-06, 3e-07, 3e-07, 3e-07]], [null, 10000000, [3e-07, 2.5e-06, 3e-07, 3e-07, 3e-07]], ["flex", 1, [1.5e-07, 1.25e-06, 1.5e-07, 1.5e-07, 1.5e-07]], ["flex", 10000000, [1.5e-07, 1.25e-06, 1.5e-07, 1.5e-07, 1.5e-07]]]}, +{"info": {"cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, "output_cost_per_token_above_200k_tokens": 2.25e-05, "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, "cache_read_input_token_cost_above_200k_tokens": 6e-07, "cache_read_input_token_cost_batches": 1.5e-07, "input_cost_per_token_batches": 1.5e-06, "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, "cache_creation_input_token_cost_batches": 1.88e-06}, "inclusive": false, "checks": [[null, 1, [3e-06, 1.5e-05, 3.75e-06, 6e-06, 3e-07]], [null, 200000, [3e-06, 1.5e-05, 3.75e-06, 6e-06, 3e-07]], [null, 200001, [6e-06, 2.25e-05, 7.5e-06, 1.2e-05, 6e-07]], [null, 10000000, [6e-06, 2.25e-05, 7.5e-06, 1.2e-05, 6e-07]]]}, +{"info": {"input_cost_per_token": 7.4e-07, "input_cost_per_token_flex": 3.7e-07, "output_cost_per_token": 2.22e-06, "output_cost_per_token_flex": 1.11e-06}, "inclusive": false, "checks": [[null, 1, [7.4e-07, 2.22e-06, 7.4e-07, 7.4e-07, 7.4e-07]], [null, 10000000, [7.4e-07, 2.22e-06, 7.4e-07, 7.4e-07, 7.4e-07]], ["flex", 1, [3.7e-07, 1.11e-06, 3.7e-07, 3.7e-07, 3.7e-07]], ["flex", 10000000, [3.7e-07, 1.11e-06, 3.7e-07, 3.7e-07, 3.7e-07]]]}, +{"info": {"cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, "output_cost_per_token_above_200k_tokens": 2.25e-05, "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, "cache_read_input_token_cost_above_200k_tokens": 6e-07, "output_cost_per_token": 1.5e-05}, "inclusive": false, "checks": [[null, 1, [3e-06, 1.5e-05, 3.75e-06, 3.75e-06, 3e-07]], [null, 200000, [3e-06, 1.5e-05, 3.75e-06, 3.75e-06, 3e-07]], [null, 200001, [6e-06, 2.25e-05, 7.5e-06, 7.5e-06, 6e-07]], [null, 10000000, [6e-06, 2.25e-05, 7.5e-06, 7.5e-06, 6e-07]]]}, +{"info": {"cache_creation_input_token_cost": 4.5e-06, "cache_creation_input_token_cost_above_1hr": 7.2e-06, "cache_read_input_token_cost": 3.6e-07, "input_cost_per_token": 3.6e-06, "input_cost_per_token_above_200k_tokens": 7.2e-06, "output_cost_per_token_above_200k_tokens": 2.7e-05, "cache_creation_input_token_cost_above_200k_tokens": 9e-06, "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.44e-05, "cache_read_input_token_cost_above_200k_tokens": 7.2e-07, "output_cost_per_token": 1.8e-05}, "inclusive": false, "checks": [[null, 1, [3.6e-06, 1.8e-05, 4.5e-06, 7.2e-06, 3.6e-07]], [null, 200000, [3.6e-06, 1.8e-05, 4.5e-06, 7.2e-06, 3.6e-07]], [null, 200001, [7.2e-06, 2.7e-05, 9e-06, 1.44e-05, 7.2e-07]], [null, 10000000, [7.2e-06, 2.7e-05, 9e-06, 1.44e-05, 7.2e-07]]]}, +{"info": {"cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, "output_cost_per_token_above_200k_tokens": 2.25e-05, "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, "cache_read_input_token_cost_above_200k_tokens": 6e-07, "output_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 1.5e-06, "output_cost_per_token_batches": 7.5e-06}, "inclusive": false, "checks": [[null, 1, [3e-06, 1.5e-05, 3.75e-06, 6e-06, 3e-07]], [null, 200000, [3e-06, 1.5e-05, 3.75e-06, 6e-06, 3e-07]], [null, 200001, [6e-06, 2.25e-05, 7.5e-06, 1.2e-05, 6e-07]], [null, 10000000, [6e-06, 2.25e-05, 7.5e-06, 1.2e-05, 6e-07]]]}, +{"info": {"input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token": 1e-05, "output_cost_per_token_above_200k_tokens": 1.5e-05}, "inclusive": false, "checks": [[null, 1, [1.25e-06, 1e-05, 1.25e-06, 1.25e-06, 1.25e-06]], [null, 200000, [1.25e-06, 1e-05, 1.25e-06, 1.25e-06, 1.25e-06]], [null, 200001, [2.5e-06, 1.5e-05, 2.5e-06, 2.5e-06, 2.5e-06]], [null, 10000000, [2.5e-06, 1.5e-05, 2.5e-06, 2.5e-06, 2.5e-06]]]}, +{"info": {"cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 5e-06}, "inclusive": false, "checks": [[null, 1, [5e-06, 0.0, 5e-06, 5e-06, 1.25e-06]], [null, 10000000, [5e-06, 0.0, 5e-06, 5e-06, 1.25e-06]]]}, +{"info": {"input_cost_per_token": 3e-07, "input_cost_per_token_above_512k_tokens": 6e-07, "output_cost_per_token": 1.2e-06, "output_cost_per_token_above_512k_tokens": 2.4e-06, "cache_read_input_token_cost": 6e-08, "cache_read_input_token_cost_above_512k_tokens": 1.2e-07}, "inclusive": false, "checks": [[null, 1, [3e-07, 1.2e-06, 3e-07, 3e-07, 6e-08]], [null, 512000, [3e-07, 1.2e-06, 3e-07, 3e-07, 6e-08]], [null, 512001, [6e-07, 2.4e-06, 6e-07, 6e-07, 1.2e-07]], [null, 10000000, [6e-07, 2.4e-06, 6e-07, 6e-07, 1.2e-07]]]}, +{"info": {"cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_batches": 6.25e-07, "input_cost_per_token": 5e-06, "input_cost_per_token_batches": 2.5e-06}, "inclusive": false, "checks": [[null, 1, [5e-06, 0.0, 5e-06, 5e-06, 1.25e-06]], [null, 10000000, [5e-06, 0.0, 5e-06, 5e-06, 1.25e-06]]]}, +{"info": {"cache_read_input_token_cost": 5e-09, "cache_read_input_token_cost_batches": 2.5e-09, "cache_read_input_token_cost_flex": 2.5e-09, "input_cost_per_token": 5e-08, "input_cost_per_token_batches": 2.5e-08, "input_cost_per_token_flex": 2.5e-08, "input_cost_per_token_priority": 2.5e-06, "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, "output_cost_per_token_flex": 2e-07}, "inclusive": false, "checks": [[null, 1, [5e-08, 4e-07, 5e-08, 5e-08, 5e-09]], [null, 10000000, [5e-08, 4e-07, 5e-08, 5e-08, 5e-09]], ["priority", 1, [2.5e-06, 4e-07, 2.5e-06, 2.5e-06, 5e-09]], ["priority", 10000000, [2.5e-06, 4e-07, 2.5e-06, 2.5e-06, 5e-09]], ["flex", 1, [2.5e-08, 2e-07, 2.5e-08, 2.5e-08, 2.5e-09]], ["flex", 10000000, [2.5e-08, 2e-07, 2.5e-08, 2.5e-08, 2.5e-09]]]}, +{"info": {"cache_read_input_token_cost": 2e-08, "cache_read_input_token_cost_batches": 1e-08, "cache_read_input_token_cost_flex": 1e-08, "input_cost_per_token": 2e-07, "output_cost_per_token": 1.25e-06, "input_cost_per_token_batches": 1e-07, "input_cost_per_token_flex": 1e-07, "output_cost_per_token_batches": 6.25e-07, "output_cost_per_token_flex": 6.25e-07}, "inclusive": false, "checks": [[null, 1, [2e-07, 1.25e-06, 2e-07, 2e-07, 2e-08]], [null, 10000000, [2e-07, 1.25e-06, 2e-07, 2e-07, 2e-08]], ["flex", 1, [1e-07, 6.25e-07, 1e-07, 1e-07, 1e-08]], ["flex", 10000000, [1e-07, 6.25e-07, 1e-07, 1e-07, 1e-08]]]}, +{"info": {"input_cost_per_token": 3e-05, "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, "input_cost_per_token_above_272k_tokens_batches": 3e-05, "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, "output_cost_per_token_above_272k_tokens_batches": 0.000135, "input_cost_per_token_above_272k_tokens_flex": 3e-05, "output_cost_per_token_above_272k_tokens_flex": 0.000135}, "inclusive": false, "checks": [[null, 1, [3e-05, 0.00018, 3e-05, 3e-05, 3e-05]], [null, 272000, [3e-05, 0.00018, 3e-05, 3e-05, 3e-05]], [null, 272001, [6e-05, 0.00027, 6e-05, 6e-05, 6e-05]], [null, 10000000, [6e-05, 0.00027, 6e-05, 6e-05, 6e-05]], ["flex", 1, [1.5e-05, 9e-05, 1.5e-05, 1.5e-05, 1.5e-05]], ["flex", 272000, [1.5e-05, 9e-05, 1.5e-05, 1.5e-05, 1.5e-05]], ["flex", 272001, [3e-05, 0.000135, 3e-05, 3e-05, 3e-05]], ["flex", 10000000, [3e-05, 0.000135, 3e-05, 3e-05, 3e-05]]]}, +{"info": {"cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_batches": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens_batches": 5e-07, "cache_read_input_token_cost_flex": 2.5e-07, "cache_read_input_token_cost_priority": 1.25e-06, "input_cost_per_token": 5e-06, "input_cost_per_token_above_272k_tokens": 1e-05, "input_cost_per_token_flex": 2.5e-06, "input_cost_per_token_batches": 2.5e-06, "input_cost_per_token_above_272k_tokens_batches": 5e-06, "input_cost_per_token_priority": 1.25e-05, "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, "output_cost_per_token_above_272k_tokens_batches": 2.25e-05, "output_cost_per_token_priority": 7.5e-05, "input_cost_per_token_above_272k_tokens_flex": 5e-06, "output_cost_per_token_above_272k_tokens_flex": 2.25e-05, "cache_read_input_token_cost_above_272k_tokens_flex": 5e-07}, "inclusive": false, "checks": [[null, 1, [5e-06, 3e-05, 5e-06, 5e-06, 5e-07]], [null, 272000, [5e-06, 3e-05, 5e-06, 5e-06, 5e-07]], [null, 272001, [1e-05, 4.5e-05, 1e-05, 1e-05, 1e-06]], [null, 10000000, [1e-05, 4.5e-05, 1e-05, 1e-05, 1e-06]], ["priority", 1, [1.25e-05, 7.5e-05, 1.25e-05, 1.25e-05, 1.25e-06]], ["priority", 272000, [1.25e-05, 7.5e-05, 1.25e-05, 1.25e-05, 1.25e-06]], ["priority", 272001, [1e-05, 4.5e-05, 1e-05, 1e-05, 1e-06]], ["priority", 10000000, [1e-05, 4.5e-05, 1e-05, 1e-05, 1e-06]], ["flex", 1, [2.5e-06, 1.5e-05, 2.5e-06, 2.5e-06, 2.5e-07]], ["flex", 272000, [2.5e-06, 1.5e-05, 2.5e-06, 2.5e-06, 2.5e-07]], ["flex", 272001, [5e-06, 2.25e-05, 5e-06, 5e-06, 5e-07]], ["flex", 10000000, [5e-06, 2.25e-05, 5e-06, 5e-06, 5e-07]]]}, +{"info": {"input_cost_per_token": 3e-05, "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, "input_cost_per_token_above_272k_tokens_batches": 3e-05, "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, "output_cost_per_token_above_272k_tokens_batches": 0.000135}, "inclusive": false, "checks": [[null, 1, [3e-05, 0.00018, 3e-05, 3e-05, 3e-05]], [null, 272000, [3e-05, 0.00018, 3e-05, 3e-05, 3e-05]], [null, 272001, [6e-05, 0.00027, 6e-05, 6e-05, 6e-05]], [null, 10000000, [6e-05, 0.00027, 6e-05, 6e-05, 6e-05]], ["flex", 1, [1.5e-05, 9e-05, 1.5e-05, 1.5e-05, 1.5e-05]], ["flex", 272000, [1.5e-05, 9e-05, 1.5e-05, 1.5e-05, 1.5e-05]], ["flex", 272001, [6e-05, 0.00027, 6e-05, 6e-05, 6e-05]], ["flex", 10000000, [6e-05, 0.00027, 6e-05, 6e-05, 6e-05]]]}, +{"info": {"cache_creation_input_token_cost": 1.25e-07, "cache_creation_input_token_cost_decisions": 1.25e-07, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, "cache_creation_input_token_cost_above_272k_tokens_decisions": 2.5e-07, "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-07, "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-07, "cache_creation_input_token_cost_flex": 6.25e-08, "cache_creation_input_token_cost_priority": 2.5e-07, "cache_read_input_token_cost": 1e-08, "cache_read_input_token_cost_decisions": 1e-08, "cache_read_input_token_cost_above_272k_tokens": 2e-08, "cache_read_input_token_cost_above_272k_tokens_decisions": 2e-08, "cache_read_input_token_cost_above_272k_tokens_flex": 1e-08, "cache_read_input_token_cost_above_272k_tokens_priority": 4e-08, "cache_read_input_token_cost_batches": 5e-09, "cache_read_input_token_cost_above_272k_tokens_batches": 1e-08, "cache_creation_input_token_cost_batches": 6.25e-08, "cache_creation_input_token_cost_above_272k_tokens_batches": 1.25e-07, "cache_read_input_token_cost_flex": 5e-09, "cache_read_input_token_cost_priority": 2e-08, "input_cost_per_token": 1e-07, "input_cost_per_token_decisions": 1e-07, "input_cost_per_token_above_272k_tokens": 2e-07, "input_cost_per_token_above_272k_tokens_decisions": 2e-07, "input_cost_per_token_above_272k_tokens_flex": 1e-07, "input_cost_per_token_above_272k_tokens_priority": 4e-07, "input_cost_per_token_batches": 5e-08, "input_cost_per_token_above_272k_tokens_batches": 1e-07, "input_cost_per_token_flex": 5e-08, "input_cost_per_token_priority": 2e-07, "output_cost_per_token": 5e-07, "output_cost_per_token_decisions": 0.0, "output_cost_per_token_above_272k_tokens": 7.5e-07, "output_cost_per_token_above_272k_tokens_decisions": 0.0, "output_cost_per_token_above_272k_tokens_flex": 3.75e-07, "output_cost_per_token_above_272k_tokens_priority": 1.5e-06, "output_cost_per_token_batches": 2.5e-07, "output_cost_per_token_above_272k_tokens_batches": 3.75e-07, "output_cost_per_token_flex": 2.5e-07, "output_cost_per_token_priority": 1e-06}, "inclusive": false, "checks": [[null, 1, [1e-07, 5e-07, 1.25e-07, 1.25e-07, 1e-08]], [null, 272000, [1e-07, 5e-07, 1.25e-07, 1.25e-07, 1e-08]], [null, 272001, [2e-07, 7.5e-07, 2.5e-07, 2.5e-07, 2e-08]], [null, 10000000, [2e-07, 7.5e-07, 2.5e-07, 2.5e-07, 2e-08]], ["priority", 1, [2e-07, 1e-06, 2.5e-07, 2.5e-07, 2e-08]], ["priority", 272000, [2e-07, 1e-06, 2.5e-07, 2.5e-07, 2e-08]], ["priority", 272001, [4e-07, 1.5e-06, 5e-07, 5e-07, 4e-08]], ["priority", 10000000, [4e-07, 1.5e-06, 5e-07, 5e-07, 4e-08]], ["flex", 1, [5e-08, 2.5e-07, 6.25e-08, 6.25e-08, 5e-09]], ["flex", 272000, [5e-08, 2.5e-07, 6.25e-08, 6.25e-08, 5e-09]], ["flex", 272001, [1e-07, 3.75e-07, 1.25e-07, 1.25e-07, 1e-08]], ["flex", 10000000, [1e-07, 3.75e-07, 1.25e-07, 1.25e-07, 1e-08]]]}, +{"info": {"cache_creation_input_token_cost": 1.25e-07, "cache_creation_input_token_cost_above_100k_tokens": 6.25e-07, "cache_read_input_token_cost": 1e-08, "cache_read_input_token_cost_above_100k_tokens": 5e-08, "input_cost_per_token": 1e-07, "input_cost_per_token_above_100k_tokens": 5e-07, "output_cost_per_token": 5e-07, "output_cost_per_token_above_100k_tokens": 2.5e-06}, "inclusive": false, "checks": [[null, 1, [1e-07, 5e-07, 1.25e-07, 1.25e-07, 1e-08]], [null, 100000, [1e-07, 5e-07, 1.25e-07, 1.25e-07, 1e-08]], [null, 100001, [5e-07, 2.5e-06, 6.25e-07, 6.25e-07, 5e-08]], [null, 10000000, [5e-07, 2.5e-06, 6.25e-07, 6.25e-07, 5e-08]]]}, +{"info": {"input_cost_per_token": 0, "input_cost_per_token_above_128k_tokens": 0, "output_cost_per_token": 0, "output_cost_per_token_above_128k_tokens": 0}, "inclusive": false, "checks": [[null, 1, [0.0, 0.0, 0.0, 0.0, 0.0]], [null, 128000, [0.0, 0.0, 0.0, 0.0, 0.0]], [null, 128001, [0.0, 0.0, 0.0, 0.0, 0.0]], [null, 10000000, [0.0, 0.0, 0.0, 0.0, 0.0]]]}, +{"info": {"cache_read_input_token_cost": 5e-08, "cache_read_input_token_cost_priority": 9e-08, "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, "input_cost_per_token_flex": 2.5e-07, "input_cost_per_token_priority": 9e-07, "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, "output_cost_per_token_flex": 1.5e-06, "output_cost_per_token_priority": 5.4e-06}, "inclusive": false, "checks": [[null, 1, [5e-07, 3e-06, 5e-07, 5e-07, 5e-08]], [null, 10000000, [5e-07, 3e-06, 5e-07, 5e-07, 5e-08]], ["priority", 1, [9e-07, 5.4e-06, 9e-07, 9e-07, 9e-08]], ["priority", 10000000, [9e-07, 5.4e-06, 9e-07, 9e-07, 9e-08]], ["flex", 1, [2.5e-07, 1.5e-06, 2.5e-07, 2.5e-07, 5e-08]], ["flex", 10000000, [2.5e-07, 1.5e-06, 2.5e-07, 2.5e-07, 5e-08]]]}, +{"info": {"cache_read_input_token_cost": 3e-08, "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, "input_cost_per_token_priority": 5.4e-07, "output_cost_per_token": 2.5e-06}, "inclusive": false, "checks": [[null, 1, [3e-07, 2.5e-06, 3e-07, 3e-07, 3e-08]], [null, 10000000, [3e-07, 2.5e-06, 3e-07, 3e-07, 3e-08]], ["priority", 1, [5.4e-07, 2.5e-06, 5.4e-07, 5.4e-07, 3e-08]], ["priority", 10000000, [5.4e-07, 2.5e-06, 5.4e-07, 5.4e-07, 3e-08]], ["flex", 1, [1.5e-07, 2.5e-06, 1.5e-07, 1.5e-07, 3e-08]], ["flex", 10000000, [1.5e-07, 2.5e-06, 1.5e-07, 1.5e-07, 3e-08]]]}, +{"info": {"cache_read_input_token_cost": 8.3e-08, "input_cost_per_token": 1.65e-07, "input_cost_per_token_batches": 8.3e-08, "output_cost_per_token": 6.6e-07}, "inclusive": false, "checks": [[null, 1, [1.65e-07, 6.6e-07, 1.65e-07, 1.65e-07, 8.3e-08]], [null, 10000000, [1.65e-07, 6.6e-07, 1.65e-07, 1.65e-07, 8.3e-08]]]}, +{"info": {"cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "input_cost_per_token_batches": 1e-06, "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, "output_cost_per_token_batches": 6e-06, "input_cost_per_token_priority": 3.6e-06, "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, "output_cost_per_token_priority": 2.16e-05, "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07}, "inclusive": false, "checks": [[null, 1, [2e-06, 1.2e-05, 2e-06, 2e-06, 2e-07]], [null, 200000, [2e-06, 1.2e-05, 2e-06, 2e-06, 2e-07]], [null, 200001, [4e-06, 1.8e-05, 2.5e-07, 2.5e-07, 4e-07]], [null, 10000000, [4e-06, 1.8e-05, 2.5e-07, 2.5e-07, 4e-07]], ["priority", 1, [3.6e-06, 2.16e-05, 3.6e-06, 3.6e-06, 3.6e-07]], ["priority", 200000, [3.6e-06, 2.16e-05, 3.6e-06, 3.6e-06, 3.6e-07]], ["priority", 200001, [7.2e-06, 3.24e-05, 2.5e-07, 2.5e-07, 7.2e-07]], ["priority", 10000000, [7.2e-06, 3.24e-05, 2.5e-07, 2.5e-07, 7.2e-07]]]}, +{"info": {"cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, "cache_read_input_token_cost_priority": 3.6e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, "input_cost_per_token_batches": 1e-06, "input_cost_per_token_flex": 1e-06, "input_cost_per_token_priority": 3.6e-06, "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "output_cost_per_token_batches": 6e-06, "output_cost_per_token_flex": 6e-06, "output_cost_per_token_priority": 2.16e-05}, "inclusive": false, "checks": [[null, 1, [2e-06, 1.2e-05, 2e-06, 2e-06, 2e-07]], [null, 200000, [2e-06, 1.2e-05, 2e-06, 2e-06, 2e-07]], [null, 200001, [4e-06, 1.8e-05, 2.5e-07, 2.5e-07, 4e-07]], [null, 10000000, [4e-06, 1.8e-05, 2.5e-07, 2.5e-07, 4e-07]], ["priority", 1, [3.6e-06, 2.16e-05, 3.6e-06, 3.6e-06, 3.6e-07]], ["priority", 200000, [3.6e-06, 2.16e-05, 3.6e-06, 3.6e-06, 3.6e-07]], ["priority", 200001, [7.2e-06, 3.24e-05, 2.5e-07, 2.5e-07, 7.2e-07]], ["priority", 10000000, [7.2e-06, 3.24e-05, 2.5e-07, 2.5e-07, 7.2e-07]], ["flex", 1, [1e-06, 6e-06, 1e-06, 1e-06, 2e-07]], ["flex", 200000, [1e-06, 6e-06, 1e-06, 1e-06, 2e-07]], ["flex", 200001, [4e-06, 1.8e-05, 2.5e-07, 2.5e-07, 4e-07]], ["flex", 10000000, [4e-06, 1.8e-05, 2.5e-07, 2.5e-07, 4e-07]]]}, +{"info": {"cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "input_cost_per_token_batches": 1e-06, "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, "output_cost_per_token_batches": 6e-06}, "inclusive": false, "checks": [[null, 1, [2e-06, 1.2e-05, 2e-06, 2e-06, 2e-07]], [null, 200000, [2e-06, 1.2e-05, 2e-06, 2e-06, 2e-07]], [null, 200001, [4e-06, 1.8e-05, 2.5e-07, 2.5e-07, 4e-07]], [null, 10000000, [4e-06, 1.8e-05, 2.5e-07, 2.5e-07, 4e-07]]]}, +{"info": {"cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 4.5e-07, "cache_read_input_token_cost_flex": 1.25e-07, "cache_read_input_token_cost_priority": 2.25e-07, "input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token": 1e-05, "output_cost_per_token_above_200k_tokens": 1.5e-05, "input_cost_per_token_above_200k_tokens_priority": 4.5e-06, "input_cost_per_token_batches": 6.25e-07, "input_cost_per_token_flex": 6.25e-07, "input_cost_per_token_priority": 2.25e-06, "output_cost_per_token_above_200k_tokens_priority": 2.7e-05, "output_cost_per_token_batches": 5e-06, "output_cost_per_token_flex": 5e-06, "output_cost_per_token_priority": 1.8e-05}, "inclusive": false, "checks": [[null, 1, [1.25e-06, 1e-05, 1.25e-06, 1.25e-06, 1.25e-07]], [null, 200000, [1.25e-06, 1e-05, 1.25e-06, 1.25e-06, 1.25e-07]], [null, 200001, [2.5e-06, 1.5e-05, 2.5e-07, 2.5e-07, 2.5e-07]], [null, 10000000, [2.5e-06, 1.5e-05, 2.5e-07, 2.5e-07, 2.5e-07]], ["priority", 1, [2.25e-06, 1.8e-05, 2.25e-06, 2.25e-06, 2.25e-07]], ["priority", 200000, [2.25e-06, 1.8e-05, 2.25e-06, 2.25e-06, 2.25e-07]], ["priority", 200001, [4.5e-06, 2.7e-05, 2.5e-07, 2.5e-07, 4.5e-07]], ["priority", 10000000, [4.5e-06, 2.7e-05, 2.5e-07, 2.5e-07, 4.5e-07]], ["flex", 1, [6.25e-07, 5e-06, 6.25e-07, 6.25e-07, 1.25e-07]], ["flex", 200000, [6.25e-07, 5e-06, 6.25e-07, 6.25e-07, 1.25e-07]], ["flex", 200001, [2.5e-06, 1.5e-05, 2.5e-07, 2.5e-07, 2.5e-07]], ["flex", 10000000, [2.5e-06, 1.5e-05, 2.5e-07, 2.5e-07, 2.5e-07]]]}, +{"info": {"cache_creation_input_token_cost": 1.875e-06, "input_cost_per_token": 3.75e-06, "output_cost_per_token": 1.5e-05}, "inclusive": false, "checks": [[null, 1, [3.75e-06, 1.5e-05, 1.875e-06, 1.875e-06, 3.75e-06]], [null, 10000000, [3.75e-06, 1.5e-05, 1.875e-06, 1.875e-06, 3.75e-06]]]}, +{"info": {"cache_read_input_token_cost": 2.1e-07, "cache_read_input_token_cost_flex": 1.05e-07, "input_cost_per_token": 8.4e-07, "input_cost_per_token_flex": 4.2e-07, "output_cost_per_token": 3.36e-06, "output_cost_per_token_flex": 1.68e-06}, "inclusive": false, "checks": [[null, 1, [8.4e-07, 3.36e-06, 8.4e-07, 8.4e-07, 2.1e-07]], [null, 10000000, [8.4e-07, 3.36e-06, 8.4e-07, 8.4e-07, 2.1e-07]], ["flex", 1, [4.2e-07, 1.68e-06, 4.2e-07, 4.2e-07, 1.05e-07]], ["flex", 10000000, [4.2e-07, 1.68e-06, 4.2e-07, 4.2e-07, 1.05e-07]]]}, +{"info": {"input_cost_per_token": 2.5e-06, "output_cost_per_token": 1.25e-05, "cache_read_input_token_cost": 2.5e-07, "input_cost_per_token_balanced": 2e-06, "output_cost_per_token_balanced": 1e-05, "cache_read_input_token_cost_balanced": 2e-07, "input_cost_per_token_flex": 1.25e-06, "output_cost_per_token_flex": 6.25e-06, "cache_read_input_token_cost_flex": 1.5e-07}, "inclusive": false, "checks": [[null, 1, [2.5e-06, 1.25e-05, 2.5e-06, 2.5e-06, 2.5e-07]], [null, 10000000, [2.5e-06, 1.25e-05, 2.5e-06, 2.5e-06, 2.5e-07]], ["flex", 1, [1.25e-06, 6.25e-06, 1.25e-06, 1.25e-06, 1.5e-07]], ["flex", 10000000, [1.25e-06, 6.25e-06, 1.25e-06, 1.25e-06, 1.5e-07]], ["balanced", 1, [2e-06, 1e-05, 2e-06, 2e-06, 2e-07]], ["balanced", 10000000, [2e-06, 1e-05, 2e-06, 2e-06, 2e-07]]]}, +{"info": {"tiered_pricing": [{"input_cost_per_query": 0.00166, "max_results_range": [1, 10]}, {"input_cost_per_query": 0.00332, "max_results_range": [11, 20]}, {"input_cost_per_query": 0.00498, "max_results_range": [21, 30]}, {"input_cost_per_query": 0.00664, "max_results_range": [31, 40]}, {"input_cost_per_query": 0.0083, "max_results_range": [41, 50]}, {"input_cost_per_query": 0.00996, "max_results_range": [51, 60]}, {"input_cost_per_query": 0.01162, "max_results_range": [61, 70]}, {"input_cost_per_query": 0.01328, "max_results_range": [71, 80]}, {"input_cost_per_query": 0.01494, "max_results_range": [81, 90]}, {"input_cost_per_query": 0.0166, "max_results_range": [91, 100]}]}, "inclusive": false, "checks": [[null, 1, [0.0, 0.0, 0.0, 0.0, 0.0]], [null, 10000000, [0.0, 0.0, 0.0, 0.0, 0.0]]]}, +{"info": {"tiered_pricing": [{"input_cost_per_query": 0.005, "max_results_range": [0, 25]}, {"input_cost_per_query": 0.025, "max_results_range": [26, 100]}]}, "inclusive": false, "checks": [[null, 1, [0.0, 0.0, 0.0, 0.0, 0.0]], [null, 10000000, [0.0, 0.0, 0.0, 0.0, 0.0]]]}, +{"info": {"tiered_pricing": [{"cache_read_input_token_cost": 8e-08, "input_cost_per_token": 4e-07, "output_cost_per_token": 1.6e-06, "range": [0, 256000.0]}, {"cache_read_input_token_cost": 2.4e-07, "input_cost_per_token": 1.2e-06, "output_cost_per_token": 4.8e-06, "range": [256000.0, 1000000.0]}]}, "inclusive": false, "checks": [[null, 1, [4e-07, 1.6e-06, 4e-07, 4e-07, 8e-08]], [null, 256000, [4e-07, 1.6e-06, 4e-07, 4e-07, 8e-08]], [null, 256001, [1.2e-06, 4.8e-06, 1.2e-06, 1.2e-06, 2.4e-07]], [null, 10000000, [1.2e-06, 4.8e-06, 1.2e-06, 1.2e-06, 2.4e-07]]]}, +{"info": {"tiered_pricing": [{"input_cost_per_token": 5e-08, "output_cost_per_token": 4e-07, "range": [0, 256000.0]}, {"input_cost_per_token": 2.5e-07, "output_cost_per_token": 2e-06, "range": [256000.0, 1000000.0]}]}, "inclusive": false, "checks": [[null, 1, [5e-08, 4e-07, 5e-08, 5e-08, 5e-08]], [null, 256000, [5e-08, 4e-07, 5e-08, 5e-08, 5e-08]], [null, 256001, [2.5e-07, 2e-06, 2.5e-07, 2.5e-07, 2.5e-07]], [null, 10000000, [2.5e-07, 2e-06, 2.5e-07, 2.5e-07, 2.5e-07]]]}, +{"info": {"tiered_pricing": [{"input_cost_per_token": 3e-07, "output_cost_per_token": 1.5e-06, "range": [0, 32000.0]}, {"input_cost_per_token": 5e-07, "output_cost_per_token": 2.5e-06, "range": [32000.0, 128000.0]}, {"input_cost_per_token": 8e-07, "output_cost_per_token": 4e-06, "range": [128000.0, 256000.0]}, {"input_cost_per_token": 1.6e-06, "output_cost_per_token": 9.6e-06, "range": [256000.0, 1000000.0]}]}, "inclusive": false, "checks": [[null, 1, [3e-07, 1.5e-06, 3e-07, 3e-07, 3e-07]], [null, 32000, [3e-07, 1.5e-06, 3e-07, 3e-07, 3e-07]], [null, 32001, [5e-07, 2.5e-06, 5e-07, 5e-07, 5e-07]], [null, 128000, [5e-07, 2.5e-06, 5e-07, 5e-07, 5e-07]], [null, 128001, [8e-07, 4e-06, 8e-07, 8e-07, 8e-07]], [null, 256000, [8e-07, 4e-06, 8e-07, 8e-07, 8e-07]], [null, 256001, [1.6e-06, 9.6e-06, 1.6e-06, 1.6e-06, 1.6e-06]], [null, 10000000, [1.6e-06, 9.6e-06, 1.6e-06, 1.6e-06, 1.6e-06]]]}, +{"info": {"tiered_pricing": [{"cache_read_input_token_cost": 8e-08, "input_cost_per_token": 3e-07, "output_cost_per_token": 1.5e-06, "range": [0, 32000.0]}, {"cache_read_input_token_cost": 1.2e-07, "input_cost_per_token": 5e-07, "output_cost_per_token": 2.5e-06, "range": [32000.0, 128000.0]}, {"cache_read_input_token_cost": 2e-07, "input_cost_per_token": 8e-07, "output_cost_per_token": 4e-06, "range": [128000.0, 256000.0]}, {"cache_read_input_token_cost": 4e-07, "input_cost_per_token": 1.6e-06, "output_cost_per_token": 9.6e-06, "range": [256000.0, 1000000.0]}]}, "inclusive": false, "checks": [[null, 1, [3e-07, 1.5e-06, 3e-07, 3e-07, 8e-08]], [null, 32000, [3e-07, 1.5e-06, 3e-07, 3e-07, 8e-08]], [null, 32001, [5e-07, 2.5e-06, 5e-07, 5e-07, 1.2e-07]], [null, 128000, [5e-07, 2.5e-06, 5e-07, 5e-07, 1.2e-07]], [null, 128001, [8e-07, 4e-06, 8e-07, 8e-07, 2e-07]], [null, 256000, [8e-07, 4e-06, 8e-07, 8e-07, 2e-07]], [null, 256001, [1.6e-06, 9.6e-06, 1.6e-06, 1.6e-06, 4e-07]], [null, 10000000, [1.6e-06, 9.6e-06, 1.6e-06, 1.6e-06, 4e-07]]]}, +{"info": {"tiered_pricing": [{"input_cost_per_token": 4e-07, "output_cost_per_reasoning_token": 4e-06, "output_cost_per_token": 1.2e-06, "range": [0, 256000.0]}, {"input_cost_per_token": 1.2e-06, "output_cost_per_reasoning_token": 1.2e-05, "output_cost_per_token": 3.6e-06, "range": [256000.0, 1000000.0]}]}, "inclusive": false, "checks": [[null, 1, [4e-07, 1.2e-06, 4e-07, 4e-07, 4e-07]], [null, 256000, [4e-07, 1.2e-06, 4e-07, 4e-07, 4e-07]], [null, 256001, [1.2e-06, 3.6e-06, 1.2e-06, 1.2e-06, 1.2e-06]], [null, 10000000, [1.2e-06, 3.6e-06, 1.2e-06, 1.2e-06, 1.2e-06]]]}, +{"info": {"cache_creation_input_token_cost": 1.375e-06, "cache_read_input_token_cost": 1.1e-07, "input_cost_per_token": 1.1e-06, "output_cost_per_token": 5.5e-06, "input_cost_per_token_batches": 5.5e-07, "output_cost_per_token_batches": 2.75e-06}, "inclusive": false, "checks": [[null, 1, [1.1e-06, 5.5e-06, 1.375e-06, 1.375e-06, 1.1e-07]], [null, 10000000, [1.1e-06, 5.5e-06, 1.375e-06, 1.375e-06, 1.1e-07]]]}, +{"info": {"cache_read_input_token_cost": 2.75e-07, "cache_read_input_token_cost_above_272k_tokens": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens_batches": 2.75e-07, "cache_read_input_token_cost_batches": 1.43e-07, "cache_read_input_token_cost_priority": 5.5e-07, "input_cost_per_token": 2.75e-06, "input_cost_per_token_priority": 5.5e-06, "output_cost_per_token": 1.65e-05, "output_cost_per_token_priority": 3.3e-05, "input_cost_per_token_above_272k_tokens": 5.5e-06, "input_cost_per_token_above_272k_tokens_batches": 2.75e-06, "input_cost_per_token_batches": 1.375e-06, "output_cost_per_token_above_272k_tokens": 2.475e-05, "output_cost_per_token_above_272k_tokens_batches": 1.2375e-05, "output_cost_per_token_batches": 8.25e-06}, "inclusive": false, "checks": [[null, 1, [2.75e-06, 1.65e-05, 2.75e-06, 2.75e-06, 2.75e-07]], [null, 272000, [2.75e-06, 1.65e-05, 2.75e-06, 2.75e-06, 2.75e-07]], [null, 272001, [5.5e-06, 2.475e-05, 5.5e-06, 5.5e-06, 5.5e-07]], [null, 10000000, [5.5e-06, 2.475e-05, 5.5e-06, 5.5e-06, 5.5e-07]], ["priority", 1, [5.5e-06, 3.3e-05, 5.5e-06, 5.5e-06, 5.5e-07]], ["priority", 272000, [5.5e-06, 3.3e-05, 5.5e-06, 5.5e-06, 5.5e-07]], ["priority", 272001, [5.5e-06, 2.475e-05, 5.5e-06, 5.5e-06, 5.5e-07]], ["priority", 10000000, [5.5e-06, 2.475e-05, 5.5e-06, 5.5e-06, 5.5e-07]]]}, +{"info": {"cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_above_272k_tokens_batches": 5e-07, "cache_read_input_token_cost_priority": 1.25e-06, "cache_read_input_token_cost_above_272k_tokens_priority": 2e-06, "cache_read_input_token_cost_batches": 2.5e-07, "cache_read_input_token_cost_flex": 2.5e-07, "input_cost_per_token": 5e-06, "input_cost_per_token_above_272k_tokens": 1e-05, "input_cost_per_token_above_272k_tokens_batches": 5e-06, "input_cost_per_token_above_272k_tokens_flex": 5e-06, "input_cost_per_token_priority": 1.25e-05, "input_cost_per_token_above_272k_tokens_priority": 2e-05, "input_cost_per_token_batches": 2.5e-06, "input_cost_per_token_flex": 2.5e-06, "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, "output_cost_per_token_above_272k_tokens_batches": 2.25e-05, "output_cost_per_token_priority": 7.5e-05, "output_cost_per_token_above_272k_tokens_priority": 9e-05, "output_cost_per_token_batches": 1.5e-05}, "inclusive": false, "checks": [[null, 1, [5e-06, 3e-05, 5e-06, 5e-06, 5e-07]], [null, 272000, [5e-06, 3e-05, 5e-06, 5e-06, 5e-07]], [null, 272001, [1e-05, 4.5e-05, 1e-05, 1e-05, 1e-06]], [null, 10000000, [1e-05, 4.5e-05, 1e-05, 1e-05, 1e-06]], ["priority", 1, [1.25e-05, 7.5e-05, 1.25e-05, 1.25e-05, 1.25e-06]], ["priority", 272000, [1.25e-05, 7.5e-05, 1.25e-05, 1.25e-05, 1.25e-06]], ["priority", 272001, [2e-05, 9e-05, 2e-05, 2e-05, 2e-06]], ["priority", 10000000, [2e-05, 9e-05, 2e-05, 2e-05, 2e-06]], ["flex", 1, [2.5e-06, 3e-05, 2.5e-06, 2.5e-06, 2.5e-07]], ["flex", 272000, [2.5e-06, 3e-05, 2.5e-06, 2.5e-06, 2.5e-07]], ["flex", 272001, [5e-06, 4.5e-05, 5e-06, 5e-06, 1e-06]], ["flex", 10000000, [5e-06, 4.5e-05, 5e-06, 5e-06, 1e-06]]]}, +{"info": {"cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, "cache_creation_input_token_cost_priority": 1e-05, "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, "cache_read_input_token_cost": 4e-07, "cache_read_input_token_cost_above_272k_tokens": 8e-07, "cache_read_input_token_cost_priority": 8e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, "input_cost_per_token": 4e-06, "input_cost_per_token_above_272k_tokens": 8e-06, "input_cost_per_token_priority": 8e-06, "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, "output_cost_per_token_priority": 4e-05, "output_cost_per_token_above_272k_tokens_priority": 6e-05}, "inclusive": false, "checks": [[null, 1, [4e-06, 2e-05, 5e-06, 5e-06, 4e-07]], [null, 272000, [4e-06, 2e-05, 5e-06, 5e-06, 4e-07]], [null, 272001, [8e-06, 3e-05, 1e-05, 1e-05, 8e-07]], [null, 10000000, [8e-06, 3e-05, 1e-05, 1e-05, 8e-07]], ["priority", 1, [8e-06, 4e-05, 1e-05, 1e-05, 8e-07]], ["priority", 272000, [8e-06, 4e-05, 1e-05, 1e-05, 8e-07]], ["priority", 272001, [1.6e-05, 6e-05, 2e-05, 2e-05, 1.6e-06]], ["priority", 10000000, [1.6e-05, 6e-05, 2e-05, 2e-05, 1.6e-06]]]}, +{"info": {"cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06, "cache_creation_input_token_cost_priority": 1e-05, "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, "cache_creation_input_token_cost_flex": 2.5e-06, "cache_read_input_token_cost": 4e-07, "cache_read_input_token_cost_above_272k_tokens": 8e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07, "cache_read_input_token_cost_priority": 8e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, "cache_read_input_token_cost_flex": 2e-07, "input_cost_per_token": 4e-06, "input_cost_per_token_above_272k_tokens": 8e-06, "input_cost_per_token_above_272k_tokens_flex": 4e-06, "input_cost_per_token_priority": 8e-06, "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, "input_cost_per_token_flex": 2e-06, "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, "output_cost_per_token_priority": 4e-05, "output_cost_per_token_above_272k_tokens_priority": 6e-05, "output_cost_per_token_flex": 1e-05}, "inclusive": false, "checks": [[null, 1, [4e-06, 2e-05, 5e-06, 5e-06, 4e-07]], [null, 272000, [4e-06, 2e-05, 5e-06, 5e-06, 4e-07]], [null, 272001, [8e-06, 3e-05, 1e-05, 1e-05, 8e-07]], [null, 10000000, [8e-06, 3e-05, 1e-05, 1e-05, 8e-07]], ["priority", 1, [8e-06, 4e-05, 1e-05, 1e-05, 8e-07]], ["priority", 272000, [8e-06, 4e-05, 1e-05, 1e-05, 8e-07]], ["priority", 272001, [1.6e-05, 6e-05, 2e-05, 2e-05, 1.6e-06]], ["priority", 10000000, [1.6e-05, 6e-05, 2e-05, 2e-05, 1.6e-06]], ["flex", 1, [2e-06, 1e-05, 2.5e-06, 2.5e-06, 2e-07]], ["flex", 272000, [2e-06, 1e-05, 2.5e-06, 2.5e-06, 2e-07]], ["flex", 272001, [4e-06, 1.5e-05, 5e-06, 5e-06, 4e-07]], ["flex", 10000000, [4e-06, 1.5e-05, 5e-06, 5e-06, 4e-07]]]}, +{"info": {"cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "input_cost_per_token": 3e-05, "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_above_272k_tokens_batches": 3e-05, "input_cost_per_token_above_272k_tokens_flex": 3e-05, "input_cost_per_token_batches": 1.5e-05, "input_cost_per_token_flex": 1.5e-05, "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_above_272k_tokens_batches": 0.000135, "output_cost_per_token_above_272k_tokens_flex": 0.000135, "output_cost_per_token_batches": 9e-05, "output_cost_per_token_flex": 9e-05}, "inclusive": false, "checks": [[null, 1, [3e-05, 0.00018, 3e-05, 3e-05, 3e-06]], [null, 272000, [3e-05, 0.00018, 3e-05, 3e-05, 3e-06]], [null, 272001, [6e-05, 0.00027, 6e-05, 6e-05, 6e-06]], [null, 10000000, [6e-05, 0.00027, 6e-05, 6e-05, 6e-06]], ["flex", 1, [1.5e-05, 9e-05, 1.5e-05, 1.5e-05, 3e-06]], ["flex", 272000, [1.5e-05, 9e-05, 1.5e-05, 1.5e-05, 3e-06]], ["flex", 272001, [3e-05, 0.000135, 3e-05, 3e-05, 6e-06]], ["flex", 10000000, [3e-05, 0.000135, 3e-05, 3e-05, 6e-06]]]}, +{"info": {"cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, "cache_read_input_token_cost_above_272k_tokens_batches": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 2.5e-07, "cache_read_input_token_cost_priority": 5e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 1e-06, "cache_read_input_token_cost_batches": 1.3e-07, "input_cost_per_token": 2.5e-06, "input_cost_per_token_above_272k_tokens": 5e-06, "input_cost_per_token_priority": 5e-06, "input_cost_per_token_above_272k_tokens_priority": 1e-05, "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, "output_cost_per_token_priority": 3e-05, "output_cost_per_token_above_272k_tokens_priority": 4.5e-05, "input_cost_per_token_above_272k_tokens_batches": 2.5e-06, "input_cost_per_token_above_272k_tokens_flex": 2.5e-06, "input_cost_per_token_batches": 1.25e-06, "input_cost_per_token_flex": 1.25e-06, "output_cost_per_token_above_272k_tokens_batches": 1.125e-05, "output_cost_per_token_above_272k_tokens_flex": 1.125e-05, "output_cost_per_token_batches": 7.5e-06, "output_cost_per_token_flex": 7.5e-06}, "inclusive": false, "checks": [[null, 1, [2.5e-06, 1.5e-05, 2.5e-06, 2.5e-06, 2.5e-07]], [null, 272000, [2.5e-06, 1.5e-05, 2.5e-06, 2.5e-06, 2.5e-07]], [null, 272001, [5e-06, 2.25e-05, 5e-06, 5e-06, 5e-07]], [null, 10000000, [5e-06, 2.25e-05, 5e-06, 5e-06, 5e-07]], ["priority", 1, [5e-06, 3e-05, 5e-06, 5e-06, 5e-07]], ["priority", 272000, [5e-06, 3e-05, 5e-06, 5e-06, 5e-07]], ["priority", 272001, [1e-05, 4.5e-05, 1e-05, 1e-05, 1e-06]], ["priority", 10000000, [1e-05, 4.5e-05, 1e-05, 1e-05, 1e-06]], ["flex", 1, [1.25e-06, 7.5e-06, 1.25e-06, 1.25e-06, 2.5e-07]], ["flex", 272000, [1.25e-06, 7.5e-06, 1.25e-06, 1.25e-06, 2.5e-07]], ["flex", 272001, [2.5e-06, 1.125e-05, 2.5e-06, 2.5e-06, 2.5e-07]], ["flex", 10000000, [2.5e-06, 1.125e-05, 2.5e-06, 2.5e-06, 2.5e-07]]]}, +{"info": {"cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_priority": 1e-06, "cache_read_input_token_cost_above_272k_tokens_priority": 2e-06, "input_cost_per_token": 5e-06, "input_cost_per_token_above_272k_tokens": 1e-05, "input_cost_per_token_priority": 1e-05, "input_cost_per_token_above_272k_tokens_priority": 2e-05, "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, "output_cost_per_token_priority": 6e-05, "output_cost_per_token_above_272k_tokens_priority": 9e-05}, "inclusive": false, "checks": [[null, 1, [5e-06, 3e-05, 5e-06, 5e-06, 5e-07]], [null, 272000, [5e-06, 3e-05, 5e-06, 5e-06, 5e-07]], [null, 272001, [1e-05, 4.5e-05, 1e-05, 1e-05, 1e-06]], [null, 10000000, [1e-05, 4.5e-05, 1e-05, 1e-05, 1e-06]], ["priority", 1, [1e-05, 6e-05, 1e-05, 1e-05, 1e-06]], ["priority", 272000, [1e-05, 6e-05, 1e-05, 1e-05, 1e-06]], ["priority", 272001, [2e-05, 9e-05, 2e-05, 2e-05, 2e-06]], ["priority", 10000000, [2e-05, 9e-05, 2e-05, 2e-05, 2e-06]]]}, +{"info": {"cache_read_input_token_cost": 3.125e-07, "input_cost_per_token": 1.25e-06, "input_cost_per_token_batches": 6.25e-07, "input_cost_per_token_priority": 2.1875e-06, "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, "output_cost_per_token_priority": 1.75e-05}, "inclusive": false, "checks": [[null, 1, [1.25e-06, 1e-05, 1.25e-06, 1.25e-06, 3.125e-07]], [null, 10000000, [1.25e-06, 1e-05, 1.25e-06, 1.25e-06, 3.125e-07]], ["priority", 1, [2.1875e-06, 1.75e-05, 2.1875e-06, 2.1875e-06, 3.125e-07]], ["priority", 10000000, [2.1875e-06, 1.75e-05, 2.1875e-06, 2.1875e-06, 3.125e-07]]]} +]} diff --git a/litellm-rust/crates/model-catalog/tests/registry_validation.rs b/litellm-rust/crates/model-catalog/tests/registry_validation.rs deleted file mode 100644 index 3b65ef13d78..00000000000 --- a/litellm-rust/crates/model-catalog/tests/registry_validation.rs +++ /dev/null @@ -1,93 +0,0 @@ -use std::path::{Path, PathBuf}; - -use litellm_model_catalog::{ - Catalog, FallbackGeneralizations, Provenance, validate_model_entry, validate_registry, -}; -use rstest::{fixture, rstest}; -use serde_json::{Map, Value}; - -#[fixture] -fn repo_root() -> PathBuf { - Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..") -} - -#[rstest] -#[case("model_prices_and_context_window.json")] -#[case("litellm/model_prices_and_context_window_backup.json")] -#[ignore] -fn checked_in_registry_passes_strict_validation(repo_root: PathBuf, #[case] filename: &str) { - let body = std::fs::read(repo_root.join(filename)).unwrap(); - let catalog = Catalog::parse(&body, Provenance::default()).unwrap(); - validate_registry(&catalog).unwrap(); -} - -#[rstest] -#[ignore] -fn fallback_generalizations_are_typed(repo_root: PathBuf) { - let body = std::fs::read(repo_root.join("model_prices_and_context_window.json")).unwrap(); - let document: Map = serde_json::from_slice(&body).unwrap(); - let Some(raw_rules) = document.get("fallback_generalizations") else { - return; - }; - let _: FallbackGeneralizations = serde_json::from_value(raw_rules.clone()).unwrap(); - let catalog = Catalog::parse(&body, Provenance::default()).unwrap(); - assert!( - catalog - .fallback_rules() - .is_some_and(|rules| !rules.is_empty()) - ); -} - -#[rstest] -#[case::missing_provider(serde_json::json!({"mode": "chat"}), "litellm_provider")] -#[case::unknown_field(serde_json::json!({"litellm_provider": "test", "typo": true}), "unknown field")] -#[case::negative_price(serde_json::json!({"litellm_provider": "test", "input_cost_per_token": -1}), "nonnegative")] -#[case::negative_nested_price(serde_json::json!({"litellm_provider": "test", "guardrail_cost_per_unit": {"unit": -1}}), "nonnegative")] -#[case::invalid_mode(serde_json::json!({"litellm_provider": "test", "mode": "invalid"}), "unknown variant")] -#[case::invalid_date(serde_json::json!({"litellm_provider": "test", "deprecation_date": "2026-02-31"}), "deprecation_date")] -#[case::invalid_hours(serde_json::json!({"litellm_provider": "test", "off_peak_pricing": {"hours_utc": "25:00-01:00"}}), "hours_utc")] -#[case::empty_windows(serde_json::json!({"litellm_provider": "test", "off_peak_pricing": {"windows": []}}), "windows is empty")] -#[case::invalid_weekday(serde_json::json!({"litellm_provider": "test", "off_peak_pricing": {"windows": [{"hours_utc": "00:00-01:00", "weekdays": [0]}]}}), "weekdays is invalid")] -#[case::invalid_aliases(serde_json::json!({"litellm_provider": "test", "aliases": ["good", 7]}), "aliases must contain strings")] -#[case::null_aliases(serde_json::json!({"litellm_provider": "test", "aliases": null}), "aliases must be an array")] -#[ignore] -fn registry_validation_rejects_malformed_entries(#[case] entry: Value, #[case] expected: &str) { - let error = validate_model_entry("test", &entry).unwrap_err(); - assert!( - litellm_error::Report::new(&error) - .to_string() - .contains(expected) - ); -} - -#[test] -#[ignore] -fn checked_in_catalog_and_backup_match() { - let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../.."); - let current = std::fs::read(root.join("model_prices_and_context_window.json")).unwrap(); - let backup = - std::fs::read(root.join("litellm/model_prices_and_context_window_backup.json")).unwrap(); - assert_eq!(current, backup); - let catalog = Catalog::parse(¤t, Provenance::default()).unwrap(); - assert!( - catalog.alias_issues().is_empty(), - "invalid registry aliases" - ); -} - -#[rstest] -#[case::input("input_cost_per_token_above_100k_tokens")] -#[case::input_batches("input_cost_per_token_above_100k_tokens_batches")] -#[case::output("output_cost_per_token_above_100k_tokens")] -#[case::output_batches("output_cost_per_token_above_100k_tokens_batches")] -#[case::cache_creation("cache_creation_input_token_cost_above_100k_tokens")] -#[case::cache_creation_batches("cache_creation_input_token_cost_above_100k_tokens_batches")] -#[case::cache_creation_1hr("cache_creation_input_token_cost_above_1hr_above_100k_tokens")] -#[case::cache_read("cache_read_input_token_cost_above_100k_tokens")] -#[case::cache_read_batches("cache_read_input_token_cost_above_100k_tokens_batches")] -fn registry_validation_keeps_above_100k_tier_rates(#[case] field: &str) { - let mut entry = Map::new(); - entry.insert("litellm_provider".into(), "anthropic".into()); - entry.insert(field.into(), 5e-7.into()); - validate_model_entry("test", &Value::Object(entry)).unwrap(); -} diff --git a/litellm-rust/crates/model-catalog/tests/schema.rs b/litellm-rust/crates/model-catalog/tests/schema.rs index f9a028a78ab..464cf069cac 100644 --- a/litellm-rust/crates/model-catalog/tests/schema.rs +++ b/litellm-rust/crates/model-catalog/tests/schema.rs @@ -1,16 +1,17 @@ #![cfg(feature = "schema")] -use std::collections::BTreeSet; -use std::path::Path; - -use litellm_model_catalog::{model_entry_json_schema, registry_json_schema}; +use litellm_model_catalog::{BUNDLED_REGISTRY, model_entry_json_schema, registry_json_schema}; use rstest::rstest; -use serde_json::{Value, json}; +use serde_json::{Map, Value, json}; fn schema() -> Value { serde_json::to_value(model_entry_json_schema()).expect("generated schema serializes") } +fn checked_in_catalog() -> Value { + serde_json::from_slice(BUNDLED_REGISTRY).unwrap() +} + fn registry_validator() -> jsonschema::Validator { let schema = serde_json::to_value(registry_json_schema()).unwrap(); jsonschema::options() @@ -19,26 +20,21 @@ fn registry_validator() -> jsonschema::Validator { .expect("generated registry schema is valid") } -#[rstest] -#[case("model_prices_and_context_window.json")] -#[case("litellm/model_prices_and_context_window_backup.json")] -#[ignore] -fn generated_registry_schema_validates_checked_in_catalog(#[case] path: &str) { - let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../.."); - let catalog: Value = serde_json::from_slice(&std::fs::read(root.join(path)).unwrap()).unwrap(); +#[test] +fn generated_registry_schema_validates_checked_in_catalog() { + let catalog = checked_in_catalog(); let validator = registry_validator(); let errors: Vec<_> = validator .iter_errors(&catalog) .map(|error| error.to_string()) .collect(); - assert!(errors.is_empty(), "{path}: {errors:?}"); + assert!(errors.is_empty(), "{errors:?}"); } #[rstest] #[case(json!({"example": {"litellm_provider": "test"}}))] #[case(json!({"example": {"litellm_provider": "test", "future_field": true}}))] #[case(json!({"sample_spec": {"litellm_provider": "placeholder"}}))] -#[ignore] fn generated_registry_schema_keeps_reader_compatibility(#[case] document: Value) { assert!(registry_validator().is_valid(&document)); } @@ -59,47 +55,55 @@ fn generated_registry_schema_keeps_reader_compatibility(#[case] document: Value) #[case::invalid_weekday(json!({"litellm_provider": "test", "off_peak_pricing": {"windows": [{"hours_utc": "00:00-01:00", "weekdays": [0]}]}}))] #[case::invalid_aliases(json!({"litellm_provider": "test", "aliases": "wrong"}))] #[case::non_object_model(json!(4))] -#[ignore] fn generated_registry_schema_rejects_invalid_entries(#[case] entry: Value) { assert!(!registry_validator().is_valid(&json!({"example": entry}))); } -#[rstest] -#[case("model_prices_and_context_window.json")] -#[case("litellm/model_prices_and_context_window_backup.json")] -#[ignore] -fn generated_schema_covers_catalog_fields(#[case] path: &str) { - let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../.."); - let catalog: Value = serde_json::from_slice(&std::fs::read(root.join(path)).unwrap()).unwrap(); +#[test] +fn generated_schema_covers_catalog_fields() { + let catalog = checked_in_catalog(); let schema = schema(); - let properties = schema["properties"] + let any_value = |keys: &Value| -> Map { + keys.as_object() + .expect("schema keys are an object") + .keys() + .map(|key| (key.clone(), json!({}))) + .collect() + }; + let keys_only = json!({ + "type": "object", + "properties": any_value(&schema["properties"]), + "patternProperties": any_value(&schema["patternProperties"]), + "additionalProperties": false, + }); + let registry = serde_json::to_value(registry_json_schema()).unwrap(); + let sections = registry["properties"] .as_object() - .expect("ModelInfo schema has properties"); - let fields: BTreeSet<&str> = catalog + .expect("registry has sections"); + let fields: Map = catalog .as_object() .expect("catalog is an object") .iter() - .filter(|(name, _)| *name != "sample_spec" && *name != "fallback_generalizations") + .filter(|(name, _)| !sections.contains_key(*name)) .flat_map(|(_, entry)| entry.as_object().expect("model entry is an object").keys()) - .map(String::as_str) - .filter(|name| *name != "aliases") + .map(|name| (name.clone(), Value::Null)) .collect(); - let missing: Vec<_> = fields - .into_iter() - .filter(|name| !properties.contains_key(*name)) + let missing: Vec<_> = jsonschema::validator_for(&keys_only) + .unwrap() + .iter_errors(&Value::Object(fields)) + .map(|error| error.to_string()) .collect(); assert!( missing.is_empty(), - "{path}: fields missing from schema: {missing:?}" + "fields missing from schema: {missing:?}" ); } #[rstest] #[case("Mode", "chat")] -#[case("ReasoningEffort", "high")] +#[case("EffortLevel", "high")] #[case("InputModality", "image")] -#[ignore] fn generated_schema_includes_enum_values(#[case] definition: &str, #[case] value: &str) { let schema = schema(); let variants = schema["$defs"][definition]["enum"] @@ -110,7 +114,6 @@ fn generated_schema_includes_enum_values(#[case] definition: &str, #[case] value } #[test] -#[ignore] fn generated_schema_includes_nested_pricing_types() { let schema = schema(); let definitions = schema["$defs"].as_object().expect("schema has definitions"); diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs index a1a56baef8f..c21ce6d76b6 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs @@ -13,7 +13,7 @@ use pyo3::{ use super::{ errors::route_failure, - project::{OcrHostHandles, SETTINGS_ERROR_MARKER, project_request}, + project::{OcrHostHandles, ROUTE_HOST_MODULE, SETTINGS_ERROR_MARKER, project_request}, }; use crate::{marshal::public_response, routes::RequestView}; @@ -77,7 +77,7 @@ impl OcrPythonHost { _ => "", }; let mapped = py - .import("litellm.rust_bridge.host.routes.ocr") + .import(ROUTE_HOST_MODULE) .and_then(|module| module.getattr("map_failure")) .and_then(|map| map.call1((error.value(py), self.view.request(py)?, provider))) .and_then(|mapped| mapped.extract::>().map_err(PyErr::from)); @@ -115,7 +115,7 @@ impl PythonBinding for OcrPythonHost { py: Python<'_>, response: LiteLLMOcrResponse, ) -> PyResult> { - public_response(py, "litellm.rust_bridge.host.routes.ocr", &response) + public_response(py, ROUTE_HOST_MODULE, &response) } fn encode_stream_head( diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs index 40287c669eb..c2c3fe51600 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs @@ -5,7 +5,7 @@ use litellm_inference_ocr::{ types::{LiteLLMOcrRequest, OcrDocumentInput}, wire::{OcrWireRequest, consumed_optional_params, decode_document, decode_request_input}, }; -use litellm_llms::base_llm::ocr::error::Error; +use litellm_llms::base_llm::ocr::{error::Error, transformation::DROP_PARAMS_PARAM}; use pyo3::{prelude::*, types::PyDict}; use serde_json::{Map, Value}; @@ -19,6 +19,8 @@ use crate::{ python_settings::{PythonSettings, Snapshot}, }; +pub(super) const ROUTE_HOST_MODULE: &str = "litellm.rust_bridge.host.routes.ocr"; + /// What the host keeps after projection: the caller's token callable that answers the /// token operation, and the provider name the failure mapping reports. pub(super) struct OcrHostHandles { @@ -139,6 +141,15 @@ pub(super) fn project_request( Some(defaults) => fold_module_globals(bound.py(), optional_params, &globals, &defaults)?, None => optional_params, }; + let optional_params = if names.contains(&DROP_PARAMS_PARAM) { + fold_drop_params( + bound.py(), + optional_params, + present(kwargs, bound, DROP_PARAMS_PARAM)?, + )? + } else { + optional_params + }; let input_sources = request_input_sources( kwargs, names @@ -216,6 +227,23 @@ fn fold_module_globals( .collect()) } +fn fold_drop_params( + py: Python<'_>, + optional_params: Map, + requested: Option>, +) -> PyResult> { + let drops: bool = py + .import(ROUTE_HOST_MODULE)? + .getattr("drops_unsupported_params")? + .call1((requested,))? + .extract()?; + Ok(optional_params + .into_iter() + .filter(|(name, _)| name != DROP_PARAMS_PARAM) + .chain(drops.then(|| (DROP_PARAMS_PARAM.to_string(), Value::Bool(true)))) + .collect()) +} + /// A misconfigured litellm setting is the operator's error, not the request's or the /// provider's, so the host raises it as is instead of mapping it onto a provider failure. pub(super) const SETTINGS_ERROR_MARKER: &str = "native_settings_error"; diff --git a/litellm-rust/crates/router/Cargo.toml b/litellm-rust/crates/router/Cargo.toml index a356bedcf16..988d1b68805 100644 --- a/litellm-rust/crates/router/Cargo.toml +++ b/litellm-rust/crates/router/Cargo.toml @@ -7,6 +7,7 @@ repository.workspace = true [dependencies] litellm-config.workspace = true +litellm-core-utils.workspace = true litellm-inference.workspace = true litellm-inference-messages.workspace = true litellm-router-types.workspace = true diff --git a/litellm-rust/crates/router/README.md b/litellm-rust/crates/router/README.md index 972d37d14be..a734e634104 100644 --- a/litellm-rust/crates/router/README.md +++ b/litellm-rust/crates/router/README.md @@ -1,4 +1,4 @@ -`litellm-router` scaffolds the model-list setup and deployment lookup portion of Python's `litellm.Router`. `Router::from_model_list(&config.model_list)` maps configured public names to provider deployments. Programmatic callers can collect `(String, Deployment)` entries into a `Router` +`litellm-router` scaffolds the model-list setup and deployment lookup portion of Python's `litellm.Router`. `Router::from_config(&config, &env)` maps configured public names to provider deployments and reads `drop_params` from `litellm_settings` and each deployment, resolving an `os.environ/NAME` value through `env`. Programmatic callers can collect `(String, Deployment)` entries into a `Router` Lookup is exact and returns `None` for an unknown name. This extraction preserves the gateway's existing behavior: the last entry wins when public names repeat. Multiple deployments per model group, routing strategies, retries, cooldowns, and fallbacks are not implemented yet diff --git a/litellm-rust/crates/router/src/lib.rs b/litellm-rust/crates/router/src/lib.rs index fb48213e8a7..b75c1a14446 100644 --- a/litellm-rust/crates/router/src/lib.rs +++ b/litellm-rust/crates/router/src/lib.rs @@ -2,16 +2,33 @@ mod deployment; use std::collections::HashMap; -use litellm_config::Model; +use litellm_config::{Config, Spelled}; +use litellm_core_utils::{serde_compat::parse_pydantic_bool, settings::Lookup}; +use litellm_inference_messages::{MessagesSettings, MessagesShaping}; pub use deployment::Deployment; #[derive(Clone, Debug, Default)] pub struct Router(HashMap); +fn is_on(flag: Option<&Spelled>, env: &impl Lookup) -> bool { + match flag { + Some(Spelled::Value(on)) => *on, + Some(Spelled::Text(text)) => match text.strip_prefix("os.environ/") { + Some(name) => env + .get(name) + .is_some_and(|value| parse_pydantic_bool(&value) == Some(true)), + None => parse_pydantic_bool(text) == Some(true), + }, + None => false, + } +} + impl Router { - pub fn from_model_list(model_list: &[Model]) -> Self { - model_list + pub fn from_config(config: &Config, env: &impl Lookup) -> Self { + let drops_params_globally = is_on(config.litellm_settings.drop_params.as_ref(), env); + config + .model_list .iter() .map(|model| { ( @@ -26,6 +43,14 @@ impl Router { api_base: model.litellm_params.api_base.clone(), custom_llm_provider: model.litellm_params.custom_llm_provider.clone(), litellm_params: model.litellm_params.clone(), + shaping: MessagesShaping { + settings: MessagesSettings { + drop_params: drops_params_globally + || is_on(model.litellm_params.drop_params.as_ref(), env), + ..MessagesSettings::default() + }, + ..MessagesShaping::default() + }, ..Deployment::default() }, ) @@ -43,3 +68,60 @@ impl FromIterator<(String, Deployment)> for Router { Self(entries.into_iter().collect()) } } + +#[cfg(test)] +mod tests { + use litellm_config::Config; + use rstest::rstest; + + use super::Router; + + #[rstest] + #[case::unset("", "", false)] + #[case::deployment_flag("", "\n drop_params: true", true)] + #[case::deployment_word("", "\n drop_params: \" True \"", true)] + #[case::deployment_off("", "\n drop_params: false", false)] + #[case::deployment_unknown_word("", "\n drop_params: maybe", false)] + #[case::deployment_yes("", "\n drop_params: yes", true)] + #[case::deployment_no("", "\n drop_params: \"no\"", false)] + #[case::deployment_env_reference("", "\n drop_params: os.environ/DROP_PARAMS_FLAG", true)] + #[case::deployment_unset_env_reference( + "", + "\n drop_params: os.environ/MISSING_FLAG", + false + )] + #[case::global_env_reference( + "litellm_settings:\n drop_params: os.environ/DROP_PARAMS_FLAG\n", + "", + true + )] + #[case::global_flag("litellm_settings:\n drop_params: true\n", "", true)] + #[case::global_word("litellm_settings:\n drop_params: \"true\"\n", "", true)] + #[case::global_off("litellm_settings:\n drop_params: false\n", "", false)] + #[case::global_over_deployment_off( + "litellm_settings:\n drop_params: true\n", + "\n drop_params: false", + true + )] + fn configured_drop_params_reach_the_deployment( + #[case] settings: &str, + #[case] deployment: &str, + #[case] expected: bool, + ) { + let config = Config::from_yaml(&format!( + "{settings}model_list:\n - model_name: public-model\n litellm_params:\n model: provider/model{deployment}" + )) + .unwrap(); + + assert_eq!( + Router::from_config(&config, &|name: &str| (name == "DROP_PARAMS_FLAG") + .then(|| " Yes ".to_owned())) + .get("public-model") + .unwrap() + .shaping + .settings + .drop_params, + expected + ); + } +} diff --git a/litellm-rust/crates/router/tests/router.rs b/litellm-rust/crates/router/tests/router.rs index bbaeb806b96..03e78ab9d11 100644 --- a/litellm-rust/crates/router/tests/router.rs +++ b/litellm-rust/crates/router/tests/router.rs @@ -16,7 +16,7 @@ fn configuration_preserves_deployment_parameters(#[case] parameters: &str) { "model_list:\n - model_name: public-model\n litellm_params:\n model: provider/model\n {parameters}" )) .unwrap(); - let router = Router::from_model_list(&config.model_list); + let router = Router::from_config(&config, &|_: &str| None); let deployment = router.get(&config.model_list[0].model_name).unwrap(); let params = &config.model_list[0].litellm_params; @@ -44,7 +44,7 @@ fn configuration_types_the_provider_params_of_a_deployment() { azure_ad_token: ignored-by-every-group", ) .unwrap(); - let router = Router::from_model_list(&config.model_list); + let router = Router::from_config(&config, &|_: &str| None); let params = &router.get("public-model").unwrap().litellm_params; assert_eq!( @@ -74,7 +74,7 @@ fn lookup_uses_public_names(#[case] name: &str, #[case] expected: Option<&str>) model: provider/b", ) .unwrap(); - let router = Router::from_model_list(&config.model_list); + let router = Router::from_config(&config, &|_: &str| None); assert_eq!(router.get(name).map(|entry| entry.model.as_str()), expected); } @@ -84,7 +84,7 @@ fn empty_configuration_has_no_deployment() { let config = Config::from_yaml("model_list: []").unwrap(); assert!( - Router::from_model_list(&config.model_list) + Router::from_config(&config, &|_: &str| None) .get("") .is_none() ); diff --git a/litellm/__init__.py b/litellm/__init__.py index c649a48211a..a1bda4aa9f7 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -39,6 +39,7 @@ from litellm._logging import ( turn_on_json, log_level, ) +from litellm._logging import suppress_startup_output as suppress_startup_output import re from litellm.constants import ( DEFAULT_BATCH_SIZE, @@ -591,7 +592,7 @@ vertex_ai_safety_settings: Optional[dict] = None ####### COMPLETION MODELS ################### from typing import Set -open_ai_chat_completion_models: Set = set() +open_ai_chat_completion_models: Set[str] = set() open_ai_text_completion_models: Set = set() cohere_models: Set = set() cohere_chat_models: Set = set() @@ -619,6 +620,8 @@ vertex_openai_models: Set = set() vertex_minimax_models: Set = set() vertex_moonshot_models: Set = set() vertex_zai_models: Set = set() +vertex_qwen_models: Final[Set[str]] = set() +vertex_ai_models: Final[Set[str]] = set() ai21_models: Set = set() ai21_chat_models: Set = set() nlp_cloud_models: Set = set() @@ -635,6 +638,7 @@ watsonx_models: Set = set() gemini_models: Set = set() xai_models: Set = set() zai_models: Set = set() +bigmodel_models: Final[set[str]] = set() # mutable-ok: provider registry is populated and extended at runtime deepseek_models: Set = set() tencent_models: Set = set() runwayml_models: Set = set() @@ -681,6 +685,8 @@ moonshot_models: Set = set() publicai_models: Set = set() darkbloom_models: Set = set() openinfer_models: Final[Set[str]] = set() +kyma_models: Final[set[str]] = set() # mutable-ok: populated when the shared model catalog loads +streamlake_models: Final[Set[str]] = set() zerogpu_models: Final[Set[str]] = set() v0_models: Set = set() morph_models: Set = set() @@ -832,6 +838,10 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje elif value.get("litellm_provider") == "vertex_ai-zai_models": key = key.replace("vertex_ai/", "") vertex_zai_models.add(key) + elif value.get("litellm_provider") == "vertex_ai-qwen_models": + vertex_qwen_models.add(key.removeprefix("vertex_ai/")) + elif value.get("litellm_provider") == "vertex_ai": + vertex_ai_models.add(key.removeprefix("vertex_ai/")) elif value.get("litellm_provider") == "ai21": if value.get("mode") == "chat": ai21_chat_models.add(key) @@ -876,6 +886,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje xai_models.add(key) elif value.get("litellm_provider") == "zai": zai_models.add(key) + elif value.get("litellm_provider") == "bigmodel": + bigmodel_models.add(key) elif value.get("litellm_provider") == "fal_ai": fal_ai_models.add(key) elif value.get("litellm_provider") == "deepseek": @@ -970,6 +982,10 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje darkbloom_models.add(key) elif value.get("litellm_provider") == "openinfer": openinfer_models.add(key) + elif value.get("litellm_provider") == "kyma": + kyma_models.add(key) + elif value.get("litellm_provider") == "streamlake": + streamlake_models.add(key) elif value.get("litellm_provider") == "zerogpu": zerogpu_models.add(key) elif value.get("litellm_provider") == "v0": @@ -1104,6 +1120,7 @@ model_list = list( | text_completion_inception_models | xai_models | zai_models + | bigmodel_models | fal_ai_models | deepseek_models | modelscope_models @@ -1145,6 +1162,8 @@ model_list = list( | publicai_models | darkbloom_models | openinfer_models + | kyma_models + | streamlake_models | zerogpu_models | v0_models | morph_models @@ -1200,7 +1219,16 @@ def _build_models_by_provider() -> dict: | vertex_deepseek_models | vertex_minimax_models | vertex_moonshot_models - | vertex_zai_models, + | vertex_zai_models + | vertex_qwen_models + | vertex_mistral_models + | vertex_llama3_models + | vertex_ai_ai21_models + | vertex_openai_models + | vertex_ai_image_models + | vertex_ai_video_models + | vertex_embedding_models + | vertex_ai_models, "ai21": ai21_models | ai21_chat_models, "bedrock": bedrock_models | bedrock_converse_models, "petals": petals_models, @@ -1217,6 +1245,7 @@ def _build_models_by_provider() -> dict: "text-completion-inception": text_completion_inception_models, "xai": xai_models, "zai": zai_models, + "bigmodel": bigmodel_models, "fal_ai": fal_ai_models, "deepseek": deepseek_models, "tencent": tencent_models, @@ -1265,6 +1294,8 @@ def _build_models_by_provider() -> dict: "publicai": publicai_models, "darkbloom": darkbloom_models, "openinfer": openinfer_models, + "kyma": kyma_models, + "streamlake": streamlake_models, "zerogpu": zerogpu_models, "v0": v0_models, "morph": morph_models, diff --git a/litellm/_internal_context.py b/litellm/_internal_context.py index 244efc8047b..c2243b90fbf 100644 --- a/litellm/_internal_context.py +++ b/litellm/_internal_context.py @@ -36,6 +36,8 @@ REDIS_FAMILIES_METADATA_KEY: Final = "families" _service_caller: Final[ContextVar[str | None]] = ContextVar("service_caller", default=None) +_mirrored_call: Final[ContextVar[bool]] = ContextVar("mirrored_call", default=False) + @contextmanager def internal_sub_call() -> Generator[None]: @@ -123,6 +125,30 @@ def current_service_caller() -> str | None: return _service_caller.get() +@contextmanager +def mirrored_call() -> Generator[None]: + """A silent_model mirror of a request: every litellm call inside, its retries and fallbacks included.""" + token: Final = _mirrored_call.set(True) + try: + yield + finally: + _restore(_mirrored_call, token) + + +def in_mirrored_call() -> bool: + return _mirrored_call.get() + + +@contextmanager +def outside_mirrored_call() -> Generator[None]: + """Logging a mirrored call: the calls its callbacks start are not mirrors themselves.""" + token: Final = _mirrored_call.set(False) + try: + yield + finally: + _restore(_mirrored_call, token) + + @contextmanager def pinned_billing_time(moment: datetime) -> Generator[None]: """Price every rate lookup inside this block at ``moment`` rather than at each one's own clock read.""" diff --git a/litellm/_logging.py b/litellm/_logging.py index 8974f85f517..5279cb233b4 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -10,10 +10,12 @@ import sys from collections.abc import Iterator, Mapping from datetime import datetime from logging import Formatter +from types import FrameType from typing import Final, TextIO from urllib.parse import unquote from pydantic import TypeAdapter +from typing_extensions import ReadOnly, TypedDict import litellm from litellm.constants import ( @@ -598,6 +600,7 @@ _LOG_FORMAT_PREFIX: Final = "%(asctime)s - %(name)s:%(levelname)s" _LOG_FORMAT_SUFFIX: Final = ": %(filename)s:%(lineno)s - %(message)s" _PLAIN_LOG_FORMAT: Final = _LOG_FORMAT_PREFIX + _LOG_FORMAT_SUFFIX _COLOR_LOG_FORMAT: Final = f"\033[92m{_LOG_FORMAT_PREFIX}\033[0m{_LOG_FORMAT_SUFFIX}" +_ANSI_ESCAPE: Final = re.compile(r"\x1b\[[0-9;]*m") def _stream_is_tty(stream: TextIO | None) -> bool: @@ -612,15 +615,28 @@ def _stream_is_tty(stream: TextIO | None) -> bool: return False -def _plain_log_format(stdout: TextIO | None, stderr: TextIO | None) -> str: - """The plain-text log format, colorized only when both streams are an interactive terminal. +def _color_enabled(stream: TextIO | None) -> bool: + """True when ANSI colors written to the stream reach an interactive terminal. Honors the NO_COLOR convention from no-color.org: color is disabled when NO_COLOR is present with a non-empty value. """ - if os.environ.get("NO_COLOR"): - return _PLAIN_LOG_FORMAT - return _COLOR_LOG_FORMAT if _stream_is_tty(stdout) and _stream_is_tty(stderr) else _PLAIN_LOG_FORMAT + return not os.environ.get("NO_COLOR") and _stream_is_tty(stream) + + +def _terminal_colors(stdout: TextIO | None, stderr: TextIO | None) -> bool: + """True when both streams are an interactive color terminal, the one rule for log lines and startup prints.""" + return _color_enabled(stdout) and _color_enabled(stderr) + + +def terminal_text(message: str) -> str: + """The message with its ANSI colors kept only on a color terminal, by the rule the plain log format uses.""" + return message if _terminal_colors(sys.stdout, sys.stderr) else _ANSI_ESCAPE.sub("", message) + + +def _plain_log_format(stdout: TextIO | None, stderr: TextIO | None) -> str: + """The plain-text log format, colorized only when both streams are an interactive terminal.""" + return _COLOR_LOG_FORMAT if _terminal_colors(stdout, stderr) else _PLAIN_LOG_FORMAT class LevelRoutingStreamHandler(logging.StreamHandler): @@ -642,8 +658,8 @@ class LevelRoutingStreamHandler(logging.StreamHandler): super().emit(record) -def _parse_json_logs_env(value: str | None) -> bool: - """Strict opt-in parse for the JSON_LOGS env var: only "true" (any case) enables JSON logs. +def _parse_env_flag(value: str | None) -> bool: + """Strict opt-in parse for a boolean env var such as JSON_LOGS: only "true" (any case) enables it. Matches the reader in litellm-proxy-extras/_logging.py. The previous bool(os.getenv(...)) treated any non-empty value, including "false" and "0", @@ -656,7 +672,18 @@ def resolve_log_level(log_level: str) -> int: return getattr(logging, log_level.upper()) -json_logs: Final = _parse_json_logs_env(os.getenv("JSON_LOGS")) +json_logs: Final = _parse_env_flag(os.getenv("JSON_LOGS")) +suppress_startup_output: Final = _parse_env_flag(os.getenv("LITELLM_SUPPRESS_STARTUP_OUTPUT")) + + +class StartupOutputSettings(TypedDict, total=False): + json_logs: ReadOnly[object] + suppress_startup_output: ReadOnly[object] + + +STARTUP_OUTPUT_SETTINGS: Final = TypeAdapter(StartupOutputSettings) + + # Create a handler for the logger (you may need to adapt this based on your needs) log_level: Final = os.getenv("LITELLM_LOG", "DEBUG") numeric_level: Final[int] = resolve_log_level(log_level) @@ -743,7 +770,7 @@ class JsonFormatter(Formatter): return dt.isoformat() def format(self, record): - message_str: Final = record.getMessage() + message_str: Final = _ANSI_ESCAPE.sub("", record.getMessage()) json_record: Final[dict[str, object]] = { "message": message_str, "level": record.levelname, @@ -799,12 +826,14 @@ class CorrelationPlainFormatter(logging.Formatter): """Appends trace_id/session_id to plain-text log lines stamped by CorrelationContextFilter. Mirrors JsonFormatter's handling of these two fields so request_correlation_in_logs - behaves the same whether or not json_logs is enabled. + behaves the same whether or not json_logs is enabled. Unless it was built with the color format, + it also drops the ANSI color codes a message carries, which a piped log would store as text. """ def format(self, record: logging.LogRecord) -> str: rendered: Final = super().format(record) - formatted: Final = rendered if _is_redacted(record) else redact_string(rendered) + uncolored: Final = rendered if self._fmt == _COLOR_LOG_FORMAT else _ANSI_ESCAPE.sub("", rendered) + formatted: Final = uncolored if _is_redacted(record) else redact_string(uncolored) trace_id: Final = getattr(record, "trace_id", None) session_id: Final = getattr(record, "session_id", None) if not trace_id and not session_id: @@ -898,29 +927,46 @@ verbose_proxy_logger.addFilter(_diagnostic_filter) verbose_proxy_stdout_logger.addFilter(_diagnostic_filter) verbose_logger.addFilter(_diagnostic_filter) -_ANSI_ESCAPE: Final = re.compile(r"\x1b\[[0-9;]*m") - def print_startup_message(message: str) -> None: """Write a boot-time message meant for the operator's terminal. - Plain mode prints it unchanged, colors and spacing included. With JSON logs on, every line of - stdout is read by a log pipeline, so the message goes out as one INFO record of the proxy logger - instead, on stdout like the print whatever handlers the logging config gave that logger, with the - ANSI color codes and surrounding blank lines removed, located at the caller and emitted at any log - level, whole and unredacted as the print showed it; a blank spacer produces no record. + LITELLM_SUPPRESS_STARTUP_OUTPUT or litellm_settings.suppress_startup_output turns every one of them + off; a warning goes through print_startup_warning instead. Plain mode prints it with its spacing, + keeping the ANSI colors only on a color terminal by the same check as the plain log format, so a + container log never stores escape codes. + With JSON logs on, every line of stdout is read by a log pipeline, so the message goes out as one + INFO record of the proxy logger instead, on stdout like the print whatever handlers the logging + config gave that logger, with the ANSI color codes and surrounding blank lines removed, located at + the caller and emitted at any log level, whole and unredacted as the print showed it; a blank + spacer produces no record. """ + if litellm.suppress_startup_output is True: + return + frame: Final = inspect.currentframe() + _write_startup_message(message, frame.f_back if frame is not None else None, logging.INFO) + + +def print_startup_warning(message: str) -> None: + """Write a boot-time warning as print_startup_message writes a message, even with suppress_startup_output + on, since it names something the operator has to fix. With JSON logs on it is a WARNING record, on + stdout where the print put it, so a collector that filters on the level field keeps it.""" + frame: Final = inspect.currentframe() + _write_startup_message(message, frame.f_back if frame is not None else None, logging.WARNING) + + +def _write_startup_message(message: str, caller: FrameType | None, level: int) -> None: if not litellm.json_logs: - print(message) # noqa: T201 # plain mode: this is the operator-terminal print it replaces + import click + + click.echo(terminal_text(message)) return text: Final = _ANSI_ESCAPE.sub("", message).strip("\n") if not text: return - frame: Final = inspect.currentframe() - caller: Final = frame.f_back if frame is not None else None record: Final = verbose_proxy_logger.makeRecord( verbose_proxy_logger.name, - logging.INFO, + level, caller.f_code.co_filename if caller is not None else __file__, caller.f_lineno if caller is not None else 0, text, @@ -934,8 +980,8 @@ def print_startup_message(message: str) -> None: def _startup_json_handler() -> logging.Handler: - """The proxy's JSON handler on stdout, filters included, for one startup message.""" - handler: Final = LevelRoutingStreamHandler() + """The proxy's JSON handler on stdout, filters included, for one startup message or warning.""" + handler: Final = logging.StreamHandler(sys.stdout) handler.setFormatter(JsonFormatter()) handler.addFilter(_secret_filter) handler.addFilter(_correlation_filter) diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index 5f5b8210096..1ced13445de 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -455,17 +455,33 @@ class Cache: 3. Else use the `model` passed in kwargs, plus `model_id` when set (e.g. a SageMaker inference component) """ litellm_params: Final[dict] = kwargs.get("litellm_params", {}) or {} - metadata_sources: Final[tuple[dict, ...]] = ( - kwargs.get("metadata") or {}, - kwargs.get("litellm_metadata") or {}, - litellm_params.get("metadata") or {}, - litellm_params.get("litellm_metadata") or {}, + metadata_sources: Final = cast( # cast-ok: each source is the untyped request's metadata dict or empty + "tuple[dict[str, object], dict[str, object], dict[str, object], dict[str, object]]", + ( + kwargs.get("metadata") or {}, + kwargs.get("litellm_metadata") or {}, + litellm_params.get("metadata") or {}, + litellm_params.get("litellm_metadata") or {}, + ), ) - model_group: Final[str | None] = next( - (source["model_group"] for source in metadata_sources if source.get("model_group")), None + metadata, litellm_metadata, params_metadata, params_litellm_metadata = metadata_sources + call_model_info: Final[object] = kwargs.get("model_info") + routed_source: Final = next( + ( + source + for source in (litellm_metadata, metadata, params_litellm_metadata, params_metadata) + if call_model_info and source.get("model_group") and source.get("model_info") == call_model_info + ), + None, ) + model_group: Final = ( + str(routed_source["model_group"]) + if routed_source is not None + else next((str(source["model_group"]) for source in metadata_sources if source.get("model_group")), None) + ) + caching_group_sources: Final = (routed_source,) if routed_source is not None else metadata_sources caching_group: Final = next( - (group for source in metadata_sources if (group := self._get_caching_group(source, model_group))), None + (group for source in caching_group_sources if (group := self._get_caching_group(source, model_group))), None ) model_id_suffix: Final = ( f" model_id: {kwargs['model_id']}" if "model_id" in kwargs and kwargs["model_id"] is not None else "" diff --git a/litellm/completion_extras/litellm_responses_transformation/handler.py b/litellm/completion_extras/litellm_responses_transformation/handler.py index 3f7017eccc2..76d17ea9693 100644 --- a/litellm/completion_extras/litellm_responses_transformation/handler.py +++ b/litellm/completion_extras/litellm_responses_transformation/handler.py @@ -260,6 +260,7 @@ class ResponsesToCompletionBridgeHandler: structured_output=is_structured_output_request( cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict ), + replays_openai_history=custom_llm_provider == "openai", ) streamwrapper: Final = CustomStreamWrapper( completion_stream=completion_stream, @@ -363,6 +364,7 @@ class ResponsesToCompletionBridgeHandler: structured_output=is_structured_output_request( cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict ), + replays_openai_history=custom_llm_provider == "openai", ) streamwrapper: Final = CustomStreamWrapper( completion_stream=completion_stream, diff --git a/litellm/completion_extras/litellm_responses_transformation/history.py b/litellm/completion_extras/litellm_responses_transformation/history.py new file mode 100644 index 00000000000..4435de64cde --- /dev/null +++ b/litellm/completion_extras/litellm_responses_transformation/history.py @@ -0,0 +1,432 @@ +import hashlib +import json +from collections.abc import Iterator, Mapping, Sequence +from dataclasses import dataclass +from itertools import accumulate, chain, pairwise +from types import MappingProxyType +from typing import Annotated, Final, Literal, TypeAlias + +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError + +REPLAY_FIELD: Final = "openai_responses_replay" +PHASE_FIELD: Final = "phase" + +Phase: TypeAlias = Literal["commentary", "final_answer"] + + +class _Frozen(BaseModel): + model_config = ConfigDict(frozen=True) + + +class _OutputText(_Frozen): + type: Literal["output_text"] + text: str + + +class _OutputMessage(_Frozen): + type: Literal["message"] + role: Literal["assistant"] + content: tuple[_OutputText, ...] + phase: Phase | None = None + + +class _OutputReasoning(_Frozen): + type: Literal["reasoning"] + id: str + + +class _OutputToolCall(_Frozen): + type: Literal["function_call", "custom_tool_call"] + call_id: str + id: str | None = None + + +class _OutputToolSearch(_Frozen): + type: Literal["tool_search_call", "tool_search_output"] + + +_OutputItem: TypeAlias = Annotated[ + _OutputMessage | _OutputReasoning | _OutputToolCall | _OutputToolSearch, Field(discriminator="type") +] +_OUTPUT_ITEM: Final[TypeAdapter[_OutputItem]] = TypeAdapter(_OutputItem) + + +class _AnyMessage(_Frozen): + type: Literal["message"] + + +class _MessageEntry(_Frozen): + type: Literal["message"] + parts: tuple[Annotated[int, Field(ge=0)], ...] + phase: Phase | None = None + + +class _IndexedEntry(_Frozen): + type: Literal["reasoning", "function_call", "custom_tool_call", "tool_search_call", "tool_search_output"] + index: Annotated[int, Field(ge=0)] + + +_Entry: TypeAlias = Annotated[_MessageEntry | _IndexedEntry, Field(discriminator="type")] + + +class _Replay(_Frozen): + version: Literal[1] + items: tuple[_Entry, ...] + checksum: str + + +class _Namespaced(_Frozen): + namespace: str | None = None + + +class _ToolFunction(_Frozen): + name: str = "" + arguments: str = "" + provider_specific_fields: _Namespaced | None = None + + +class _ToolCall(_Frozen): + id: str + function: _ToolFunction + provider_specific_fields: _Namespaced | None = None + + +class _SummaryText(_Frozen): + type: str = "summary_text" + text: str = "" + + +class _Reasoning(_Frozen): + id: str | None = None + encrypted_content: str | None = None + summary: tuple[_SummaryText, ...] | None = None + + +class _ProviderFields(_Frozen): + phase: Phase | None = None + tool_search_items: tuple[Mapping[str, object], ...] | None = None + openai_responses_replay: object = None + + +class _ChatMessage(_Frozen): + role: str | None = None + content: str | None = None + tool_calls: tuple[_ToolCall, ...] | None = None + reasoning_items: tuple[_Reasoning, ...] | None = None + phase: Phase | None = None + provider_specific_fields: _ProviderFields | None = None + + +class _PhaseFields(_Frozen): + phase: Phase | None = None + + +class _RequestedPhaseFields(_Frozen): + phase: object = None + + +class _RequestedPhase(_Frozen): + phase: object = None + provider_specific_fields: _RequestedPhaseFields | None = None + + +@dataclass(frozen=True, slots=True) +class UnknownPhase: + value: object + + +_PHASE: Final[TypeAdapter[Phase]] = TypeAdapter(Phase) + + +class _RetainedFields(_PhaseFields): + openai_responses_replay: object = None + + +@dataclass(frozen=True, slots=True) +class ReplayedTurn: + items: tuple[Mapping[str, object], ...] + custom_call_ids: frozenset[str] + + +def _as_json(value: object) -> object: + return value.model_dump() if isinstance(value, BaseModel) else value + + +def _parsed_output_item(item: object) -> _OutputItem | None: + try: + return _OUTPUT_ITEM.validate_python(_as_json(item)) + except ValidationError: + return None + + +def _is_message(item: object) -> bool: + try: + _AnyMessage.model_validate(_as_json(item)) + except ValidationError: + return False + return True + + +def _chat_message(message: object) -> _ChatMessage | None: + try: + return _ChatMessage.model_validate(_as_json(message)) + except ValidationError: + return None + + +def requested_phase(message: object) -> Phase | UnknownPhase | None: + try: + requested: Final = _RequestedPhase.model_validate(_as_json(message)) + except ValidationError: + return None + fields: Final = requested.provider_specific_fields + value: Final = requested.phase or (fields.phase if fields else None) + if value is None: + return None + try: + return _PHASE.validate_python(value) + except ValidationError: + return UnknownPhase(value) + + +def reasoning_input_item( + item_id: str | None, summary: Sequence[object] | None, encrypted_content: str | None +) -> Mapping[str, object]: + return { + "type": "reasoning", + **({"id": item_id} if item_id else {}), + "summary": list(summary or ()), + **({"encrypted_content": encrypted_content} if encrypted_content else {}), + } + + +def retained_replay_fields(provider_specific_fields: object) -> Mapping[str, object] | None: + try: + retained: Final = _RetainedFields.model_validate(provider_specific_fields) + except ValidationError: + return None + return retained.model_dump(exclude_none=True) or None + + +def output_message_text(output_items: Sequence[object], *, first_message_only: bool) -> str: + parsed: Final = map(_parsed_output_item, output_items) + messages: Final = tuple(item for item in parsed if isinstance(item, _OutputMessage)) + return "".join(_texts(messages[:1] if first_message_only else messages)) + + +def _tool_search_entries(message: _ChatMessage) -> tuple[Mapping[str, object], ...]: + fields: Final = message.provider_specific_fields + items: Final = (fields.tool_search_items if fields else None) or () + return tuple(item for item in items if item.get("type") in ("tool_search_call", "tool_search_output")) + + +def _checksum(message: _ChatMessage, entries: Sequence[_MessageEntry | _IndexedEntry]) -> str: + fields: Final = message.provider_specific_fields + bound: Final = { + "content": message.content or "", + "tool_calls": [[call.id, call.function.name, call.function.arguments] for call in message.tool_calls or ()], + "reasoning": [ + [item.id, item.encrypted_content, [part.text for part in item.summary or ()]] + for item in message.reasoning_items or () + ], + "tool_search_items": list((fields.tool_search_items if fields else None) or ()), + "phase": message.phase or (fields.phase if fields else None), + "items": [entry.model_dump(mode="json", exclude_none=True) for entry in entries], + } + encoded: Final = json.dumps(bound, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + return hashlib.sha256(encoded.encode("utf-8", "surrogatepass")).hexdigest() + + +def _kind(item: _OutputItem) -> str: + match item: + case _OutputToolCall(): + return "tool_call" + case _OutputToolSearch(): + return "tool_search" + case _OutputMessage() | _OutputReasoning(): + return item.type + + +def _entry(items: Sequence[_OutputItem], position: int) -> _MessageEntry | _IndexedEntry: + item: Final = items[position] + if isinstance(item, _OutputMessage): + return _MessageEntry(type="message", parts=tuple(len(part.text) for part in item.content), phase=item.phase) + ordinal: Final = sum(1 for earlier in items[:position] if _kind(earlier) == _kind(item)) + return _IndexedEntry(type=item.type, index=ordinal) + + +def _texts(items: Sequence[_OutputItem]) -> Iterator[str]: + for item in items: + if isinstance(item, _OutputMessage): + yield from (part.text for part in item.content) + + +def _matches(items: Sequence[_OutputItem], message: _ChatMessage) -> bool: + reasoning_ids: Final = tuple(item.id for item in items if isinstance(item, _OutputReasoning)) + calls: Final = tuple(item for item in items if isinstance(item, _OutputToolCall)) + tool_calls: Final = message.tool_calls or () + searches: Final = sum(1 for item in items if isinstance(item, _OutputToolSearch)) + return ( + "".join(_texts(items)) == (message.content or "") + and reasoning_ids == tuple(item.id for item in message.reasoning_items or ()) + and len(calls) == len(tool_calls) + and all(chat.id in (call.call_id, call.id) for call, chat in zip(calls, tool_calls)) + and searches == len(_tool_search_entries(message)) + ) + + +def _content_phase( + output_items: Sequence[object], parsed: Sequence[_OutputItem | None], *, first_message_only: bool +) -> Phase | None: + messages: Final = tuple(item for raw, item in zip(output_items, parsed) if _is_message(raw)) + if not messages: + return None + content_message: Final = messages[0] if first_message_only else messages[-1] + return content_message.phase if isinstance(content_message, _OutputMessage) else None + + +def _with_phase(message: _ChatMessage, phase: Phase | None) -> _ChatMessage: + if phase is None: + return message + fields: Final = (message.provider_specific_fields or _ProviderFields()).model_copy(update={PHASE_FIELD: phase}) + return message.model_copy(update={"provider_specific_fields": fields}) + + +def replay_fields( + output_items: Sequence[object], message: object, *, completed: bool, first_message_only: bool +) -> Mapping[str, object]: + parsed: Final = tuple(map(_parsed_output_item, output_items)) + phase: Final = _content_phase(output_items, parsed, first_message_only=first_message_only) + phase_fields: Final[Mapping[str, object]] = MappingProxyType({PHASE_FIELD: phase} if phase else {}) + items: Final = tuple(item for item in parsed if item is not None) + parsed_message: Final = _chat_message(message) + chat: Final = _with_phase(parsed_message, phase) if parsed_message is not None else None + if not completed or not items or len(items) != len(parsed) or chat is None or not _matches(items, chat): + return phase_fields + entries: Final = tuple(_entry(items, position) for position in range(len(items))) + replay: Final = _Replay(version=1, items=entries, checksum=_checksum(chat, entries)) + return MappingProxyType({**phase_fields, REPLAY_FIELD: replay.model_dump(mode="json", exclude_none=True)}) + + +def _stored_replay(message: _ChatMessage) -> _Replay | None: + fields: Final = message.provider_specific_fields + if fields is None or fields.openai_responses_replay is None: + return None + try: + return _Replay.model_validate(fields.openai_responses_replay) + except ValidationError: + return None + + +def _message_texts(entries: Sequence[_MessageEntry], content: str) -> tuple[tuple[str, ...], ...]: + lengths: Final = tuple(chain.from_iterable(entry.parts for entry in entries)) + bounds: Final = tuple(accumulate(lengths, initial=0)) + parts: Final = tuple(content[start:end] for start, end in pairwise(bounds)) + groups: Final = tuple(accumulate((len(entry.parts) for entry in entries), initial=0)) + return tuple(parts[start:end] for start, end in pairwise(groups)) + + +def _message_item(texts: Sequence[str], phase: Phase | None) -> Mapping[str, object]: + return { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": text} for text in texts], + **({PHASE_FIELD: phase} if phase else {}), + } + + +def _function_call_item(call: _ToolCall) -> Mapping[str, object]: + namespaced: Final = call.provider_specific_fields or call.function.provider_specific_fields + namespace: Final = namespaced.namespace if namespaced else None + return { + "type": "function_call", + "call_id": call.id, + "name": call.function.name, + "arguments": call.function.arguments, + **({"namespace": namespace} if namespace else {}), + } + + +def _custom_tool_call_item(call: _ToolCall) -> Mapping[str, object]: + return { + "type": "custom_tool_call", + "call_id": call.id, + "name": call.function.name, + "input": call.function.arguments, + } + + +def _reasoning_item(item: _Reasoning) -> Mapping[str, object]: + summary: Final = tuple(part.model_dump() for part in item.summary or ()) + return reasoning_input_item(item.id, summary, item.encrypted_content) + + +@dataclass(frozen=True, slots=True) +class _Resolved: + message: _ChatMessage + texts: Mapping[int, tuple[str, ...]] + tool_searches: tuple[Mapping[str, object], ...] + + +def _input_item(resolved: _Resolved, position: int, entry: _MessageEntry | _IndexedEntry) -> Mapping[str, object]: + if isinstance(entry, _MessageEntry): + return _message_item(resolved.texts[position], entry.phase) + tool_calls: Final = resolved.message.tool_calls or () + match entry.type: + case "reasoning": + return _reasoning_item((resolved.message.reasoning_items or ())[entry.index]) + case "function_call": + return _function_call_item(tool_calls[entry.index]) + case "custom_tool_call": + return _custom_tool_call_item(tool_calls[entry.index]) + case "tool_search_call" | "tool_search_output": + return dict(resolved.tool_searches[entry.index]) + + +def _fits(message: _ChatMessage, entries: Sequence[_MessageEntry | _IndexedEntry]) -> bool: + tool_calls: Final = len(message.tool_calls or ()) + tool_searches: Final = len(_tool_search_entries(message)) + sizes: Final = MappingProxyType( + { + "reasoning": len(message.reasoning_items or ()), + "function_call": tool_calls, + "custom_tool_call": tool_calls, + "tool_search_call": tool_searches, + "tool_search_output": tool_searches, + } + ) + parts: Final = chain.from_iterable(entry.parts for entry in entries if isinstance(entry, _MessageEntry)) + indexed: Final = (entry for entry in entries if isinstance(entry, _IndexedEntry)) + return sum(parts) == len(message.content or "") and all(entry.index < sizes[entry.type] for entry in indexed) + + +def _resolve( + message: _ChatMessage, entries: Sequence[_MessageEntry | _IndexedEntry], replay_tool_search: bool +) -> ReplayedTurn | None: + indexed: Final = tuple(entry for entry in entries if isinstance(entry, _IndexedEntry)) + if not replay_tool_search and any(entry.type in ("tool_search_call", "tool_search_output") for entry in indexed): + return None + messages: Final = tuple(entry for entry in entries if isinstance(entry, _MessageEntry)) + calls: Final = tuple(entry for entry in indexed if entry.type in ("function_call", "custom_tool_call")) + texts: Final = _message_texts(messages, message.content or "") + positions: Final = tuple(position for position, entry in enumerate(entries) if isinstance(entry, _MessageEntry)) + resolved: Final = _Resolved( + message=message, + texts=MappingProxyType(dict(zip(positions, texts, strict=True))), + tool_searches=_tool_search_entries(message), + ) + tool_calls: Final = message.tool_calls or () + return ReplayedTurn( + items=tuple(_input_item(resolved, position, entry) for position, entry in enumerate(entries)), + custom_call_ids=frozenset(tool_calls[entry.index].id for entry in calls if entry.type == "custom_tool_call"), + ) + + +def replay_items(message: object, *, replay_tool_search: bool) -> ReplayedTurn | None: + chat: Final = _chat_message(message) + stored: Final = _stored_replay(chat) if chat is not None and chat.role == "assistant" else None + if chat is None or stored is None or stored.checksum != _checksum(chat, stored.items): + return None + if not _fits(chat, stored.items): + return None + return _resolve(chat, stored.items, replay_tool_search) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index ee3639ac387..580e55273f6 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -24,9 +24,19 @@ from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError import litellm from litellm import ModelResponse from litellm._logging import verbose_logger +from litellm.completion_extras.litellm_responses_transformation.history import ( + PHASE_FIELD, + UnknownPhase, + output_message_text, + reasoning_input_item, + replay_fields, + replay_items, + requested_phase, +) from litellm.integrations.anthropic_cache_control_hook import supports_openai_prompt_cache_breakpoint from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params from litellm.litellm_core_utils.prompt_templates.common_utils import ( + prompt_cache_breakpoint_for_wire, responses_reasoning_items_from_thinking_blocks, with_prompt_cache_breakpoint, ) @@ -52,7 +62,6 @@ from litellm.types.llms.openai import ( ChatCompletionToolCallChunk, ChatCompletionToolCallFunctionChunk, ChatCompletionToolParamFunctionChunk, - PromptCacheBreakpoint, Reasoning, ResponsesAPIOptionalRequestParams, ResponsesAPIResponse, @@ -490,7 +499,7 @@ def _fold_choices_into_one( choices: Sequence["Choices"], tool_calls: Sequence[Mapping[str, object]], trailing_reasoning_content: str | None, - trailing_reasoning_item: _BuiltReasoningItem | None, + trailing_reasoning_items: Sequence[_BuiltReasoningItem], tool_search_items: Sequence[Mapping[str, object]], structured_output: bool = False, ) -> "Choices": @@ -509,7 +518,7 @@ def _fold_choices_into_one( reasoning_contents: Final = (*(_message_reasoning_content(m) for m in messages), trailing_reasoning_content) reasoning_items: Final = ( *chain.from_iterable(_message_reasoning_items(m) for m in messages), - *(() if trailing_reasoning_item is None else (trailing_reasoning_item,)), + *trailing_reasoning_items, ) return Choices( message=Message( @@ -526,6 +535,61 @@ def _fold_choices_into_one( ) +def _with_replay_fields( + choice: "Choices", output_items: Sequence[object], *, completed: bool, structured_output: bool +) -> "Choices": + fields: Final = { + **(choice.message.provider_specific_fields or {}), + **replay_fields(output_items, choice.message, completed=completed, first_message_only=structured_output), + } + message: Final = choice.message.model_copy(update={"provider_specific_fields": fields or None}) + return choice.model_copy(update={"message": message}) + + +_STREAMED_OUTPUT_ITEMS: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter( + tuple[Mapping[str, object], ...] +) + + +def completed_stream_history( + output_payload: object, + tool_search_fields: Mapping[str, object] | None, + *, + completed: bool, + structured_output: bool, +) -> dict[str, object] | None: # mutable-ok: Delta.provider_specific_fields is typed as a dict + items: Final = _STREAMED_OUTPUT_ITEMS.validate_python(output_payload) + calls: Final = tuple(item for item in items if item.get("type") in ("function_call", "custom_tool_call")) + streamed_message: Final = { + "role": "assistant", + "content": output_message_text(items, first_message_only=structured_output), + "tool_calls": [tool_call_dict_from_output_item(call, index) for index, call in enumerate(calls)], + "reasoning_items": list(_reasoning_items_from_output_items(items)), + "provider_specific_fields": tool_search_fields, + } + fields: Final = { + **(tool_search_fields or {}), + **replay_fields(items, streamed_message, completed=completed, first_message_only=structured_output), + } + return fields or None + + +def _assistant_phase_field(msg: object, replay_openai_history: bool, drop_params: bool) -> Mapping[str, object]: + match requested_phase(msg) if replay_openai_history else None: + case None: + return {} + case UnknownPhase(value=value) if drop_params: + verbose_logger.debug("Chat provider: dropping assistant phase=%r under drop_params", value) + return {} + case UnknownPhase(value=value): + raise litellm.UnsupportedParamsError( + message=f"phase must be 'commentary' or 'final_answer', got {value!r}. " + "To drop it, set `litellm.drop_params=True` or pass `drop_params=True`." + ) + case phase: + return {PHASE_FIELD: phase} + + def _responses_event_type(chunk: Mapping[str, object]) -> object: event_type: Final = chunk.get("type") return event_type.value if isinstance(event_type, ResponsesAPIStreamEvents) else event_type @@ -567,19 +631,6 @@ def _map_incomplete_reason_to_finish_reason(incomplete_reason: str | None) -> Li return "length" -_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(PromptCacheBreakpoint) - - -def _prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object: - if marker is None or not drop_params: - return marker - try: - return _PROMPT_CACHE_BREAKPOINT.validate_python(marker) - except ValidationError: - verbose_logger.debug("Chat provider: dropping malformed prompt_cache_breakpoint %r under drop_params", marker) - return None - - def _input_file_from_file_value(file_value: object) -> dict[str, object]: if not isinstance(file_value, dict): return {"type": "input_file"} @@ -667,15 +718,7 @@ def _reasoning_item_to_response_input( An item without an id is sent without one: the Responses API accepts that and verifies the encrypted content on its own, while it rejects any id it did not mint. """ - item_id: Final = r_item.get("id") - encrypted_content: Final = r_item.get("encrypted_content") - return { - "type": "reasoning", - **({"id": item_id} if item_id else {}), - # summary is always required by the Responses API, even when empty - "summary": r_item.get("summary") or [], - **({"encrypted_content": encrypted_content} if encrypted_content else {}), - } + return dict(reasoning_input_item(r_item.get("id"), r_item.get("summary"), r_item.get("encrypted_content"))) def _requested_defer_loading(tool: Mapping[str, object], function_tool: Mapping[str, object]) -> object: @@ -804,12 +847,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): keep_prompt_cache_breakpoints: bool = False, replay_tool_search: bool = False, include_namespace: bool = False, + replay_openai_history: bool = False, ) -> tuple[list[object], str | None]: converted_input_items, instructions = self._convert_chat_completion_messages_to_responses_input( messages, drop_params=drop_params, replay_tool_search=replay_tool_search, include_namespace=include_namespace, + replay_openai_history=replay_openai_history, ) return ( converted_input_items @@ -825,10 +870,18 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): drop_params: bool = False, replay_tool_search: bool = False, include_namespace: bool = False, + replay_openai_history: bool = False, ) -> tuple[list[object], str | None]: input_items: Final[list[object]] = [] instructions: str | None = None - custom_tool_call_ids: Final = frozenset( + replayed_turns: Final = tuple( + replay_items(msg, replay_tool_search=replay_tool_search) if replay_openai_history else None + for msg in messages + ) + replayed_custom_call_ids: Final = frozenset( + chain.from_iterable(turn.custom_call_ids for turn in replayed_turns if turn is not None) + ) + custom_tool_call_ids: Final = replayed_custom_call_ids | frozenset( tool_call["id"] for msg in messages if msg.get("role") == "assistant" and isinstance(msg.get("tool_calls"), list) @@ -849,7 +902,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): tool_calls = msg.get("tool_calls") tool_call_id = msg.get("tool_call_id") - if role == "system": + if (replayed_turn := replayed_turns[index]) is not None: + input_items.extend(replayed_turn.items) + elif role == "system": # Extract system message as instructions if isinstance(content, str) and index < leading_system_count: if instructions: @@ -915,6 +970,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): "content": self._convert_content_to_responses_format( content, "assistant", drop_params=drop_params ), + **_assistant_phase_field(msg, replay_openai_history, drop_params), } ) for tool_call in tool_calls: @@ -955,6 +1011,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): "content": self._convert_content_to_responses_format( content, cast(str, role), drop_params=drop_params ), + **_assistant_phase_field(msg, replay_openai_history and role == "assistant", drop_params), } ) elif role == "assistant": @@ -1081,6 +1138,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): keep_prompt_cache_breakpoints=supports_prompt_cache_breakpoint, replay_tool_search=_requests_tool_search(cast(object, optional_params)), # cast-ok: upcast include_namespace=True, + replay_openai_history=provider == "openai", ) # OpenAI's Responses API rejects an empty input. For a system-only # request, carry the system message as a system-role input item instead @@ -1165,6 +1223,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): output_items: Sequence[object], handle_raw_dict_callback: Callable[..., tuple["Choices | None", int]] | None = None, structured_output: bool = False, + replay_openai_history: bool = False, + completed: bool = False, ) -> list["Choices"]: """ Convert Responses API output items to chat completion choices. @@ -1174,6 +1234,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): handle_raw_dict_callback: Optional callback for handling raw dict items structured_output: The chat request carried a json response_format, so only the first message item becomes content + replay_openai_history: Record the output's item sequence for the next turn + completed: The response finished, so its item sequence is whole Returns: List of Choices objects @@ -1196,7 +1258,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): choices: Final[list[Choices]] = [] index = 0 reasoning_content: str | None = None - pending_reasoning_item: _BuiltReasoningItem | None = None + pending_reasoning_items: tuple[_BuiltReasoningItem, ...] = () # Collect all tool calls to put them in a single choice # (Chat Completions API expects all tool calls in one message) @@ -1205,12 +1267,19 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): for item in output_items: if isinstance(item, ResponseReasoningItem): - pending_reasoning_item = _build_reasoning_item( - item_id=item.id, - encrypted_content=getattr(item, "encrypted_content", None), - summary_raw=item.summary, + pending_reasoning_items = ( + *(pending_reasoning_items if replay_openai_history else ()), + _build_reasoning_item( + item_id=item.id, + encrypted_content=getattr(item, "encrypted_content", None), + summary_raw=item.summary, + ), + ) + reasoning_content = " ".join( + s["text"] + for s in chain.from_iterable(built["summary"] for built in pending_reasoning_items) + if s.get("text") ) - reasoning_content = " ".join(s["text"] for s in pending_reasoning_item["summary"] if s.get("text")) elif isinstance(item, ResponseOutputMessage): for content in item.content: @@ -1227,7 +1296,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): annotations=annotations, reasoning_items=cast( list[ChatCompletionReasoningItem] | None, - ([pending_reasoning_item] if pending_reasoning_item is not None else None), + (list(pending_reasoning_items) or None), ), ) @@ -1240,7 +1309,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ) reasoning_content = None # flush - pending_reasoning_item = None # flush + pending_reasoning_items = () # flush index += 1 elif isinstance(item, ResponseFunctionToolCall): @@ -1300,16 +1369,17 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): output_items, "stop", tool_search_items=tool_search_items ) ] - return [ - _fold_choices_into_one( - choices=choices, - tool_calls=accumulated_tool_calls, - trailing_reasoning_content=reasoning_content, - trailing_reasoning_item=pending_reasoning_item, - tool_search_items=tool_search_items, - structured_output=structured_output, - ) - ] + folded: Final = _fold_choices_into_one( + choices=choices, + tool_calls=accumulated_tool_calls, + trailing_reasoning_content=reasoning_content, + trailing_reasoning_items=pending_reasoning_items, + tool_search_items=tool_search_items, + structured_output=structured_output, + ) + if not replay_openai_history: + return [folded] + return [_with_replay_fields(folded, output_items, completed=completed, structured_output=structured_output)] @staticmethod def _build_empty_choice( @@ -1443,6 +1513,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): structured_output=is_structured_output_request( cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict ), + replay_openai_history=_served_provider( + cast("Mapping[str, object]", litellm_params) # cast-ok: litellm_params is an untyped dict + ) + == "openai", + completed=raw_response.status == "completed", ) response_is_incomplete: Final = raw_response.status == "incomplete" or ( @@ -1506,9 +1581,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): sync_stream: bool, json_mode: bool | None = False, structured_output: bool = False, + replays_openai_history: bool = False, ) -> BaseModelResponseIterator: return OpenAiResponsesToChatCompletionStreamIterator( - streaming_response, sync_stream, json_mode, structured_output=structured_output + streaming_response, + sync_stream, + json_mode, + structured_output=structured_output, + replays_openai_history=replays_openai_history, ) def _convert_content_str_to_input_text(self, content: str, role: str) -> dict[str, object]: @@ -1586,7 +1666,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): if original_type == "text": converted = with_prompt_cache_breakpoint( self._convert_content_str_to_input_text(item.get("text", ""), role), - _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), + prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), ) result.append(converted) verbose_logger.debug("Chat provider: text -> %s", converted) @@ -1599,14 +1679,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): cast(ChatCompletionImageObject, item), role ), ), - _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), + prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), ) result.append(converted) verbose_logger.debug("Chat provider: image_url -> %s", converted) elif original_type == "input_audio": converted = with_prompt_cache_breakpoint( {"type": "input_audio", "input_audio": item.get("input_audio")}, - _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), + prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), ) result.append(converted) verbose_logger.debug("Chat provider: input_audio -> %s", converted) @@ -1622,7 +1702,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): _input_file_from_file_value( cast("ChatCompletionFileObject", item).get("file"), # cast-ok: type tag checked ), - _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), + prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), ) result.append(converted) verbose_logger.debug("Chat provider: file -> %s", converted) @@ -1646,7 +1726,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): # Default to input_text for unknown types converted = with_prompt_cache_breakpoint( self._convert_content_str_to_input_text(str(item.get("text", item)), role), - _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), + prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), ) result.append(converted) verbose_logger.debug("Chat provider: unknown(%s) -> %s", original_type, converted) @@ -1890,8 +1970,10 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): sync_stream: bool, json_mode: bool | None = False, structured_output: bool = False, + replays_openai_history: bool = False, ): super().__init__(streaming_response, sync_stream, json_mode) + self._replays_openai_history: Final = replays_openai_history self._chat_completion_id: str | None = None self._served_service_tier: str | None = None self._tool_call_index_map: dict[int, int] = {} # mutable-ok: per-stream accumulator state @@ -1937,6 +2019,8 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): def translate_responses_chunk_to_openai_stream( parsed_chunk: dict | BaseModel, tool_call_index_map: dict[int, int] | None = None, # mutable-ok: per-stream state, remapped in place + replays_openai_history: bool = False, + structured_output: bool = False, ) -> "ModelResponseStream": """ Translate a Responses API streaming chunk to OpenAI chat completion streaming format. @@ -1944,6 +2028,8 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): Args: parsed_chunk: Dict containing the Responses API event chunk tool_call_index_map: Per-stream output_index -> sequential tool_call index map + replays_openai_history: Record the completed output's item sequence for the next turn + structured_output: Only the first message item became content Returns: ModelResponseStream: OpenAI-formatted streaming chunk @@ -2154,6 +2240,10 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): terminal_reasoning_items_typed: Final = _as_chat_reasoning_items( _reasoning_items_from_output_items(output_items) ) + output_payload: Final = cast(object, output_items) # cast-ok: untyped + tool_search_fields: Final = _tool_search_provider_fields( + _tool_search_items_from_output_items(output_payload) + ) usage = None if response_data.get("usage"): @@ -2167,10 +2257,17 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): StreamingChoices( index=0, delta=Delta( - content="", + content=None if replays_openai_history else "", reasoning_items=terminal_reasoning_items_typed, - provider_specific_fields=_tool_search_provider_fields( - _tool_search_items_from_output_items(cast(object, output_items)) # cast-ok: untyped + provider_specific_fields=( + completed_stream_history( + output_payload, + tool_search_fields, + completed=event_type == "response.completed", + structured_output=structured_output, + ) + if replays_openai_history + else tool_search_fields ), ), finish_reason=finish_reason, @@ -2216,7 +2313,10 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): self._empty_content_chunk() if self._is_text_delta_after_structured_answer(chunk) else OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream( - chunk, tool_call_index_map=self._tool_call_index_map + chunk, + tool_call_index_map=self._tool_call_index_map, + replays_openai_history=self._replays_openai_history, + structured_output=self._structured_output, ) ) if self._structured_output and _is_message_item_done(chunk): diff --git a/litellm/constants.py b/litellm/constants.py index 25770d960bb..d02114f56b1 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -158,7 +158,7 @@ DEFAULT_REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_int("DEFAULT_REPLICATE_ DEFAULT_IMAGE_TOKEN_COUNT: Final = get_env_int("DEFAULT_IMAGE_TOKEN_COUNT", 250) HF_CONFIG_FETCH_TIMEOUT_SECONDS: Final = 10.0 HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS: Final = get_env_int_in_range( - "HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS", 300, 1, 86400 + "HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS", 60, 1, 86400 ) # Maximum wall-clock seconds a streaming response is allowed to run. @@ -627,6 +627,7 @@ STS_CREDENTIAL_EXPIRY_SAFETY_MARGIN_SECONDS: Final = 60 BEDROCK_MIN_THINKING_BUDGET_TOKENS: Final = get_env_int("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024) # Anthropic's Messages API rejects thinking.budget_tokens < 1024. ANTHROPIC_MIN_THINKING_BUDGET_TOKENS: Final = 1024 +ANTHROPIC_BILLING_METADATA_PREFIX: Final = "x-anthropic-billing-header:" REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_float("REPLICATE_POLLING_DELAY_SECONDS", 0.5) DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS: Final = get_env_int("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 128000) DEFAULT_OCI_CHAT_MAX_TOKENS: Final = 4096 @@ -1064,6 +1065,7 @@ openai_compatible_endpoints: Final[list[str]] = [ "https://api.reka.ai/v1", "https://gigachat.devices.sberbank.ru/api/v1", "https://api.openinfer.ai/v1", + "https://vanchin.streamlake.ai/api/gateway/v1/endpoints", ] @@ -1145,6 +1147,7 @@ openai_compatible_providers: Final[list[str]] = [ "prism", "sail", "reka", + "streamlake", ] OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS: Final = frozenset({"openai"} | frozenset(openai_compatible_providers)) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 087d5b5b718..a4a9a5bee39 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -22,7 +22,7 @@ from litellm.constants import ( OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES, REALTIME_METERED_USAGE_KEY, ) -from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.core_helpers import as_str_mapping, billed_usage_cost from litellm.litellm_core_utils.cost_discount import resolve_cost_discount from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, is_decisions_call_type from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR @@ -50,6 +50,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( parse_prompt_tokens_details, select_cost_metric_for_model, ) +from litellm.litellm_core_utils.llm_response_utils.response_metadata import response_ms_since_logging_start from litellm.llms.anthropic.cost_calculation import ( cost_per_token as anthropic_cost_per_token, ) @@ -688,7 +689,7 @@ def cost_per_token( elif call_type == "atranscription" or call_type == "transcription": if _transcription_usage_has_token_details(usage_block): return generic_cost_per_token( - model=model_without_prefix, + model=model, usage=usage_block, custom_llm_provider=custom_llm_provider, service_tier=service_tier, @@ -1396,9 +1397,9 @@ def _apply_cost_margin( def _store_cost_breakdown_in_logging_obj( litellm_logging_obj: LitellmLoggingObject | None, - prompt_tokens_cost_usd_dollar: float, - completion_tokens_cost_usd_dollar: float, - cost_for_built_in_tools_cost_usd_dollar: float, + prompt_tokens_cost_usd_dollar: float | None, + completion_tokens_cost_usd_dollar: float | None, + cost_for_built_in_tools_cost_usd_dollar: float | None, total_cost_usd_dollar: float, additional_costs: dict | None = None, original_cost: float | None = None, @@ -1500,9 +1501,45 @@ def _response_time_ms_for_cost( return float(stamped) if total_time: return total_time - if litellm_logging_obj is not None: - return litellm_logging_obj.get_response_ms() - return 0.0 + if litellm_logging_obj is None: + return 0.0 + reported_ms: Final = response_ms_since_logging_start(litellm_logging_obj) + if reported_ms is not None and litellm_logging_obj.stream is not True: + return reported_ms + return litellm_logging_obj.get_response_ms() + + +def _adjusted_provider_billed_cost( + reported_cost: float, + custom_llm_provider: str | None, + model: str | None, + region_name: str | None, + litellm_logging_obj: LitellmLoggingObject | None, + service_tier: str | None, + data_residency: str | None, +) -> float: + discounted_cost, discount_percent, discount_amount = _apply_cost_discount( + base_cost=reported_cost, custom_llm_provider=custom_llm_provider, model=model, region_name=region_name + ) + adjusted_cost, margin_percent, margin_fixed_amount, margin_total_amount = _apply_cost_margin( + base_cost=discounted_cost, custom_llm_provider=custom_llm_provider + ) + _store_cost_breakdown_in_logging_obj( + litellm_logging_obj=litellm_logging_obj, + prompt_tokens_cost_usd_dollar=None, + completion_tokens_cost_usd_dollar=None, + cost_for_built_in_tools_cost_usd_dollar=None, + total_cost_usd_dollar=adjusted_cost, + original_cost=reported_cost, + discount_percent=discount_percent, + discount_amount=discount_amount, + margin_percent=margin_percent, + margin_fixed_amount=margin_fixed_amount, + margin_total_amount=margin_total_amount, + service_tier=service_tier, + data_residency=data_residency, + ) + return adjusted_cost def completion_cost( @@ -1616,6 +1653,9 @@ def completion_cost( ) explicit_pricing: Final = custom_pricing is True or base_model is not None + caller_set_prices: Final = ( + explicit_pricing or custom_cost_per_token is not None or custom_cost_per_second is not None + ) custom_priced_request_model: Final = model if custom_pricing is True else None configured_cost_per_query: Final = ( _configured_search_cost_per_query(litellm_logging_obj) if custom_pricing else None @@ -1666,6 +1706,7 @@ def completion_cost( raw_candidates if custom_pricing is True else _priced_candidates_before_unpriced_rules(raw_candidates) ) + priced_response: Final[object] = completion_response for idx, (candidate_model, candidate_pricing_provider) in enumerate(potential_model_names): model = candidate_model try: @@ -1984,6 +2025,19 @@ def completion_cost( total_time=total_time, ) model_for_discount = custom_priced_request_model or model + if ( + not caller_set_prices + and (reported_cost := billed_usage_cost(priced_response, custom_llm_provider)) is not None + ): + return _adjusted_provider_billed_cost( + reported_cost=reported_cost, + custom_llm_provider=custom_llm_provider, + model=model_for_discount, + region_name=region_name if isinstance(region_name, str) else None, + litellm_logging_obj=litellm_logging_obj, + service_tier=service_tier, + data_residency=data_residency, + ) # Calculate cost based on prompt_tokens, completion_tokens if ( "togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai" diff --git a/litellm/google_genai/adapters/transformation.py b/litellm/google_genai/adapters/transformation.py index e1085dd7857..c1895376675 100644 --- a/litellm/google_genai/adapters/transformation.py +++ b/litellm/google_genai/adapters/transformation.py @@ -198,6 +198,11 @@ def _genai_message_part(part: object) -> _GenAIMessagePart | None: return ChatCompletionTextObject(type="text", text=text) if isinstance(text, str) else _genai_media_part(fields) +def _genai_content_parts(value: object) -> Sequence[object]: + single_part: Final = _validated(_CONFIG_FIELDS, value) + return (single_part,) if single_part is not None else _validated(_GENAI_CONTENT_PARTS, value) or () + + def _genai_message_parts(parts: object) -> tuple[_GenAIMessagePart, ...]: items: Final = _validated(_GENAI_CONTENT_PARTS, parts) or () return tuple(converted for part in items if (converted := _genai_message_part(part)) is not None) @@ -248,7 +253,7 @@ def _genai_content_messages(content: object) -> tuple[AllMessageValues, ...]: fields: Final = _validated(_CONFIG_FIELDS, content) if fields is None: return () - parts: Final = _validated(_GENAI_CONTENT_PARTS, fields.get("parts", ())) or () + parts: Final = _genai_content_parts(fields.get("parts", ())) converted: Final = _genai_message_parts(parts) role: Final = fields.get("role", "user") if role == "user": diff --git a/litellm/images/main.py b/litellm/images/main.py index 6e77cbb7046..b3247ff938b 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -641,6 +641,8 @@ def image_generation( timeout=timeout, client=custom_client, ) + elif custom_llm_provider == LlmProviders.KYMA.value: + raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider) return model_response except Exception as e: diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index b0deec24c1a..fc1f1830129 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -72,6 +72,8 @@ MAX_CACHE_CONTROL_BLOCKS: Final = 4 CACHE_BREAKPOINT_KEYS: Final = ("cache_control", "prompt_cache_breakpoint") OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION: Final = (5, 6) _GPT_VERSION_PATTERN: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?") +_FINE_TUNED_BASE_PATTERN: Final = re.compile(r"^ft:([^:]+)") +_O_SERIES_PATTERN: Final = re.compile(r"^o\d+(?:-|$)") OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES: Final = frozenset( {"text", "image", "image_url", "file", "input_audio", "input_text", "input_image", "input_file"} ) @@ -125,6 +127,10 @@ def _served_provider_for_model(model: str) -> str | None: def supports_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None = None) -> bool: + return openai_prompt_cache_breakpoint_support(model, custom_llm_provider) is True + + +def openai_prompt_cache_breakpoint_support(model: str, custom_llm_provider: str | None = None) -> bool | None: hosted_flag: Final = ( None if custom_llm_provider is None @@ -135,9 +141,19 @@ def supports_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str model_map_flag: Final = _model_map_prompt_cache_breakpoint_flag(model) if model_map_flag is not None: return model_map_flag - version_match: Final = _GPT_VERSION_PATTERN.match(model.rsplit("/", 1)[-1].lower()) - if version_match is None: + fine_tuned_base: Final = _FINE_TUNED_BASE_PATTERN.match(model.rsplit("/", 1)[-1]) + if fine_tuned_base is not None: + return False if _named_model_breakpoint_support(fine_tuned_base.group(1)) is False else None + return _named_model_breakpoint_support(model) + + +def _named_model_breakpoint_support(model: str) -> bool | None: + bare_model: Final = model.rsplit("/", 1)[-1].lower() + if _O_SERIES_PATTERN.match(bare_model) is not None: return False + version_match: Final = _GPT_VERSION_PATTERN.match(bare_model) + if version_match is None: + return None version: Final = (int(version_match.group(1)), int(version_match.group(2) or 0)) return version >= OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION diff --git a/litellm/integrations/datadog/datadog_cost_management.py b/litellm/integrations/datadog/datadog_cost_management.py index 1a81fa019dc..d4b75fc0bce 100644 --- a/litellm/integrations/datadog/datadog_cost_management.py +++ b/litellm/integrations/datadog/datadog_cost_management.py @@ -205,11 +205,12 @@ class DatadogCostManagementLogger(CustomBatchLogger): # see _RESERVED_TAG_KEYS for the rationale. if self.cost_tag_keys: allow: Final = set(self.cost_tag_keys) + client_allow: Final = allow - {"is_silent_experiment"} for rt in log.get("request_tags") or []: if not isinstance(rt, str) or ":" not in rt: continue k, _, v = rt.partition(":") - if k in allow and v: + if k in client_allow and v: self._set_custom_tag(tags, k, v) for k, v in metadata.items(): if k in allow and v is not None and not isinstance(v, (dict, list)): @@ -218,7 +219,7 @@ class DatadogCostManagementLogger(CustomBatchLogger): nested = metadata.get(nested_key) if isinstance(nested, dict): for k, v in nested.items(): - if k in allow and v is not None and not isinstance(v, (dict, list)): + if k in client_allow and v is not None and not isinstance(v, (dict, list)): self._set_custom_tag(tags, k, str(v)) return tags diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index b8c6e91d0d7..bd68ffe8cd6 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -833,6 +833,12 @@ class LangFuseLogger: ("vertex_location", vertex_location, bool(vertex_location)), ("aws_region_name", aws_region_name, bool(aws_region_name)), ("cache_hit", kwargs.get("cache_hit") or False, "cache_hit" in kwargs), + ( + "is_silent_experiment", + "is_silent_experiment" in allowlisted_metadata + and allowlisted_metadata["is_silent_experiment"] is True, + "is_silent_experiment" in allowlisted_metadata, + ), ) enrichments: Final[Mapping[str, object]] = { key: value for key, value, include in candidate_enrichments if include diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index bc8344fa686..e62b99f8b0f 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -1173,8 +1173,8 @@ class PrometheusLogger(CustomLogger): # Build label filters from valid configurations label_filters: Final = self._build_label_filters(parsed_configs) - # Pretty print the processed configuration - self._pretty_print_prometheus_config(label_filters) + if litellm.suppress_startup_output is not True: + self._pretty_print_prometheus_config(label_filters) return label_filters def _parse_exclude_config(self) -> tuple[frozenset[str], frozenset[str]]: @@ -5148,6 +5148,12 @@ def _get_combined_custom_metadata_from_standard_logging_payload( **(requester_metadata if isinstance(requester_metadata, dict) else {}), **(user_api_key_auth_metadata if isinstance(user_api_key_auth_metadata, dict) else {}), **(spend_logs_metadata if isinstance(spend_logs_metadata, dict) else {}), + "is_silent_experiment": ( + "true" + if "is_silent_experiment" in standard_logging_metadata + and standard_logging_metadata["is_silent_experiment"] is True + else None + ), } diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 6e1ba3eea08..c25fcfe3e70 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -37,6 +37,7 @@ from litellm.integrations.websearch_interception.transformation import ( from litellm.litellm_core_utils.agentic_loop_settings import ( validated_max_agentic_loops, ) +from litellm.litellm_core_utils.core_helpers import qualify_provider_stripped_model from litellm.llms.base_llm.search.transformation import SearchResponse from litellm.types.integrations.custom_logger import ( CHAT_COMPLETION_AGENTIC_SURFACE, @@ -114,6 +115,18 @@ class _WebSearchSettingsView(TypedDict): websearch_interception_params: WebSearchInterceptionConfig +def _caller_model(model: str, logging_obj: "LiteLLMLoggingObj | None", kwargs: Mapping[str, object]) -> str: + agentic_view: Final[_AgenticLoopParamsView] = { + "agentic_loop_params": logging_obj.model_call_details.get("agentic_loop_params", {}) if logging_obj else {} + } + recorded_model: Final = agentic_view["agentic_loop_params"].get("model") + custom_llm_provider: Final = kwargs.get("custom_llm_provider") + return qualify_provider_stripped_model( + recorded_model if isinstance(recorded_model, str) and recorded_model else model, + custom_llm_provider if isinstance(custom_llm_provider, str) else "", + ) + + class _SearchToolLitellmParams(TypedDict, total=False): search_provider: ReadOnly[str | None] @@ -1155,6 +1168,7 @@ class WebSearchInterceptionLogger(CustomLogger): tool_calls=tool_calls, optional_params=optional_params, kwargs=kwargs, + logging_obj=logging_obj, response_format=response_format, ) return AgenticLoopPlan( @@ -1181,6 +1195,7 @@ class WebSearchInterceptionLogger(CustomLogger): tool_calls=tool_calls, optional_params=optional_params, kwargs=kwargs, + logging_obj=logging_obj, ) return AgenticLoopPlan( run_agentic_loop=True, @@ -1195,6 +1210,7 @@ class WebSearchInterceptionLogger(CustomLogger): tool_calls: list[dict], optional_params: dict, kwargs: dict, + logging_obj: "LiteLLMLoggingObj | None", ) -> AgenticLoopRequestPatch: """Execute litellm.asearch() and build a Responses API rerun patch.""" search_tasks: Final = [ @@ -1254,9 +1270,7 @@ class WebSearchInterceptionLogger(CustomLogger): } } - full_model_name = model - if "/" not in model and isinstance(kwargs.get("custom_llm_provider"), str): - full_model_name = f"{kwargs['custom_llm_provider']}/{model}" + full_model_name: Final = _caller_model(model, logging_obj, kwargs) verbose_logger.debug( "WebSearchInterception: Built responses request patch model=%s input_items=%d searches=%d", @@ -1814,6 +1828,7 @@ class WebSearchInterceptionLogger(CustomLogger): tool_calls=tool_calls, optional_params=dict[str, object](optional_params), kwargs=dict[str, object](kwargs), + logging_obj=logging_obj, response_format=response_format, ) if request_patch.messages is None: @@ -1836,7 +1851,8 @@ class WebSearchInterceptionLogger(CustomLogger): messages: list[dict], tool_calls: list[dict], optional_params: dict, - kwargs: dict, + kwargs: dict[str, object], + logging_obj: "LiteLLMLoggingObj | None", response_format: str = "openai", ) -> AgenticLoopRequestPatch: """Execute litellm.search() and build chat-completion rerun patch.""" @@ -1922,11 +1938,7 @@ class WebSearchInterceptionLogger(CustomLogger): k: v for k, v in kwargs.items() if not k.startswith("_websearch_interception") and k not in internal_params } - full_model_name = model - if "custom_llm_provider" in kwargs: - custom_llm_provider: Final = kwargs["custom_llm_provider"] - if not model.startswith(custom_llm_provider) and "/" not in model: - full_model_name = f"{custom_llm_provider}/{model}" + full_model_name: Final = _caller_model(model, logging_obj, kwargs) verbose_logger.debug( "WebSearchInterception: Built chat completion request patch model=%s messages=%d", diff --git a/litellm/interactions/litellm_responses_transformation/handler.py b/litellm/interactions/litellm_responses_transformation/handler.py index d84f0b5ffbd..f95b5611831 100644 --- a/litellm/interactions/litellm_responses_transformation/handler.py +++ b/litellm/interactions/litellm_responses_transformation/handler.py @@ -103,6 +103,7 @@ class LiteLLMResponsesInteractionsHandler: return LiteLLMResponsesInteractionsConfig.transform_responses_response_to_interactions_response( responses_response=responses_api_response, model=model, + custom_llm_provider=custom_llm_provider, ) async def async_interactions_api_handler( @@ -138,4 +139,5 @@ class LiteLLMResponsesInteractionsHandler: return LiteLLMResponsesInteractionsConfig.transform_responses_response_to_interactions_response( responses_response=responses_api_response, model=model, + custom_llm_provider=responses_request.get("custom_llm_provider"), ) diff --git a/litellm/interactions/litellm_responses_transformation/transformation.py b/litellm/interactions/litellm_responses_transformation/transformation.py index ade33831cdc..2e9b62dcfe2 100644 --- a/litellm/interactions/litellm_responses_transformation/transformation.py +++ b/litellm/interactions/litellm_responses_transformation/transformation.py @@ -12,6 +12,7 @@ from typing import Final, cast from pydantic import BaseModel +from litellm.litellm_core_utils.core_helpers import billed_usage_cost from litellm.types.interactions import ( InteractionInput, InteractionsAPIOptionalRequestParams, @@ -190,6 +191,7 @@ class LiteLLMResponsesInteractionsConfig: def transform_responses_response_to_interactions_response( responses_response: ResponsesAPIResponse, model: str | None = None, + custom_llm_provider: str | None = None, ) -> InteractionsAPIResponse: """ Transform a Responses API response to an Interactions API response. @@ -266,9 +268,11 @@ class LiteLLMResponsesInteractionsConfig: # (total_input_tokens, total_output_tokens) usage: Final[object] = getattr(responses_response, "usage", None) if usage: + reported_cost: Final = billed_usage_cost(responses_response, custom_llm_provider) interactions_response_dict["usage"] = { "total_input_tokens": getattr(usage, "input_tokens", 0), "total_output_tokens": getattr(usage, "output_tokens", 0), + **({} if reported_cost is None else {"cost": reported_cost}), } # Add updated (same as created for now) diff --git a/litellm/litellm_core_utils/chat_completion_agentic_loop.py b/litellm/litellm_core_utils/chat_completion_agentic_loop.py index 628486f377f..2ba47cbb400 100644 --- a/litellm/litellm_core_utils/chat_completion_agentic_loop.py +++ b/litellm/litellm_core_utils/chat_completion_agentic_loop.py @@ -15,6 +15,7 @@ from litellm.litellm_core_utils.agentic_loop_settings import ( DEFAULT_MAX_AGENTIC_LOOPS, validated_max_agentic_loops, ) +from litellm.litellm_core_utils.core_helpers import qualify_agentic_followup_model from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject from litellm.llms.base_llm.base_model_iterator import MockResponseIterator from litellm.types.integrations.custom_logger import ( @@ -182,9 +183,7 @@ async def _execute_chat_completion_agentic_plan( if patch.messages is None: raise ValueError("Agentic loop plan missing patched messages") - full_model_name = patch.model or model - if "/" not in full_model_name: - full_model_name = f"{custom_llm_provider}/{full_model_name}" + full_model_name: Final = qualify_agentic_followup_model(patch.model, model, custom_llm_provider) optional_params_for_followup: Final = {**optional_params, **patch.optional_params} if patch.tools is not None: diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 55025508e40..d940baf8b56 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -2,6 +2,7 @@ ## Helper utilities import copy import logging +import math import re from collections.abc import Collection, Iterable, Iterator, Mapping from types import MappingProxyType @@ -34,6 +35,49 @@ def is_codex_user_agent(user_agent: str) -> bool: return bool(_CODEX_CLIENT_PREFIX_RE.match(user_agent)) +def qualify_provider_stripped_model(model: str, custom_llm_provider: str) -> str: + """Put the provider prefix back on a provider-stripped model.""" + if not custom_llm_provider or model.startswith(f"{custom_llm_provider}/"): + return model + if _already_routes_to(model, custom_llm_provider): + return model + return f"{custom_llm_provider}/{model}" + + +def _already_routes_to(model: str, custom_llm_provider: str) -> bool: + from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider + + return _leads_with_provider(model) and inferred_provider(model) == custom_llm_provider + + +def _leads_with_provider(model: str) -> bool: + leading_segment, separator, _ = model.partition("/") + return bool(separator) and names_a_provider(leading_segment) + + +def names_a_provider(candidate: str) -> bool: + """Whether a model name's leading segment is a provider rather than part of the model.""" + import litellm + from litellm.litellm_core_utils.get_llm_provider_logic import is_registered_custom_provider + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + provider_names: Final = {getattr(provider, "value", provider) for provider in litellm.provider_list} + return ( + candidate in provider_names + or JSONProviderRegistry.exists(candidate) + or is_registered_custom_provider(candidate) + ) + + +def qualify_agentic_followup_model(patch_model: str | None, model: str, custom_llm_provider: str) -> str: + """Resolve the model an agentic follow-up re-dispatches, keeping a hook's own provider.""" + if not patch_model or patch_model == model: + return qualify_provider_stripped_model(model, custom_llm_provider) + if _leads_with_provider(patch_model): + return patch_model + return qualify_provider_stripped_model(patch_model, custom_llm_provider) + + def safe_divide_seconds(seconds: float, denominator: float, default: float | None = None) -> float | None: """ Safely divide seconds by denominator, handling zero division. @@ -973,6 +1017,16 @@ def chunk_usage(chunk: object) -> object: return getattr(chunk, "usage", None) +USAGE_COST_BILLED_PROVIDERS: Final = frozenset({"kyma"}) + + +def billed_usage_cost(response: object, custom_llm_provider: str | None) -> float | None: + if custom_llm_provider not in USAGE_COST_BILLED_PROVIDERS: + return None + cost: Final = provider_reported_usage_cost(_usage_field(response, "usage"), custom_llm_provider) + return cost if cost is not None and math.isfinite(cost) and cost >= 0 else None + + _MESSAGES_REPORTED_COST_PROVIDERS: Final = frozenset({"opencode_go"}) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 47ec8331c83..703550dd2b5 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -18,6 +18,7 @@ from litellm.litellm_core_utils.bug_report import ( ) from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none from litellm.litellm_core_utils.secret_redaction import redact_string as redact_secret_string +from litellm.llms.bigmodel.common_utils import raise_bigmodel_business_error from litellm.types.utils import LlmProviders from ..exceptions import ( @@ -2762,6 +2763,8 @@ def exception_type( exception_provider=exception_provider, extra_information=extra_information, ) + if custom_llm_provider == "bigmodel": + raise_bigmodel_business_error(model, f"{exception_provider} - {error_str}", original_exception) # pyright: ignore[reportUnknownArgumentType] # exception_type takes untyped parameters if custom_llm_provider == "openrouter": _map_openrouter_exception( model=model, diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index 8178b7e98b8..8792fd0f58a 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -77,6 +77,7 @@ OPTIONAL_KWARGS_KEYS: Final = ( "otpm", "tpd", "rpd", + "daily_reset_time", "use_xai_oauth", "chatgpt_auth_file", "github_copilot_auth_type", diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index d9933087334..f8c8ff71f8e 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -794,6 +794,13 @@ def _get_openai_compatible_provider_info( api_base, dynamic_api_key, ) = litellm.XAIChatConfig().get_openai_compatible_provider_info(api_base, api_key) + elif custom_llm_provider == "bigmodel": + from litellm.llms.bigmodel.common_utils import get_bigmodel_api_key, get_bigmodel_url + + ( + api_base, # rebind-ok: existing provider-resolution pipeline resolves the API base in place + dynamic_api_key, # rebind-ok: existing provider-resolution pipeline resolves the API key in place + ) = get_bigmodel_url(api_base, "chat"), get_bigmodel_api_key(api_key) elif custom_llm_provider == "zai": ( api_base, diff --git a/litellm/litellm_core_utils/internal_call_metadata.py b/litellm/litellm_core_utils/internal_call_metadata.py index 1200d799018..f3dbd612bee 100644 --- a/litellm/litellm_core_utils/internal_call_metadata.py +++ b/litellm/litellm_core_utils/internal_call_metadata.py @@ -120,7 +120,7 @@ def is_unbilled_non_inference_call( origin: Final = metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) if metadata is not None else None if origin == BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN: return False - if not is_background_response(response): + if not is_background_response(_finished_response(response)): return True return origin == BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN or _is_managed_background_call(metadata) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 8b66c27b930..c0e988412a1 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -23,7 +23,7 @@ from httpx import Response, Timeout from pydantic import BaseModel, JsonValue, TypeAdapter import litellm -from litellm._internal_context import post_response_phase +from litellm._internal_context import in_mirrored_call, outside_mirrored_call, post_response_phase from litellm._logging import ( is_debugging_on, redact_string, @@ -162,9 +162,10 @@ from litellm.types.llms.openai import ( from litellm.types.mcp import MCPPostCallResponseObject from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.rerank import RerankResponse -from litellm.types.router import RouterRateLimitError +from litellm.types.router import RouterRateLimitError, SilentExperimentFlag from litellm.types.utils import ( DEPLOYMENT_SCOPED_PRICING_FIELDS, + AgenticLoopParams, CachingDetails, CallTypes, CostBreakdown, @@ -314,7 +315,7 @@ else: _in_memory_loggers: Final[list[CustomLogger]] = [] _STANDARD_LOGGING_METADATA_RESOLVED_KEYS: Final[frozenset[str]] = frozenset( - ("used_client_oauth_token", "usage_object", KUBERNETES_POD_ROUTING_KEY) + ("used_client_oauth_token", "usage_object", KUBERNETES_POD_ROUTING_KEY, "is_silent_experiment") ) _STANDARD_LOGGING_METADATA_KEYS: Final[frozenset[str]] = ( frozenset(StandardLoggingMetadata.__annotations__.keys()) - _STANDARD_LOGGING_METADATA_RESOLVED_KEYS @@ -808,6 +809,7 @@ class Logging(LiteLLMLoggingBaseClass): litellm_params = scrub_sensitive_keys_in_metadata(litellm_params) self.litellm_params = litellm_params + self.is_silent_experiment: Final = in_mirrored_call() # Initialize cost breakdown field self.cost_breakdown: CostBreakdown | None = None @@ -1130,9 +1132,6 @@ class Logging(LiteLLMLoggingBaseClass): } self.litellm_request_debug = litellm_params.get("litellm_request_debug", False) self.logger_fn = litellm_params.get("logger_fn", None) - if is_debugging_on() or self.litellm_request_debug: - verbose_logger.debug("self.optional_params: %s", self.optional_params) - self.model_call_details.update( { "model": self.model, @@ -1161,6 +1160,9 @@ class Logging(LiteLLMLoggingBaseClass): if "custom_llm_provider" in self.model_call_details: self.custom_llm_provider = self.model_call_details["custom_llm_provider"] + def record_agentic_loop_params(self, params: AgenticLoopParams) -> None: + self.model_call_details["agentic_loop_params"] = params + def record_applied_timeout(self, timeout: float | Timeout) -> None: self.applied_timeout = timeout self.litellm_params["timeout"] = timeout @@ -1940,10 +1942,10 @@ class Logging(LiteLLMLoggingBaseClass): def set_cost_breakdown( self, - input_cost: float, - output_cost: float, + input_cost: float | None, + output_cost: float | None, total_cost: float, - cost_for_built_in_tools_cost_usd_dollar: float, + cost_for_built_in_tools_cost_usd_dollar: float | None, additional_costs: dict | None = None, original_cost: float | None = None, discount_percent: float | None = None, @@ -1982,14 +1984,17 @@ class Logging(LiteLLMLoggingBaseClass): self.billed_token_rates = billed_token_rates self.cost_breakdown = CostBreakdown( - input_cost=input_cost, - output_cost=output_cost, total_cost=total_cost, - tool_usage_cost=cost_for_built_in_tools_cost_usd_dollar, service_tier=service_tier, data_residency=data_residency, vertex_location=vertex_location, ) + if input_cost is not None: + self.cost_breakdown["input_cost"] = input_cost + if output_cost is not None: + self.cost_breakdown["output_cost"] = output_cost + if cost_for_built_in_tools_cost_usd_dollar is not None: + self.cost_breakdown["tool_usage_cost"] = cost_for_built_in_tools_cost_usd_dollar if cache_read_cost is not None and cache_read_cost > 0: self.cost_breakdown["cache_read_cost"] = cache_read_cost if cache_creation_cost is not None and cache_creation_cost > 0: @@ -3252,7 +3257,7 @@ class Logging(LiteLLMLoggingBaseClass): """Restores trace_id/session_id contextvars once this attempt's own success logging (including any nested calls its callbacks trigger) is fully done.""" try: - with post_response_phase(): + with post_response_phase(), outside_mirrored_call(): return self._success_handler_body( result=result, start_time=start_time, @@ -3752,7 +3757,7 @@ class Logging(LiteLLMLoggingBaseClass): """Restores trace_id/session_id contextvars once this attempt's own success logging (including any nested calls its callbacks trigger) is fully done.""" try: - with post_response_phase(): + with post_response_phase(), outside_mirrored_call(): return await self._async_success_handler_body( result=result, start_time=start_time, end_time=end_time, cache_hit=cache_hit, **kwargs ) @@ -4269,13 +4274,14 @@ class Logging(LiteLLMLoggingBaseClass): ``for_async_request`` marks the sync run of a failure whose async run also logs it, so a CustomLogger that gets ``async_log_failure_event`` from that async run skips its sync hook.""" try: - return self._failure_handler_body( - exception=exception, - traceback_exception=traceback_exception, - start_time=start_time, - end_time=end_time, - for_async_request=for_async_request, - ) + with outside_mirrored_call(): + return self._failure_handler_body( + exception=exception, + traceback_exception=traceback_exception, + start_time=start_time, + end_time=end_time, + for_async_request=for_async_request, + ) finally: self._restore_correlation_context() @@ -4468,12 +4474,13 @@ class Logging(LiteLLMLoggingBaseClass): """Restores trace_id/session_id contextvars once this attempt's own failure logging (including any nested calls its callbacks trigger) is fully done.""" try: - return await self._async_failure_handler_body( - exception=exception, - traceback_exception=traceback_exception, - start_time=start_time, - end_time=end_time, - ) + with outside_mirrored_call(): + return await self._async_failure_handler_body( + exception=exception, + traceback_exception=traceback_exception, + start_time=start_time, + end_time=end_time, + ) finally: self._restore_correlation_context() @@ -5055,6 +5062,23 @@ class Logging(LiteLLMLoggingBaseClass): return result_copy +_SENSITIVE_KEYWORDS: Final = ( + "authorization", + "token", + "key", + "secret", + "vertex_credentials", + "credentials", + "password", + "passwd", +) + + +def has_sensitive_keyword(key: str) -> bool: + lowered: Final = key.lower() + return any(keyword in lowered for keyword in _SENSITIVE_KEYWORDS) + + def get_masked_values( sensitive_object: Mapping[str, object], ignore_sensitive_values: bool = False, @@ -5073,16 +5097,6 @@ def get_masked_values( masked_length: Optional length for the masked portion (number of *). If set, will use exactly this many * regardless of original string length. The total length will be unmasked_length + masked_length. """ - sensitive_keywords: Final = ( - "authorization", - "token", - "key", - "secret", - "vertex_credentials", - "credentials", - "password", - "passwd", - ) def _mask_value(v: object) -> object: if isinstance(v, dict): @@ -5109,12 +5123,7 @@ def get_masked_values( return v[: unmasked_length // 2] + "*" * (len(v) - unmasked_length) + v[-unmasked_length // 2 :] return { - k: ( - v - if ignore_sensitive_values - or not any(sensitive_keyword in k.lower() for sensitive_keyword in sensitive_keywords) - else _mask_value(v) - ) + k: (v if ignore_sensitive_values or not has_sensitive_keyword(k) else _mask_value(v)) for k, v in sensitive_object.items() } @@ -6560,6 +6569,7 @@ class StandardLoggingPayloadSetup: start_time: dt_object | None = None, response_id: str | None = None, custom_llm_provider: str | None = None, + is_silent_experiment: bool = False, ) -> StandardLoggingMetadata: """ Clean and filter the metadata dictionary to include only the specified keys in StandardLoggingMetadata. @@ -6633,6 +6643,7 @@ class StandardLoggingPayloadSetup: proxy_stamped_used_client_oauth_token(metadata, litellm_params), custom_llm_provider, ), + **(SilentExperimentFlag(is_silent_experiment=True) if is_silent_experiment else SilentExperimentFlag()), ) if isinstance(metadata, dict): for key in metadata.keys() & _STANDARD_LOGGING_METADATA_KEYS: @@ -7433,6 +7444,7 @@ def get_standard_logging_object_payload( start_time=start_time, response_id=id, custom_llm_provider=selected_provider if isinstance(selected_provider, str) else None, + is_silent_experiment=logging_obj.is_silent_experiment is True, ) _request_body: Final = proxy_server_request.get("body", {}) end_user_id: Final = clean_metadata["user_api_key_end_user_id"] or _request_body.get( @@ -7717,9 +7729,8 @@ def get_standard_logging_metadata( ) if isinstance(metadata, dict): # Update the clean_metadata with values from input metadata that match StandardLoggingMetadata fields - for key in StandardLoggingMetadata.__annotations__: - if key in metadata: - clean_metadata[key] = metadata[key] + for key in (metadata.keys() & StandardLoggingMetadata.__annotations__.keys()) - {"is_silent_experiment"}: + clean_metadata[key] = metadata[key] if metadata.get("user_api_key") is not None: if is_valid_sha256_hash(str(metadata.get("user_api_key"))): diff --git a/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py b/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py index a02c40b7611..78a8cb51bac 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py +++ b/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py @@ -27,13 +27,18 @@ class TranscriptionUsageObjectTransformation: if isinstance(usage_object, TranscriptionUsageDurationObject): return None elif isinstance(usage_object, TranscriptionUsageTokensObject): + input_token_details: Final = usage_object.input_token_details return Usage( prompt_tokens=usage_object.input_tokens, completion_tokens=usage_object.output_tokens, total_tokens=usage_object.total_tokens, - prompt_tokens_details=PromptTokensDetailsWrapper( - text_tokens=usage_object.input_token_details.text_tokens, - audio_tokens=usage_object.input_token_details.audio_tokens, + prompt_tokens_details=( + None + if input_token_details is None + else PromptTokensDetailsWrapper( + text_tokens=input_token_details.text_tokens, + audio_tokens=input_token_details.audio_tokens, + ) ), ) return None diff --git a/litellm/litellm_core_utils/llm_request_utils.py b/litellm/litellm_core_utils/llm_request_utils.py index 0bf59a6d1f2..db881aa16c6 100644 --- a/litellm/litellm_core_utils/llm_request_utils.py +++ b/litellm/litellm_core_utils/llm_request_utils.py @@ -1,8 +1,9 @@ -from collections.abc import Mapping +from collections.abc import Iterable, Iterator, Mapping from typing import Final, cast import litellm from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH +from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider def _form_field_value(value: object) -> str: @@ -153,24 +154,33 @@ def pick_cheapest_chat_models_from_llm_provider(custom_llm_provider: str, n=1): if custom_llm_provider not in litellm.models_by_provider: return [] - known_models: Final = litellm.models_by_provider.get(custom_llm_provider, []) - model_costs: Final = [] + known_models: Final[Iterable[str]] = litellm.models_by_provider.get(custom_llm_provider, ()) + ranked_rows: Final = ( + (route_rank, cost, model) + for model, cost in _chat_row_costs(known_models, custom_llm_provider) + if (route_rank := _route_rank(model, custom_llm_provider)) is not None + ) + return [model for _, _, model in sorted(ranked_rows, key=lambda row: row[:2])[:n]] - for model in known_models: + +def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float]]: + for model in models: try: model_info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) except Exception: continue if model_info.get("mode") != "chat": continue - _cost = (model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0) - model_costs.append((model, _cost)) + yield model, (model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0) - # Sort by cost (ascending) - model_costs.sort(key=lambda x: x[1]) - # Return the top n cheapest models - return [model for model, _ in model_costs[:n]] +def _route_rank(model: str, custom_llm_provider: str) -> int | None: + routed_provider: Final = inferred_provider(model) + if routed_provider is None: + return 1 + if routed_provider == inferred_provider(f"{custom_llm_provider}/{model}"): + return 0 + return None def get_proxy_server_request_headers(litellm_params: dict | None) -> dict: diff --git a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py index ace57a64aae..051a8fdf256 100644 --- a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py +++ b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py @@ -27,6 +27,15 @@ def _timing_window_start( return start_time, False +def response_ms_since_logging_start(logging_obj: LiteLLMLoggingObject) -> float | None: + reported_ms: Final = logging_obj.response_timing_metrics.get("_response_ms") + if reported_ms is None: + return None + logging_start: Final = logging_obj.start_time + window_start: Final = _timing_window_start(logging_start, logging_obj)[0] + return reported_ms - max(0.0, logging_start.timestamp() - window_start.timestamp()) * 1000 + + def _union_duration_ms(windows: object, lower: float, upper: float) -> float | None: if not isinstance(windows, (list, tuple)): return None diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 101f069c0f8..9bc53791b36 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -17,7 +17,7 @@ from pathlib import Path from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError import litellm from litellm import verbose_logger @@ -2073,6 +2073,19 @@ def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT: return cast(_MarkedT, marked) # cast-ok: same block shape as the input plus the marker key +_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(PromptCacheBreakpoint) + + +def prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object: + if marker is None or not drop_params: + return marker + try: + return _PROMPT_CACHE_BREAKPOINT.validate_python(marker) + except ValidationError: + verbose_logger.debug("Dropping malformed prompt_cache_breakpoint %r under drop_params", marker) + return None + + def cache_control_as_prompt_cache_breakpoint(block: _MarkedT) -> _MarkedT: """Anthropic's ``cache_control`` mark rewritten as OpenAI's ``prompt_cache_breakpoint`` on the same block.""" if block.get("cache_control") is None: diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index fad4caa59a0..8cfa335422e 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -32,6 +32,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import place_mid_conversation_system, split_leading_system_run, ) +from litellm.litellm_core_utils.tool_fields import tools_without_hosted_mcp from litellm.litellm_core_utils.url_utils import async_safe_get, safe_get from litellm.llms.custom_httpx.http_handler import HTTPHandler, get_async_httpx_client, header_value from litellm.types.files import get_file_extension_from_mime_type @@ -5855,8 +5856,13 @@ def function_call_prompt( llm_provider: str, drop_params: bool, ) -> list[dict[str, object]]: + prompt_functions: Final = tools_without_hosted_mcp( + functions, model=model, route=llm_provider, drop_params=drop_params + ) + if not prompt_functions: + return messages header: Final = """To call a function, reply with JSON ONLY in this format {"name": "function_name", "arguments":{"argument_name": "argument_value"}}. Once a function result answers the request, reply to the user in plain text instead of calling a function again. The following functions are available to you:""" - function_prompt: Final = header + "".join(f"\n{function}\n" for function in functions) + function_prompt: Final = header + "".join(f"\n{function}\n" for function in prompt_functions) if not any(_is_system_message(message) for message in messages): return [*messages, {"role": "system", "content": function_prompt}] return [ diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 8d243e572fa..7c91ffdff76 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -12,12 +12,17 @@ from typing_extensions import ReadOnly, Required import litellm from litellm._logging import verbose_logger -from litellm.litellm_core_utils.core_helpers import chunk_usage, provider_reported_usage_cost +from litellm.litellm_core_utils.core_helpers import ( + USAGE_COST_BILLED_PROVIDERS, + chunk_usage, + provider_reported_usage_cost, +) from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import BEDROCK_TRACE_FIELDS, object_items, object_mapping from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_content_from_model_response, # pyright: ignore[reportUnknownVariableType] # response param accepts an untyped dict ) +from litellm.litellm_core_utils.token_counter import usage_tokenizer_model from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionAssistantContentValue, @@ -51,7 +56,7 @@ from litellm.types.utils import ( UnreportedUsage, Usage, ) -from litellm.utils import print_verbose, token_counter +from litellm.utils import get_base_model_from_metadata, print_verbose, token_counter if TYPE_CHECKING: from openai.types.completion_usage import CompletionUsage @@ -1351,7 +1356,7 @@ def stream_chunk_builder_text_completion( return TextCompletionResponse(**response) -_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value}) +_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value}) | USAGE_COST_BILLED_PROVIDERS def _logged_provider(logging_obj: Optional["Logging"]) -> object: @@ -1552,6 +1557,9 @@ def stream_chunk_builder( return stream_chunk_builder_text_completion(chunks=chunks, messages=messages) model: Final = chunks[0]["model"] + usage_model: Final = usage_tokenizer_model( + model, get_base_model_from_metadata(logging_obj.model_call_details) if logging_obj is not None else None + ) # Initialize the response dictionary response: Final = processor.build_base_response(chunks) @@ -1575,7 +1583,7 @@ def stream_chunk_builder( completion_output = get_content_from_model_response(response) usage = processor.calculate_usage( chunks=chunks, - model=model, + model=usage_model, completion_output=completion_output, messages=messages, reasoning_tokens=0, @@ -1766,7 +1774,7 @@ def stream_chunk_builder( usage = processor.calculate_usage( chunks=chunks, - model=model, + model=usage_model, completion_output=completion_output, messages=messages, reasoning_tokens=reasoning_tokens, diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 2348d8e57d6..5542249d2c7 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -92,6 +92,18 @@ def _next_sync_or_exhausted(it: Iterator[object]) -> object: return _SYNC_ITER_EXHAUSTED +def _stamp_usage_cost( + usage_chunk: ModelResponse | TextCompletionResponse | ModelResponseStream, + complete_response: ModelResponse | TextCompletionResponse | ModelResponseStream, +) -> None: + bill: Final = (get_hidden_params(complete_response) or {}).get("response_cost") + usage: Final[object] = getattr(usage_chunk, "usage", None) + if usage is None or isinstance(bill, bool) or not isinstance(bill, (int, float)): + return + if bill > 0 or isinstance(getattr(usage, "cost", None), (int, float)): + setattr(usage, "cost", float(bill)) # noqa: B010 # cost is an optional extra on the usage model + + def _stamp_served_service_tier(response: ModelResponseStream, complete_streaming_response: ModelResponse) -> None: served_tier: Final = complete_streaming_response.model_dump().get("service_tier") if isinstance(served_tier, str) and served_tier: @@ -1926,6 +1938,16 @@ class CustomStreamWrapper: model_response: Final = self.model_response_creator( hidden_params=None if self.received_model_id is None else {"received_model_id": self.received_model_id} ) + served_tier: Final = next( + ( + tier + for chunk in reversed(self.chunks) + if isinstance(tier := getattr(chunk, "service_tier", None), str) and tier + ), + None, + ) + if served_tier is not None: + setattr(model_response, "service_tier", served_tier) # noqa: B010 # pydantic extra, not a declared field _finish_reason: Final = self.received_finish_reason or self.intermittent_finish_reason if _finish_reason is not None: model_response.choices[0].finish_reason = _finish_reason @@ -2141,6 +2163,7 @@ class CustomStreamWrapper: getattr(complete_streaming_response, "usage"), ) _stamp_served_service_tier(response, complete_streaming_response) + _stamp_usage_cost(response, complete_streaming_response) try: _cache_copy = complete_streaming_response.model_copy(deep=True) _log_copy = complete_streaming_response.model_copy(deep=True) @@ -2361,6 +2384,7 @@ class CustomStreamWrapper: getattr(complete_streaming_response, "usage"), ) _stamp_served_service_tier(response, complete_streaming_response) + _stamp_usage_cost(response, complete_streaming_response) try: _copy = complete_streaming_response.model_copy(deep=True) except RuntimeError: diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index f42e4eae08f..591d77bdb16 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -15,6 +15,7 @@ import anyio import anyio.lowlevel import httpx import tiktoken +from tiktoken.model import MODEL_TO_ENCODING from typing_extensions import ParamSpec, TypeVar import litellm @@ -49,6 +50,7 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.tokenizer import Encoding, HuggingFace from litellm.litellm_core_utils.url_utils import safe_get from litellm.llms.custom_httpx.http_handler import get_httpx_client, header_value +from litellm.router_utils.deployment_pricing import deployment_pricing_models from litellm.rust_bridge.tokenizer import get_encoding from litellm.types.llms.anthropic import ( AnthropicContentParamSource, @@ -609,7 +611,7 @@ T_ParamSpec: Final = ParamSpec("T_ParamSpec") T_Retval = TypeVar("T_Retval") _COUNT_OFFLOAD_LIMITER: Final = anyio.lowlevel.RunVar[anyio.CapacityLimiter]("litellm_count_offload_limiter") _ANTHROPIC_TOOL_KEYS_MAPPED_BY_BRIDGE: Final = frozenset( - ("name", "input_schema", "description", "cache_control", "strict", "type", "eager_input_streaming") + ("name", "input_schema", "description", "cache_control", "strict", "type", "eager_input_streaming", "custom") ) @@ -1005,17 +1007,144 @@ def openai_tokenizer_encoding(model: str) -> "Encoding": def openai_tokenizer_encoding_name(model: str) -> str: - """The tiktoken encoding name for `model`, without loading the encoding.""" - from litellm.utils import print_verbose + """The tiktoken encoding name for `model`, without loading the encoding. - model_to_use: Final = _fix_model_name(model) + A listed OpenAI or Azure model that tiktoken does not know is either a suffixed variant of a model it does + know (`text-embedding-ada-002-v2`) or a newer release than its table, which gets o200k_base, the encoding every + OpenAI model since gpt-4o uses. Provider and region prefixes (`azure/eu/`, `azure_ai/`) are dropped, a name + listed only under Azure (`gpt-41`, `openai/o1-mini`) counts as listed through its `azure/` cost-map key, and an + unlisted name tiktoken counts with o200k_base (`openai/gpt-5.9`) or named like a gpt-5 or o-series release + (`openai/gpt-6.2`) gets it.""" + model_to_use: Final = _openai_model_name(model) or _fix_model_name(model) if "gpt-4o" in model_to_use: return "o200k_base" try: return tiktoken.encoding_name_for_model(model_to_use) except KeyError: - print_verbose("Warning: model not found. Using cl100k_base encoding.") - return "cl100k_base" + return _suffixed_variant_encoding_name(model_to_use) or "o200k_base" + + +def unresolved_tokenizer_model(model: str) -> str: + """`model`, or `gpt-3.5-turbo` when resolving the OpenAI model `model` names moves it off cl100k_base, the encoding + tiktoken gives the name as written, so a count with the result never drops below that one.""" + as_written: Final = _fix_model_name(model) + as_written_encoding: Final = ( + "o200k_base" if "gpt-4o" in as_written else (_tiktoken_encoding_name(as_written) or "cl100k_base") + ) + moved_off_cl100k: Final = ( + as_written_encoding == "cl100k_base" and openai_tokenizer_encoding_name(model) != "cl100k_base" + ) + return "gpt-3.5-turbo" if moved_off_cl100k else model + + +def usage_tokenizer_model(model: str, base_model: str | None) -> str: + """The model to estimate missing usage with: a deployment's `base_model` when it names an OpenAI model, since a + custom deployment name says nothing about its tokenizer, else `model`.""" + return base_model if base_model is not None and is_openai_model(base_model) else model + + +def is_openai_model(model: str) -> bool: + """Whether `model` names an OpenAI or Azure OpenAI model, listed in the cost map or counted with o200k_base by + tiktoken, with or without its prefixes.""" + return _openai_model_name(model) is not None + + +def deployment_openai_model(deployment: object) -> str | None: + """The OpenAI model a router deployment serves, read from its base model first, since a custom deployment name + (`azure/my-gpt-5`) says nothing about its tokenizer, or None when it serves no OpenAI model.""" + return _as_openai_model(_served_model(deployment)) + + +def served_openai_models(deployments: Iterable[object]) -> tuple[str | None, ...]: + """The distinct OpenAI models `deployments` serve, with None standing for any deployment serving another model.""" + served: Final = dict.fromkeys(_served_model(deployment) for deployment in deployments) + return tuple(dict.fromkeys(_as_openai_model(model) for model in served)) + + +def _served_model(deployment: object) -> str | None: + return next(iter(deployment_pricing_models(deployment)), None) + + +def _as_openai_model(model: str | None) -> str | None: + return model if model is not None and is_openai_model(model) else None + + +def single_encoding_openai_model(served: Sequence[str | None]) -> str | None: + """An OpenAI model to count a request with, when every `served_openai_models` entry is an OpenAI model on the same + encoding, else None.""" + if not served or None in served: + return None + encodings: Final = frozenset(openai_tokenizer_encoding_name(model) for model in served if model is not None) + return served[0] if len(encodings) == 1 else None + + +def _openai_model_name(model: str) -> str | None: + return _listed_openai_model_name(model) or _tiktoken_o200k_model_name(model) or _versioned_openai_model_name(model) + + +_VERSIONED_OPENAI_MODEL: Final = re.compile(r"(?:gpt-(?:[5-9]|[1-9]\d+)|o[1-9]\d*)(?:[.-]|$)") + + +def _versioned_openai_model_name(model: str) -> str | None: + name: Final = model.rsplit("/", 1)[-1] + return name if _VERSIONED_OPENAI_MODEL.match(name) else None + + +def _tiktoken_o200k_model_name(model: str) -> str | None: + name: Final = model.rsplit("/", 1)[-1] + return name if _tiktoken_encoding_name(name) == "o200k_base" else None + + +def _listed_openai_model_name(model: str) -> str | None: + segments: Final = model.split("/") + suffixes: Final = ("/".join(segments[start:]) for start in range(len(segments))) + listed: Final = any( + _is_openai_cost_map_model(suffix) or _is_openai_cost_map_model(f"azure/{suffix}") for suffix in suffixes + ) + return segments[-1] if listed else None + + +def _is_openai_cost_map_model(model: str) -> bool: + return ( + model in litellm.azure_llms + or model in litellm.open_ai_chat_completion_models + or _is_azure_openai_text_model(model) + ) + + +_TEXT_MODEL_MODES: Final = frozenset({"chat", "completion", "responses", "realtime"}) + + +def _is_azure_openai_text_model(model: str) -> bool: + cost_info: Final[Mapping[str, object]] = litellm.model_cost.get(model) or {} + return ( + cost_info.get("litellm_provider") == "azure" + and cost_info.get("mode") in _TEXT_MODEL_MODES + and _is_openai_model_name(model.rsplit("/", 1)[-1]) + ) + + +def _is_openai_model_name(name: str) -> bool: + return ( + name in litellm.open_ai_chat_completion_models + or _tiktoken_encoding_name(name) is not None + or any( + name.startswith(f"{listed}-") or listed.startswith(f"{name}-") + for listed in tuple(litellm.open_ai_chat_completion_models) + ) + ) + + +def _tiktoken_encoding_name(model: str) -> str | None: + try: + return tiktoken.encoding_name_for_model(model) + except KeyError: + return None + + +def _suffixed_variant_encoding_name(model: str) -> str | None: + known: Final = tuple(name for name in MODEL_TO_ENCODING if model.startswith(f"{name}-")) + return MODEL_TO_ENCODING[max(known, key=len)] if known else None def uses_legacy_message_accounting(model: str) -> bool: diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index d371aab4581..31b0f06251d 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -432,27 +432,30 @@ class AnthropicMessagesHandler(BaseTranslation): message, _ = serialize_http_exception_detail(exc.detail) return tuple(anthropic_sse_error_frames(message)) - def _standalone_block_chunks(self, exc: "ModifyResponseException") -> list[bytes]: + def build_block_response(self, exc: "ModifyResponseException") -> "AnthropicMessagesResponse": import uuid - from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( - FakeAnthropicMessagesStreamIterator, - ) from litellm.llms.base_llm.guardrail_translation.utils import ( blocked_response_usage, ) from litellm.types.utils import AnthropicMessagesResponse - block_response: Final = AnthropicMessagesResponse( + return AnthropicMessagesResponse( id=f"msg_{uuid.uuid4()}", type="message", role="assistant", content=[{"type": "text", "text": exc.message}], model=exc.model, stop_reason="end_turn", - usage=blocked_response_usage(getattr(exc, "original_response", None)), + usage=blocked_response_usage(exc.original_response), ) - return list(FakeAnthropicMessagesStreamIterator(response=block_response)) + + def _standalone_block_chunks(self, exc: "ModifyResponseException") -> list[bytes]: + from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( + FakeAnthropicMessagesStreamIterator, + ) + + return list(FakeAnthropicMessagesStreamIterator(response=self.build_block_response(exc))) def _block_continuation_chunks( self, exc: "ModifyResponseException", responses_so_far: Sequence[object] diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index de2e477a4ca..48c1fcd45e0 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -13,6 +13,7 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm.anthropic_beta_headers_manager import get_unsupported_headers from litellm.constants import ( + ANTHROPIC_BILLING_METADATA_PREFIX, ANTHROPIC_MIN_THINKING_BUDGET_TOKENS, ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES, DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS, @@ -2080,7 +2081,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if not system_message_block["content"]: continue if self.should_strip_billing_metadata() and system_message_block["content"].startswith( - "x-anthropic-billing-header:" + ANTHROPIC_BILLING_METADATA_PREFIX ): continue anthropic_system_message_content = AnthropicSystemMessageContent( @@ -2100,7 +2101,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): self.should_strip_billing_metadata() and _content.get("type") == "text" and text_value - and text_value.startswith("x-anthropic-billing-header:") + and text_value.startswith(ANTHROPIC_BILLING_METADATA_PREFIX) ): continue anthropic_system_message_content = AnthropicSystemMessageContent( diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 1f5e4e7de39..e591f3a3fde 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -121,6 +121,54 @@ _DECODED_JSON: Final = TypeAdapter(object) _CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/") +def is_claude_tool_target(custom_llm_provider: str | None, model: str) -> bool: + if custom_llm_provider == "anthropic": + return True + model_lower: Final = model.lower() + if custom_llm_provider == "bedrock": + return "claude" in model_lower or ("arn:" in model_lower and ":bedrock:" in model_lower) + if custom_llm_provider == "azure_ai" or custom_llm_provider == "vertex_ai": + return "claude" in model_lower + return False + + +def is_claude_model_target(custom_llm_provider: str | None, model: str) -> bool: + return "claude" in model.lower() and is_claude_tool_target(custom_llm_provider, model) + + +def _hoistable_defer_loading(tool_dict: Mapping[object, object], custom: object) -> bool | None: + if not isinstance(custom, dict) or "defer_loading" in tool_dict: + return None + defer_loading: Final = OBJECT_MAPPING_ADAPTER.validate_python(custom).get("defer_loading") + return defer_loading if isinstance(defer_loading, bool) else None + + +def _tool_without_custom_field(tool: object, hoist_defer_loading: bool, drop_custom: bool) -> object: + if not isinstance(tool, dict): + return tool + + if "custom" not in tool: + return tool + + tool_dict: Final = OBJECT_MAPPING_ADAPTER.validate_python(tool) + + custom: Final = tool_dict["custom"] + without_custom: Final = {key: value for key, value in tool_dict.items() if key != "custom"} + defer_loading: Final = _hoistable_defer_loading(tool_dict, custom) if hoist_defer_loading else None + if defer_loading is None: + return without_custom if drop_custom else tool_dict + + unhoisted: Final = { + key: value for key, value in OBJECT_MAPPING_ADAPTER.validate_python(custom).items() if key != "defer_loading" + } + kept_custom: Final = {"custom": unhoisted} if unhoisted and not drop_custom else {} + return {**without_custom, "defer_loading": defer_loading, **kept_custom} + + +def tools_without_custom_field(tools: Sequence[object], hoist_defer_loading: bool, drop_custom: bool) -> list[object]: + return [_tool_without_custom_field(tool, hoist_defer_loading, drop_custom) for tool in tools] + + def is_anthropic_messages_url(url: str) -> bool: """Check whether a URL addresses Anthropic's Messages API.""" parsed_url: Final = urlparse(url) diff --git a/litellm/llms/anthropic/count_tokens/handler.py b/litellm/llms/anthropic/count_tokens/handler.py index a230294ad3f..f8bf7232d9c 100644 --- a/litellm/llms/anthropic/count_tokens/handler.py +++ b/litellm/llms/anthropic/count_tokens/handler.py @@ -39,6 +39,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): tools: list[dict[str, JsonValue]] | None = None, system: JsonValue = None, optional_params: Mapping[str, JsonValue] | None = None, + drop_params: bool = False, ) -> dict[str, JsonValue]: """ Handle a CountTokens request using httpx. @@ -69,6 +70,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): tools=tools, system=system, optional_params=optional_params, + drop_params=drop_params, ) verbose_logger.debug("Transformed request: %s", request_body) diff --git a/litellm/llms/anthropic/count_tokens/token_counter.py b/litellm/llms/anthropic/count_tokens/token_counter.py index 368bb929eeb..91fc77773d1 100644 --- a/litellm/llms/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/anthropic/count_tokens/token_counter.py @@ -11,6 +11,7 @@ from pydantic import JsonValue import litellm from litellm._logging import verbose_logger from litellm.exceptions import AuthenticationError +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler from litellm.llms.anthropic.count_tokens.transformation import ( AnthropicCountFields, @@ -131,6 +132,7 @@ class AnthropicTokenCounter(BaseTokenCounter): tools=tools, system=system, optional_params=count_options, + drop_params=drops_unsupported_params(litellm_params.get("drop_params")), ) if result is not None: diff --git a/litellm/llms/anthropic/count_tokens/transformation.py b/litellm/llms/anthropic/count_tokens/transformation.py index 0967c22f93a..e60ffe67c8d 100644 --- a/litellm/llms/anthropic/count_tokens/transformation.py +++ b/litellm/llms/anthropic/count_tokens/transformation.py @@ -26,7 +26,11 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import split_leading_system_run, ) from litellm.litellm_core_utils.url_utils import join_api_base -from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers +from litellm.llms.anthropic.common_utils import ( + is_claude_model_target, + merge_anthropic_beta_headers, + tools_without_custom_field, +) from litellm.llms.anthropic.wif import resolve_anthropic_base from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse @@ -372,6 +376,7 @@ class AnthropicCountTokensConfig: tools: list[dict[str, JsonValue]] | None = None, system: JsonValue = None, optional_params: Mapping[str, JsonValue] | None = None, + drop_params: bool = False, ) -> dict[str, JsonValue]: # mutable-ok: provider transport requires JSON dictionaries """ Transform request to Anthropic CountTokens format. @@ -391,7 +396,19 @@ class AnthropicCountTokensConfig: **MappingProxyType( { key: value - for key, value in (("system", counted_system), ("tools", tools)) + for key, value in ( + ("system", counted_system), + ( + "tools", + None + if tools is None + else tools_without_custom_field( + tools, + hoist_defer_loading=is_claude_model_target("anthropic", model), + drop_custom=drop_params, + ), + ), + ) if value is not None } ), diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index 1ed552d46e3..3524c9e73d8 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -203,7 +203,7 @@ from litellm.types.llms.openai import ( ChatCompletionUserMessage, ToolMessageContentPart, ) -from litellm.types.utils import Choices, ModelResponse, StreamingChoices, Usage +from litellm.types.utils import Choices, LlmProviders, ModelResponse, StreamingChoices, Usage from litellm.utils import supports_mid_conversation_system from .streaming_iterator import AnthropicSSEStream, AnthropicStreamWrapper @@ -362,6 +362,9 @@ class AnthropicAdapter: _BlockT: Final = TypeVar("_BlockT", bound=Mapping[str, object]) +_CLAUDE_OVER_REASONING_EFFORT_PROVIDERS: Final = frozenset({LlmProviders.KYMA.value}) + + class LiteLLMAnthropicMessagesAdapter: def __init__(self): pass @@ -817,6 +820,7 @@ class LiteLLMAnthropicMessagesAdapter: "name", "input_schema", "description", + "custom", "cache_control", "strict", "type", @@ -1127,7 +1131,8 @@ class LiteLLMAnthropicMessagesAdapter: """Translate Anthropic thinking to either thinking or reasoning_effort. A Claude-family target keeps ``thinking`` verbatim, since every bridged provider serving one - speaks that param. Carrying its adaptive effort tier alongside takes two different params, + speaks that param, except Kyma, which serves Claude over the OpenAI dialect and takes only + ``reasoning_effort``, so its Claude models get the non-Claude mapping. Carrying its adaptive effort tier alongside takes two different params, because the two are not interchangeable at the provider mapping below. Bedrock takes ``output_config`` directly, which attaches the tier and leaves ``thinking`` @@ -1158,7 +1163,9 @@ class LiteLLMAnthropicMessagesAdapter: is_bedrock_target: Final = model.startswith(("bedrock/", "converse/", "invoke/")) or self.is_bedrock_arn_model( model ) - is_claude_target: Final = self.is_anthropic_claude_model(model) or self.is_bedrock_arn_model(model) + is_claude_target: Final = ( + self.is_anthropic_claude_model(model) or self.is_bedrock_arn_model(model) + ) and custom_llm_provider not in _CLAUDE_OVER_REASONING_EFFORT_PROVIDERS output_config: Final = anthropic_message_request.get("output_config") if is_claude_target: diff --git a/litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py b/litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py index d4430ca5ab9..c4f7966f429 100644 --- a/litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py +++ b/litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py @@ -31,6 +31,7 @@ from ..constants import ( ) from ..errors import AnthropicContextManagementError from ..placeholders import build_cleared_tool_result_content +from .compact import count_system_tokens, history_after_client_compaction if TYPE_CHECKING: from litellm.types.llms.openai import ChatCompletionToolParam @@ -203,9 +204,10 @@ def _trigger_met( model: str, messages: list[dict[str, object]], tools: list[dict[str, object]] | None, + system: str | Sequence[Mapping[str, object]] | None, counted_tool_uses: int, ) -> tuple[bool, int | None]: - """Return (trigger_met, input_tokens if counted for reuse).""" + """Return (trigger_met, message and tool tokens for reuse); like the API, input_tokens adds the system prompt.""" trigger_type: Final = trigger.get("type", "input_tokens") threshold = trigger.get("value") @@ -216,14 +218,17 @@ def _trigger_met( if not isinstance(threshold, int): threshold = DEFAULT_INPUT_TOKENS_TRIGGER - current_tokens: Final = litellm.token_counter( + message_tokens: Final = litellm.token_counter( model=model, messages=messages, tools=cast("list[ChatCompletionToolParam] | None", tools), ) + if message_tokens > threshold: + return True, message_tokens + current_tokens: Final = message_tokens + count_system_tokens(model, system) verbose_logger.debug("context_management polyfill: current_tokens: %s", current_tokens) verbose_logger.debug("context_management polyfill: threshold: %s", threshold) - return current_tokens > threshold, current_tokens + return current_tokens > threshold, message_tokens def _resolve_keep_count(keep: dict[str, object]) -> int: @@ -237,7 +242,7 @@ def _resolve_keep_count(keep: dict[str, object]) -> int: def _last_completed_tool_use_id( - messages: list[dict[str, object]], + messages: Sequence[Mapping[str, object]], ) -> str | None: """Latest completed tool_result id; never cleared.""" last_id: str | None = None @@ -253,7 +258,7 @@ def _last_completed_tool_use_id( def _clear_tool_results( - messages: list[dict[str, object]], ids_to_clear: set[str] + messages: list[dict[str, object]], ids_to_clear: Set[str] ) -> tuple[list[dict[str, object]], int]: """Clear matching tool_result content; return (messages, cleared_count).""" cleared = 0 @@ -311,7 +316,11 @@ def apply_clear_tool_uses_20250919( system: str | list[dict[str, object]] | None, edit_spec: ClearToolUsesEditSpec, ) -> tuple[list[dict[str, object]], AppliedEdit | None]: - """Apply clear_tool_uses to an edit ``check_clear_tool_uses_spec`` returned; return (messages, AppliedEdit or None).""" + """Apply clear_tool_uses to an edit ``check_clear_tool_uses_spec`` returned; return (messages, AppliedEdit or None). + + Like the API, it reads only the history after a client compaction block and counts the block's summary as system. + """ + kept, counted_system = history_after_client_compaction(messages, system) fields: Final = _ClearToolUsesFields.model_validate(edit_spec) clear_at_least: Final = 0 if fields.clear_at_least is None else fields.clear_at_least.value exclude_tools: Final = frozenset(fields.exclude_tools or ()) @@ -326,7 +335,7 @@ def apply_clear_tool_uses_20250919( "value": DEFAULT_KEEP_TOOL_USES, } - blocks: Final = _all_blocks(messages) + blocks: Final = _all_blocks(kept) results: Final = _blocks_by_id(blocks, "tool_result", "tool_use_id") uses: Final = _blocks_by_id(blocks, "tool_use", "id") tiny: Final = _tiny_result_ids(results) @@ -335,15 +344,15 @@ def apply_clear_tool_uses_20250919( for tool_use_id, name in _tool_uses_in_order(blocks) if name not in exclude_tools and (tool_use_id not in tiny or _clears_input(name, input_tools)) ) - met, counted_tokens = _trigger_met(trigger, model, messages, tools, len(clearable)) - if not met: - return messages, None - keep_count: Final = _resolve_keep_count(keep) if len(clearable) <= keep_count: return messages, None - last_completed_id: Final = _last_completed_tool_use_id(messages) + met, counted_tokens = _trigger_met(trigger, model, kept, tools, counted_system, len(clearable)) + if not met: + return messages, None + + last_completed_id: Final = _last_completed_tool_use_id(kept) candidates: Final = tuple( (tool_use_id, name) for tool_use_id, name in clearable[: len(clearable) - keep_count] @@ -358,7 +367,7 @@ def apply_clear_tool_uses_20250919( _select_in_batches(candidate_ids, tokens, clear_at_least) if clear_at_least > 0 else candidate_ids ) - edited, cleared_count = _clear_tool_results(messages, set(ids_to_clear)) + edited, cleared_count = _clear_tool_results(messages, ids_to_clear) verbose_logger.debug("context_management polyfill: edited: %s", edited) if cleared_count == 0: return messages, None @@ -370,11 +379,12 @@ def apply_clear_tool_uses_20250919( counted_tokens if counted_tokens is not None else litellm.token_counter( - model=model, messages=messages, tools=cast("list[ChatCompletionToolParam] | None", tools) + model=model, messages=kept, tools=cast("list[ChatCompletionToolParam] | None", tools) ) ) + kept_after, _ = history_after_client_compaction(final_messages, system) tokens_after: Final = litellm.token_counter( - model=model, messages=final_messages, tools=cast("list[ChatCompletionToolParam] | None", tools) + model=model, messages=kept_after, tools=cast("list[ChatCompletionToolParam] | None", tools) ) cleared_input_tokens: Final = max(tokens_before - tokens_after, 0) diff --git a/litellm/llms/anthropic/messages/context_management/editors/compact.py b/litellm/llms/anthropic/messages/context_management/editors/compact.py index 254d1c4c058..0bf8cac7344 100644 --- a/litellm/llms/anthropic/messages/context_management/editors/compact.py +++ b/litellm/llms/anthropic/messages/context_management/editors/compact.py @@ -189,6 +189,21 @@ def _strip_compaction_blocks( return cleaned +def history_after_client_compaction( + messages: Sequence[dict[str, object]], + system: str | Sequence[Mapping[str, object]] | None, +) -> tuple[list[dict[str, object]], str | Sequence[Mapping[str, object]] | None]: + """Return the history the API keeps after the latest compaction block, without it, and the system with its summary.""" + sliced, block = _slice_around_compaction_block(messages) + if block is None: + return list(messages), system + tail: Final = _strip_compaction_blocks(sliced) + summary: Final = block.get("content") + if not isinstance(summary, str) or not summary: + return tail, system + return tail, _augment_system_with_summary(_system_to_text(system) or None, summary) + + def _augment_system_with_summary( system: str | list[dict[str, object]] | None, summary_text: str, @@ -320,14 +335,16 @@ def _count_effective_tokens( content: Final = compaction_block.get("content") or "" if content: total += litellm.token_counter(model=model, text=content) + return total + count_system_tokens(model, system) + + +def count_system_tokens(model: str, system: str | Sequence[Mapping[str, object]] | None) -> int: system_text: Final = _system_to_text(system) - if system_text: - total += litellm.token_counter(model=model, text=system_text) - return total + return litellm.token_counter(model=model, text=system_text) if system_text else 0 def _system_to_text( - system: str | list[dict[str, object]] | None, + system: str | Sequence[Mapping[str, object]] | None, ) -> str: """Flatten an Anthropic-style ``system`` value into a single string for token counting. Returns ``""`` when ``system`` carries no text.""" @@ -338,7 +355,7 @@ def _system_to_text( return "\n".join( text for block in system - if isinstance(block, dict) + if isinstance(block, Mapping) and block.get("type") == "text" and isinstance(text := block.get("text"), str) and text diff --git a/litellm/llms/anthropic/messages/native/transformation.py b/litellm/llms/anthropic/messages/native/transformation.py index cc53858f842..75b0329315f 100644 --- a/litellm/llms/anthropic/messages/native/transformation.py +++ b/litellm/llms/anthropic/messages/native/transformation.py @@ -6,6 +6,7 @@ import httpx import litellm from litellm._logging import verbose_logger from litellm.anthropic_beta_headers_manager import get_unsupported_headers, is_beta_header_supported_by_bundled_config +from litellm.constants import ANTHROPIC_BILLING_METADATA_PREFIX from litellm.exceptions import AuthenticationError from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -29,13 +30,16 @@ from litellm.types.llms.anthropic_tool_search import get_tool_search_beta_header from litellm.types.router import GenericLiteLLMParams from ...common_utils import ( + OBJECT_LIST_ADAPTER, AnthropicError, AnthropicModelInfo, + is_claude_model_target, merge_anthropic_beta_headers, optionally_handle_anthropic_oauth, requires_native_compaction_beta, strip_advisor_blocks_from_messages, strip_encrypted_reasoning_blocks_from_anthropic_messages, + tools_without_custom_field, ) from .mid_conversation_system import ( as_system_content_blocks, @@ -181,6 +185,16 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): """ return False + def hoists_wrapped_defer_loading(self, model: str) -> bool: + return is_claude_model_target(self._resolved_provider, model) + + def tools_for_upstream(self, tools: Sequence[object], model: str, drop_params: object) -> Sequence[object]: + return tools_without_custom_field( + tools, + hoist_defer_loading=self.hoists_wrapped_defer_loading(model), + drop_custom=drops_unsupported_params(drop_params), + ) + @staticmethod def _filter_billing_headers_from_system(system_param): """ @@ -194,7 +208,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): """ if isinstance(system_param, str): # If it's a string and starts with billing header, filter it out - if system_param.startswith("x-anthropic-billing-header:"): + if system_param.startswith(ANTHROPIC_BILLING_METADATA_PREFIX): return None return system_param elif isinstance(system_param, list): @@ -205,7 +219,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): text = content_block.get("text", "") content_type = content_block.get("type", "") # Skip text blocks that start with billing header - if content_type == "text" and text.startswith("x-anthropic-billing-header:"): + if content_type == "text" and text.startswith(ANTHROPIC_BILLING_METADATA_PREFIX): continue filtered_list.append(content_block) else: @@ -700,7 +714,8 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): # Auto-strip advisor blocks from history if advisor tool is absent. # Prevents Anthropic 400: advisor_tool_result in history requires advisor tool. - _tools: Final = anthropic_messages_optional_request_params.get("tools") or [] + tools: Final = anthropic_messages_optional_request_params.get("tools") + _tools: Final = tools or [] _has_advisor: Final = any(isinstance(t, dict) and t.get("type") == ANTHROPIC_ADVISOR_TOOL_TYPE for t in _tools) if not _has_advisor: messages = strip_advisor_blocks_from_messages(messages) @@ -713,7 +728,14 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): model=model, **anthropic_messages_optional_request_params, ) - return dict(anthropic_messages_request) + if not isinstance(tools, list): + return dict(anthropic_messages_request) + return { + **anthropic_messages_request, + "tools": self.tools_for_upstream( + OBJECT_LIST_ADAPTER.validate_python(tools), model, litellm_params.drop_params + ), + } def transform_anthropic_messages_response( self, diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index f78aa36c418..2bace3ebf3b 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -15,7 +15,11 @@ from litellm.integrations.anthropic_cache_control_hook import ( supports_openai_prompt_cache_breakpoint, ) from litellm.litellm_core_utils.core_helpers import drops_unsupported_params -from litellm.litellm_core_utils.prompt_templates.common_utils import cache_control_as_prompt_cache_breakpoint +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + cache_control_as_prompt_cache_breakpoint, + prompt_cache_breakpoint_for_wire, + with_prompt_cache_breakpoint, +) from litellm.litellm_core_utils.tool_fields import check_mcp_servers_droppable from litellm.responses.utils import auto_reasoning_summary from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS @@ -91,14 +95,28 @@ def _without_prompt_cache_breakpoint(block: Mapping[str, object]) -> Mapping[str return {key: value for key, value in block.items() if key != "prompt_cache_breakpoint"} +def _cache_control_as_wire_safe_prompt_cache_breakpoint(block: Mapping[str, object]) -> Mapping[str, object]: + if "prompt_cache_breakpoint" not in block: + return cache_control_as_prompt_cache_breakpoint(block) + wire_marker: Final = prompt_cache_breakpoint_for_wire(block["prompt_cache_breakpoint"], drop_params=True) + return cache_control_as_prompt_cache_breakpoint( + with_prompt_cache_breakpoint(_without_prompt_cache_breakpoint(block), wire_marker) + ) + + def _prompt_cache_block_rewrite(model: str, forwarded_kwargs: Mapping[str, object]) -> BlockRewrite | None: """Client cache marks become breakpoints for a deployment that reads them, and a model that takes no breakpoints drops raw ones under drop_params.""" + drops_params: Final = drops_unsupported_params(forwarded_kwargs.get("drop_params")) if _reads_prompt_cache_breakpoint(model, forwarded_kwargs): - return cache_control_as_prompt_cache_breakpoint + return ( + _cache_control_as_wire_safe_prompt_cache_breakpoint + if drops_params + else cache_control_as_prompt_cache_breakpoint + ) if _model_takes_prompt_cache_breakpoint(model, forwarded_kwargs): return None - if drops_unsupported_params(forwarded_kwargs.get("drop_params")): + if drops_params: return _without_prompt_cache_breakpoint return None diff --git a/litellm/llms/azure/azure.py b/litellm/llms/azure/azure.py index 5a181faad11..a1ba8de6d7f 100644 --- a/litellm/llms/azure/azure.py +++ b/litellm/llms/azure/azure.py @@ -33,6 +33,7 @@ from litellm.types.utils import ( from litellm.utils import ( CustomStreamWrapper, convert_to_model_response_object, + get_base_model_from_metadata, ) from ...types.llms.openai import HttpxBinaryResponseContent @@ -145,6 +146,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): azure_client: AzureOpenAI | OpenAI, data: dict, timeout: float | httpx.Timeout, + logging_obj: LiteLLMLoggingObj, ): """ Helper to: @@ -160,7 +162,12 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): except BadRequestError as e: if not is_output_token_limit_error(e): raise - return build_output_token_limit_response(e=e, data=data, is_async=False) + return build_output_token_limit_response( + e=e, + data=data, + is_async=False, + base_model=get_base_model_from_metadata(logging_obj.model_call_details), + ) except Exception as e: raise e @@ -192,7 +199,12 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): except BadRequestError as e: if not is_output_token_limit_error(e): raise - return build_output_token_limit_response(e=e, data=data, is_async=True) + return build_output_token_limit_response( + e=e, + data=data, + is_async=True, + base_model=get_base_model_from_metadata(logging_obj.model_call_details), + ) except Exception as e: raise e @@ -350,7 +362,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): ) headers, response = self.make_sync_azure_openai_chat_completion_request( - azure_client=azure_client, data=data, timeout=timeout + azure_client=azure_client, data=data, timeout=timeout, logging_obj=logging_obj ) if not isinstance(response, BaseModel): raise AzureOpenAIError( @@ -564,7 +576,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): }, ) headers, response = self.make_sync_azure_openai_chat_completion_request( - azure_client=azure_client, data=data, timeout=timeout + azure_client=azure_client, data=data, timeout=timeout, logging_obj=logging_obj ) logging_obj.model_call_details["response_headers"] = headers streamwrapper: Final = CustomStreamWrapper( diff --git a/litellm/llms/azure/chat/gpt_reasoning_transformation.py b/litellm/llms/azure/chat/gpt_reasoning_transformation.py index 5bd03f511c5..72a0500e0af 100644 --- a/litellm/llms/azure/chat/gpt_reasoning_transformation.py +++ b/litellm/llms/azure/chat/gpt_reasoning_transformation.py @@ -24,6 +24,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): GPT5_SERIES_ROUTE = "gpt5_series/" OPT_IN_REASONING_EFFORTS: ClassVar[frozenset[str]] = frozenset(("xhigh", "none")) + LOOKS_UP_BASE_MODEL: ClassVar[bool] = True @classmethod def _model_map_lookup_name(cls, model: str) -> str: @@ -88,7 +89,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): drop_params: bool, api_version: str = "", *, - declared_reasoning_efforts: Collection[str] = (), + declared_reasoning_efforts: Collection[str] | None = None, ) -> dict: return OpenAIGPT5Config.map_openai_params( self, diff --git a/litellm/llms/azure/chat/gpt_transformation.py b/litellm/llms/azure/chat/gpt_transformation.py index 26ae30635d4..990c8869de3 100644 --- a/litellm/llms/azure/chat/gpt_transformation.py +++ b/litellm/llms/azure/chat/gpt_transformation.py @@ -1,4 +1,4 @@ -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final @@ -15,6 +15,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.prompt_templates.factory import ( convert_to_azure_openai_messages, ) +from litellm.llms.base_llm.base_utils import map_developer_role_to_system_role from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.openai.chat.gpt_reasoning_transformation import GPT_REASONING_SERIES_MARKERS from litellm.types.llms.azure import ( @@ -98,6 +99,22 @@ class AzureOpenAIConfig(BaseConfig): if key != "self" and value is not None: setattr(self.__class__, key, value) + def translate_developer_role_to_system_role( + self, + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: + """ + Azure OpenAI accepts `system` messages at any position, so a developer message + only changes role. Declared here so `AzureOpenAIGPT5Config` resolves to it + ahead of the OpenAI-compatible hoist it also inherits. + """ + return map_developer_role_to_system_role(messages=messages) + @classmethod def get_config(cls): return super().get_config() diff --git a/litellm/llms/azure/common_utils.py b/litellm/llms/azure/common_utils.py index 4bd19889bcc..6bb6d584385 100644 --- a/litellm/llms/azure/common_utils.py +++ b/litellm/llms/azure/common_utils.py @@ -621,7 +621,7 @@ class BaseAzureLLM(BaseOpenAILLM): # For Azure v1 API, use standard OpenAI client instead of AzureOpenAI # See: https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#api-specs - if self._is_azure_v1_api_version(api_version): + if self.is_azure_v1_api_version(api_version): # Extract only params that OpenAI client accepts # Always use /openai/v1/ regardless of whether user passed "v1", "latest", or "preview" # The OpenAI client accepts a callable for `api_key` and re-invokes it @@ -922,7 +922,7 @@ class BaseAzureLLM(BaseOpenAILLM): else: new_url = api_base - if BaseAzureLLM._is_azure_v1_api_version(api_version): + if BaseAzureLLM.is_azure_v1_api_version(api_version): # ensure the request go to /openai/v1 and not just /openai if "/openai/v1" not in new_url: parsed_url: Final = httpx.URL(new_url) @@ -952,7 +952,7 @@ class BaseAzureLLM(BaseOpenAILLM): Returns None when ``api_version`` is a dated one, which still uses the deployment route. """ - if not BaseAzureLLM._is_azure_v1_api_version(api_version): + if not BaseAzureLLM.is_azure_v1_api_version(api_version): return None base_url: Final = httpx.URL(api_base) @@ -970,7 +970,7 @@ class BaseAzureLLM(BaseOpenAILLM): ) @staticmethod - def _is_azure_v1_api_version(api_version: str | None) -> bool: + def is_azure_v1_api_version(api_version: str | None) -> bool: if api_version is None: return False return api_version in {"preview", "latest", "v1"} diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index 647ad97e02c..09777da8669 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -54,6 +54,16 @@ def _resolve_responses_api_version(api_base: str | None, litellm_params: Mapping return api_version +def _v1_responses_url(api_base: str) -> httpx.URL: + parsed_url: Final = httpx.URL(api_base) + path: Final = parsed_url.path.rstrip("/") + root: Final = next( + (path.removesuffix(route) for route in ("/openai/v1/responses", "/openai/responses") if path.endswith(route)), + path, + ) + return parsed_url.copy_with(path=f"{root}/openai/v1/responses", query=None) + + def _with_responses_api_version(api_base: str | None, litellm_params: Mapping[str, object]) -> Mapping[str, object]: return MappingProxyType({**litellm_params, "api_version": _resolve_responses_api_version(api_base, litellm_params)}) @@ -132,7 +142,9 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): return filtered_item return item - def _validate_input_param(self, input: str | ResponseInputParam) -> str | ResponseInputParam: + def _validate_input_param( + self, input: str | ResponseInputParam, *, drops_set_created_by: bool = False + ) -> str | ResponseInputParam: """ Override parent method to also filter out 'status' field from message items. Azure OpenAI API does not accept 'status' field in input messages. @@ -140,7 +152,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): from typing import cast # First call parent's validation - validated_input: Final = super()._validate_input_param(input) + validated_input: Final = super()._validate_input_param(input, drops_set_created_by=drops_set_created_by) # Then filter out status from message items if isinstance(validated_input, list): @@ -247,20 +259,20 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): if api_base is None: raise ValueError("api_base is required for Azure WebSocket") - parsed_url: Final = httpx.URL(api_base) - path = parsed_url.path.rstrip("/") - # Strip existing /openai/responses path if the api_base already contains it - for suffix in ("/openai/v1/responses", "/openai/responses"): - if path.endswith(suffix): - path = path[: -len(suffix)] - break - scheme: Final = "wss" if parsed_url.scheme == "https" else "ws" - return str(parsed_url.copy_with(scheme=scheme, path=f"{path}/openai/v1/responses", query=None)) + v1_url: Final = _v1_responses_url(api_base) + return str(v1_url.copy_with(scheme="wss" if v1_url.scheme == "https" else "ws")) def model_in_websocket_url(self) -> bool: # Azure sends the model in the response.create body, not the URL return False + def compact_url(self, api_base: str) -> str: + params: Final = httpx.URL(api_base).params + v1_params: Final = ( + params if BaseAzureLLM.is_azure_v1_api_version(params.get("api-version")) else params.remove("api-version") + ) + return super().compact_url(str(_v1_responses_url(api_base).copy_with(params=v1_params))) + ######################################################### ########## DELETE RESPONSE API TRANSFORMATION ############## ######################################################### @@ -323,7 +335,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, - ) -> tuple[str, dict]: + ) -> tuple[str, dict[str, str]]: """ Transform the get response API request into a URL and data @@ -331,7 +343,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): - GET /v1/responses/{response_id} """ get_url: Final = self._construct_url_for_response_id_in_path(api_base=api_base, response_id=response_id) - data: Final[dict] = {} + data: Final[dict[str, str]] = {} verbose_logger.debug("get response url=%s", get_url) return get_url, data diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py index 0f8afaae681..4b3f263201a 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py @@ -13,6 +13,7 @@ from pydantic import JsonValue import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import asyncify +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.llms.anthropic.common_utils import AnthropicError from litellm.llms.azure_ai.anthropic.count_tokens.transformation import ( AzureAIAnthropicCountTokensConfig, @@ -69,6 +70,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig): tools=tools, system=system, optional_params=optional_params, + drop_params=drops_unsupported_params((litellm_params or {}).get("drop_params")), ) verbose_logger.debug("Transformed request: %s", request_body) diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 6aed7eb3c54..8bb6072d0a9 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -106,7 +106,7 @@ class AzureAIStudioConfig(OpenAIConfig): model: str, drop_params: bool, *, - declared_reasoning_efforts: Collection[str] = (), + declared_reasoning_efforts: Collection[str] | None = None, ) -> dict[str, object]: # mutable-ok: OpenAIConfig.map_openai_params signature if not azureAIGPT5Config.is_model_gpt_5_model(model): return super().map_openai_params( diff --git a/litellm/llms/azure_ai/common_utils.py b/litellm/llms/azure_ai/common_utils.py index 65d73363b30..147bf69e91e 100644 --- a/litellm/llms/azure_ai/common_utils.py +++ b/litellm/llms/azure_ai/common_utils.py @@ -140,6 +140,15 @@ def get_azure_ai_auth_headers( AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY: Final = "azure_model_router_selected_model" +FOUNDRY_CLASSIC_AGENT_ID_PREFIX: Final = "asst_" + + +def is_foundry_project_agent(model: str | None) -> bool: + if model is None or AzureFoundryModelInfo.get_azure_ai_route(model) != "agents": + return False + return not model.split("agents/", 1)[1].startswith(FOUNDRY_CLASSIC_AGENT_ID_PREFIX) + + def azure_ai_supports_native_responses(model: str | None, api_base: str | None) -> bool: resolved_base: Final = AzureFoundryModelInfo.get_api_base(api_base) if resolved_base is not None and not is_azure_openai_v1_host(resolved_base): diff --git a/litellm/llms/azure_ai/passthrough/transformation.py b/litellm/llms/azure_ai/passthrough/transformation.py index 35b7642d7de..2e5514b3975 100644 --- a/litellm/llms/azure_ai/passthrough/transformation.py +++ b/litellm/llms/azure_ai/passthrough/transformation.py @@ -5,8 +5,9 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Final import httpx -from pydantic import TypeAdapter, ValidationError +from pydantic import ConfigDict, TypeAdapter, ValidationError +from litellm._lazy_imports import get_default_encoding from litellm._logging import verbose_logger from litellm.llms.azure_ai.common_utils import ( AzureFoundryModelInfo, @@ -22,10 +23,15 @@ from litellm.llms.base_llm.passthrough.transformation import ( relayed_body, strip_leading_model_segment, ) +from litellm.llms.cohere.embed.transformation import ( + CohereEmbeddingRowShape, + CohereImageTokenBilling, + cohere_embedding_response, +) from litellm.rust_bridge.ocr.entrypoints import NATIVE_OCR_PASSTHROUGH_RESPONSE, NativeOcrPassthroughResponse from litellm.types.llms.openai import AllMessageValues from litellm.types.rerank import RerankResponse -from litellm.types.utils import CallTypes, ImageResponse, StandardPassThroughResponseObject +from litellm.types.utils import CallTypes, EmbeddingResponse, ImageResponse, StandardPassThroughResponseObject if TYPE_CHECKING: from httpx import URL, Response @@ -35,6 +41,7 @@ if TYPE_CHECKING: EMPTY_QUERY: Final[Mapping[str, object]] = MappingProxyType({}) +ECHOED_TEXTS: Final = TypeAdapter(tuple[str, ...], config=ConfigDict(hide_input_in_errors=True)) def api_version_from(litellm_params: Mapping[str, object]) -> str | None: @@ -85,9 +92,29 @@ def relay_query_params( return MappingProxyType({**(request_query_params or EMPTY_QUERY), "api-version": api_version}) +def echoed_texts(body: Mapping[str, object]) -> tuple[str, ...]: + try: + return ECHOED_TEXTS.validate_python(body.get("texts")) + except ValidationError: + return () + + +def cohere_embed_relay_response(body: Mapping[str, object]) -> EmbeddingResponse: + return cohere_embedding_response( + body, + None, + get_default_encoding(), + echoed_texts(body), + CohereImageTokenBilling.FOLDED_INTO_INPUT_TOKENS, + CohereEmbeddingRowShape.TYPED, + ) + + FOUNDRY_RELAY_SHAPES: Final = ( RelayShape("/rerank", CallTypes.arerank, RerankResponse.model_validate), RelayShape("/providers/blackforestlabs/v1/flux-2-pro", CallTypes.aimage_generation, ImageResponse.model_validate), + RelayShape("/providers/cohere/v1/embed", CallTypes.aembedding, cohere_embed_relay_response), + RelayShape("/providers/cohere/v2/embed", CallTypes.aembedding, cohere_embed_relay_response), ) diff --git a/litellm/llms/azure_ai/responses/transformation.py b/litellm/llms/azure_ai/responses/transformation.py index d23de05e148..b19915648ab 100644 --- a/litellm/llms/azure_ai/responses/transformation.py +++ b/litellm/llms/azure_ai/responses/transformation.py @@ -1,26 +1,64 @@ -from typing import Final +import os +from collections.abc import Collection, Mapping, Sequence +from typing import TYPE_CHECKING, Final, NoReturn import httpx +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError +import litellm +from litellm.exceptions import UnsupportedParamsError from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig from litellm.llms.azure_ai.chat.transformation import AzureAIGPT5Config from litellm.llms.azure_ai.common_utils import ( + AZURE_AI_AGENTS_SCOPE, AzureFoundryModelInfo, api_key_header_for_base, get_azure_ai_auth_headers, ) +from litellm.types.llms.azure_ai import FoundryAgentReference +from litellm.types.llms.openai import ( + ResponseInputParam, + ResponsesAPIOptionalRequestParams, + ResponsesAPIResponse, + ResponsesAPIStreamingResponse, +) from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + _PROJECT_PATH_PREFIX: Final = ("api", "projects") _RESPONSES_PATH: Final = ("openai", "v1", "responses") +_AGENT_OWNED_PARAMS: Final = frozenset({"temperature", "top_p", "tools", "text", "reasoning"}) +_CALLER_AGENT_KEYS: Final = ("agent", "agent_reference") +_RESPONSE_FIELDS: Final = TypeAdapter(dict[str, object]) + + +def _project_root(base_url: httpx.URL) -> tuple[str, ...]: + segments: Final = tuple(segment for segment in base_url.path.split("/") if segment) + return segments[:3] if segments[:2] == _PROJECT_PATH_PREFIX else () def _responses_url(api_base: str) -> str: base_url: Final = httpx.URL(api_base) - segments: Final = tuple(segment for segment in base_url.path.split("/") if segment) - project_root: Final = segments[:3] if segments[:2] == _PROJECT_PATH_PREFIX else () - return str(base_url.copy_with(path="/" + "/".join((*project_root, *_RESPONSES_PATH)), query=None)) + return str(base_url.copy_with(path="/" + "/".join((*_project_root(base_url), *_RESPONSES_PATH)), query=None)) + + +def _without_agent_instructions(response: ResponsesAPIResponse) -> ResponsesAPIResponse: + if (response.model_extra or {}).get("agent_reference") is None: + return response + return response.model_copy(update={"instructions": None}) + + +def _event_without_agent_instructions(event: Mapping[str, object]) -> Mapping[str, object]: + response: Final = event.get("response") + if not isinstance(response, Mapping): + return event + fields: Final = _RESPONSE_FIELDS.validate_python(response) + if fields.get("agent_reference") is None: + return event + return {**event, "response": {**fields, "instructions": None}} class AzureAIResponsesAPIConfig(AzureOpenAIResponsesAPIConfig): @@ -42,10 +80,19 @@ class AzureAIResponsesAPIConfig(AzureOpenAIResponsesAPIConfig): def validate_environment(self, headers: dict, model: str, litellm_params: GenericLiteLLMParams | None) -> dict: params: Final = litellm_params or GenericLiteLLMParams() + api_base: Final = AzureFoundryModelInfo.get_api_base(params.api_base) + on_project: Final = api_base is not None and bool(_project_root(httpx.URL(api_base))) + auth_params: Final = ( + params.model_copy( + update={"azure_scope": params.azure_scope or os.getenv("AZURE_SCOPE") or AZURE_AI_AGENTS_SCOPE} + ) + if on_project + else params + ) auth_headers: Final = get_azure_ai_auth_headers( api_key=AzureFoundryModelInfo.get_api_key(params.api_key), - litellm_params=params.model_dump(), - api_key_header=api_key_header_for_base(AzureFoundryModelInfo.get_api_base(params.api_base)), + litellm_params=auth_params.model_dump(), + api_key_header="Authorization" if on_project else api_key_header_for_base(api_base), ) return { **headers, @@ -64,3 +111,207 @@ class AzureAIResponsesAPIConfig(AzureOpenAIResponsesAPIConfig): "Set the api_base parameter or the AZURE_AI_API_BASE environment variable." ) return _responses_url(resolved_base) + + def transform_response_api_response( + self, model: str, raw_response: httpx.Response, logging_obj: "LiteLLMLoggingObj" + ) -> ResponsesAPIResponse: + return _without_agent_instructions(super().transform_response_api_response(model, raw_response, logging_obj)) + + def transform_streaming_response( + self, + model: str, + parsed_chunk: Mapping[str, object], + logging_obj: "LiteLLMLoggingObj", + ) -> ResponsesAPIStreamingResponse: + return super().transform_streaming_response( # pyright: ignore[reportUnknownMemberType] # parent types parsed_chunk as a bare dict + model=model, parsed_chunk=dict(_event_without_agent_instructions(parsed_chunk)), logging_obj=logging_obj + ) + + def transform_get_response_api_response( + self, raw_response: httpx.Response, logging_obj: "LiteLLMLoggingObj" + ) -> ResponsesAPIResponse: + return _without_agent_instructions(super().transform_get_response_api_response(raw_response, logging_obj)) + + def transform_cancel_response_api_response( + self, raw_response: httpx.Response, logging_obj: "LiteLLMLoggingObj" + ) -> ResponsesAPIResponse: + return _without_agent_instructions(super().transform_cancel_response_api_response(raw_response, logging_obj)) + + +def _agent_reference(model: str) -> FoundryAgentReference: + name, _, version = model.split("agents/", 1)[1].partition(":") + if version: + return FoundryAgentReference(type="agent_reference", name=name, version=version) + return FoundryAgentReference(type="agent_reference", name=name) + + +class _CallerAgentReference(BaseModel): + model_config = ConfigDict(frozen=True, coerce_numbers_to_str=True) + + name: str + version: str | None = None + + +def _caller_agent_reference(value: object) -> _CallerAgentReference | None: + try: + return _CallerAgentReference.model_validate(value) + except ValidationError: + return None + + +def _requested_agent( + deployment_agent: FoundryAgentReference, caller_agents: Sequence[_CallerAgentReference | None] +) -> FoundryAgentReference: + requested_versions: Final = frozenset( + agent.version + for agent in caller_agents + if agent is not None and agent.name == deployment_agent["name"] and agent.version is not None + ) + if "version" in deployment_agent or len(requested_versions) != 1: + return deployment_agent + return FoundryAgentReference( + type="agent_reference", name=deployment_agent["name"], version=next(iter(requested_versions)) + ) + + +def _caller_agent_fields(fields: Mapping[str, object]) -> tuple[tuple[str, object], ...]: + return tuple((key, fields[key]) for key in _CALLER_AGENT_KEYS if key in fields) + + +def _deployment_agent( + model: str, caller_fields: Sequence[tuple[str, object]], litellm_params: GenericLiteLLMParams +) -> FoundryAgentReference: + caller_keys: Final = tuple(key for key, _ in caller_fields) + caller_agents: Final = tuple(_caller_agent_reference(value) for _, value in caller_fields) + agent: Final = _requested_agent(_agent_reference(model), caller_agents) + other_agents: Final = tuple( + key + for key, caller_agent in zip(caller_keys, caller_agents) + if caller_agent is None + or caller_agent.name != agent["name"] + or caller_agent.version not in (None, agent.get("version")) + ) + if other_agents and not (litellm.drop_params is True or litellm_params.drop_params is True): + raise UnsupportedParamsError( + message=( + f"This model entry always calls Foundry agent {agent['name']!r}, so `{other_agents[0]}` " + "in the request cannot pick another agent or version. Remove it, call the model entry for that " + "agent, or set `drop_params: true` to drop it" + ), + llm_provider=LlmProviders.AZURE_AI.value, + model=model, + ) + return agent + + +_INPUT_ITEMS: Final = TypeAdapter(list[object]) + + +def _with_system_prompt(instructions: str, request_input: object) -> Sequence[object]: + conversation: Final = ( + [{"type": "message", "role": "user", "content": request_input}] + if isinstance(request_input, str) + else _INPUT_ITEMS.validate_python(request_input) + ) + return [{"type": "message", "role": "system", "content": instructions}, *conversation] + + +class AzureAIAgentResponsesAPIConfig(AzureAIResponsesAPIConfig): + def should_fake_stream( + self, model: str | None, stream: bool | None, custom_llm_provider: str | None = None + ) -> bool: + return False + + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: override contract + inherited: Final[list[str]] = super().get_supported_openai_params(model) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # parent returns a bare list # mutable-ok: same + return [param for param in inherited if param not in _AGENT_OWNED_PARAMS] + + def map_openai_params_for_deployment( + self, + response_api_optional_params: ResponsesAPIOptionalRequestParams, + model: str, + drop_params: bool, + *, + declared_reasoning_efforts: Collection[str] | None, + ) -> Mapping[str, object]: + mapped: Final = super().map_openai_params_for_deployment( + response_api_optional_params, model, drop_params, declared_reasoning_efforts=declared_reasoning_efforts + ) + resends_instructions: Final = bool(mapped.get("instructions")) and bool(mapped.get("previous_response_id")) + if resends_instructions and not drop_params: + raise UnsupportedParamsError( + message=( + "azure_ai Foundry agents store a turn's system message with the conversation, so `instructions` " + "with `previous_response_id` would add another copy each turn. Send `instructions` on the first " + "turn only, or set `drop_params: true` to drop it" + ), + llm_provider=LlmProviders.AZURE_AI.value, + model=model, + ) + dropped: Final = _AGENT_OWNED_PARAMS | ( + frozenset({"instructions"}) if resends_instructions else frozenset[str]() + ) + return {key: value for key, value in mapped.items() if key not in dropped} + + def transform_responses_api_request( + self, + model: str, + input: str | ResponseInputParam, + response_api_optional_request_params: dict[str, object], # mutable-ok: signature fixed by the override contract + litellm_params: GenericLiteLLMParams, + headers: dict[str, str], # mutable-ok: same + ) -> dict[str, object]: # mutable-ok: same + request: Final[dict[str, object]] = super().transform_responses_api_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # parent returns a bare dict # mutable-ok: same + model=model, + input=input, + response_api_optional_request_params=response_api_optional_request_params, + litellm_params=litellm_params, + headers=headers, + ) + instructions: Final = request.get("instructions") + return { + **{key: value for key, value in request.items() if key not in ("model", "instructions")}, + **( + {"input": _with_system_prompt(instructions, request.get("input"))} + if isinstance(instructions, str) and instructions + else {} + ), + "agent_reference": _deployment_agent( + model, _caller_agent_fields(litellm_params.model_extra or {}), litellm_params + ), + } + + def transform_extra_body( + self, + extra_body: Mapping[str, object], + request: Mapping[str, object], + model: str, + litellm_params: GenericLiteLLMParams, + ) -> Mapping[str, object]: + passthrough: Final = {key: value for key, value in extra_body.items() if key not in _CALLER_AGENT_KEYS} + if len(passthrough) == len(extra_body): + return passthrough + return { + **passthrough, + "agent_reference": _deployment_agent( + model, (*_caller_agent_fields(request), *_caller_agent_fields(extra_body)), litellm_params + ), + } + + def transform_compact_response_api_request( + self, + model: str, + input: str | ResponseInputParam, + response_api_optional_request_params: dict[str, object], # mutable-ok: signature fixed by the override contract + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict[str, str], # mutable-ok: same + ) -> NoReturn: + raise UnsupportedParamsError( + message=( + "azure_ai Foundry agents cannot be compacted, because the project's /responses/compact endpoint " + "takes a model deployment, not an agent. Call compact on a model entry for that deployment instead" + ), + llm_provider=LlmProviders.AZURE_AI.value, + model=model, + ) diff --git a/litellm/llms/base_llm/base_utils.py b/litellm/llms/base_llm/base_utils.py index a9ec1f97d20..ef8d080027f 100644 --- a/litellm/llms/base_llm/base_utils.py +++ b/litellm/llms/base_llm/base_utils.py @@ -10,14 +10,23 @@ from abc import ABC, abstractmethod from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType -from typing import Any, Final +from typing import Any, Final, TypeAlias from openai.lib import _parsing, _pydantic # pyright: ignore[reportPrivateUsage] # SDK parser internals from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError +from typing_extensions import TypeIs # noqa: TID251 # narrows a message to its system member by the role literal from litellm._logging import verbose_logger +from litellm.constants import ANTHROPIC_BILLING_METADATA_PREFIX from litellm.litellm_core_utils.core_helpers import as_str_mapping -from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionCachedContent, + ChatCompletionSystemMessage, + ChatCompletionTextObject, + ChatCompletionToolCallChunk, + ChatCompletionUserMessage, +) from litellm.types.utils import ( CountContextManagement, Message, @@ -458,7 +467,7 @@ def type_to_response_format_param( def map_developer_role_to_system_role( - messages: list[AllMessageValues], + messages: Sequence[AllMessageValues], ) -> list[AllMessageValues]: """ Translate `developer` role to `system` role for non-OpenAI providers. @@ -473,3 +482,226 @@ def map_developer_role_to_system_role( else: new_messages.append(m) return new_messages + + +SystemMessageContent: TypeAlias = str | list[object] + +_system_content_adapter: Final = TypeAdapter[SystemMessageContent | None](SystemMessageContent | None) + + +def _system_content(message: ChatCompletionSystemMessage) -> SystemMessageContent | None: + if "content" not in message: + return "" + try: + content: Final = _system_content_adapter.validate_python(message["content"]) + except ValidationError: + return None + return "" if content is None else content + + +def _message_cache_control(message: ChatCompletionSystemMessage) -> ChatCompletionCachedContent | None: + if "cache_control" not in message: + return None + return message["cache_control"] + + +_SystemPart: TypeAlias = tuple[SystemMessageContent, ChatCompletionCachedContent | None] + + +def _with_message_cache_control( + blocks: Sequence[object], cache_control: ChatCompletionCachedContent | None +) -> tuple[object, ...]: + last: Final = as_str_mapping(blocks[-1]) if blocks else None + if last is None or cache_control is None or "cache_control" in last: + return tuple(blocks) + return (*blocks[:-1], {**last, "cache_control": cache_control}) + + +def _as_system_message(message: AllMessageValues) -> AllMessageValues: + if message["role"] != "developer": + return message + content_only: Final[ChatCompletionSystemMessage] = {"role": "system", "content": message["content"]} + named: Final[ChatCompletionSystemMessage] = ( + content_only if "name" not in message else {**content_only, "name": message["name"]} + ) + if "cache_control" not in message: + return named + cached: Final[ChatCompletionSystemMessage] = {**named, "cache_control": message["cache_control"]} + return cached + + +def _text_blocks( + content: SystemMessageContent, cache_control: ChatCompletionCachedContent | None +) -> tuple[object, ...]: + if not isinstance(content, str): + return _with_message_cache_control(content, cache_control) + if not content: + return () + if cache_control: + cached_block: Final[ChatCompletionTextObject] = { + "type": "text", + "text": content, + "cache_control": cache_control, + } + return (cached_block,) + text_block: Final[ChatCompletionTextObject] = {"type": "text", "text": content} + return (text_block,) + + +def _lone_text_block_text(blocks: Sequence[object]) -> str | None: + match blocks: + case [{"type": "text", "text": str() as text} as block] if block == {"type": "text", "text": text}: + return text + case _: + return None + + +def _plain_text(content: SystemMessageContent, cache_control: ChatCompletionCachedContent | None) -> str | None: + if cache_control: + return None + text: Final = content if isinstance(content, str) else _lone_text_block_text(content) + if text is None or text.startswith(ANTHROPIC_BILLING_METADATA_PREFIX): + return None + return text + + +def _is_system_message( + message: AllMessageValues, +) -> TypeIs[ChatCompletionSystemMessage]: # guard-ok: the role literal picks the TypedDict member + return message["role"] == "system" + + +def _with_leading_separator(blocks: tuple[object, ...]) -> tuple[object, ...]: + first: Final = as_str_mapping(blocks[0]) + text: Final = None if first is None else first.get("text") + if first is None or not isinstance(text, str) or text.startswith(ANTHROPIC_BILLING_METADATA_PREFIX): + return blocks + return ({**first, "text": f"\n\n{text}"}, *blocks[1:]) + + +def _merged_system_content(parts: Sequence[_SystemPart]) -> SystemMessageContent: + texts: Final = tuple(itertools.starmap(_plain_text, parts)) + if all(text is not None for text in texts): + return "\n\n".join(text for text in texts if text) + block_runs: Final = tuple(blocks for blocks in itertools.starmap(_text_blocks, parts) if blocks) + separated: Final = (*block_runs[:1], *map(_with_leading_separator, block_runs[1:])) + return list(itertools.chain.from_iterable(separated)) + + +def _merged_system_message( + run: Sequence[ChatCompletionSystemMessage], parts: Sequence[_SystemPart] +) -> ChatCompletionSystemMessage: + content: Final = _merged_system_content(parts) + named: Final = next((message for message in reversed(run) if "name" in message), None) + if named is None: + merged: Final[ChatCompletionSystemMessage] = {"role": "system", "content": content} + return merged + with_name: Final[ChatCompletionSystemMessage] = {"role": "system", "content": content, "name": named["name"]} + return with_name + + +def _merged_run(run: tuple[AllMessageValues, ...]) -> tuple[AllMessageValues, ...]: + system_run: Final = tuple(message for message in run if _is_system_message(message)) + if len(system_run) < 2: + return run + contents: Final = tuple((_system_content(message), _message_cache_control(message)) for message in system_run) + parts: Final = tuple((content, cache_control) for content, cache_control in contents if content is not None) + if len(parts) != len(contents): + return run + return (_merged_system_message(system_run, parts),) + + +def _merged_system_runs(messages: Sequence[AllMessageValues]) -> Iterator[AllMessageValues]: + runs: Final = (tuple(run) for _, run in itertools.groupby(messages, key=_is_system_message)) + return itertools.chain.from_iterable(map(_merged_run, runs)) + + +_INSTRUCTION_ROLES: Final = frozenset(("system", "developer")) + + +def _leading_instruction_block_length(roles: Sequence[str | None]) -> int: + return next((index for index, role in enumerate(roles) if role not in _INSTRUCTION_ROLES), len(roles)) + + +def _closing_instruction_block_start(roles: Sequence[str | None], leading_length: int) -> int: + last_conversation_index: Final = next( + (index for index in range(len(roles) - 1, leading_length - 1, -1) if roles[index] not in _INSTRUCTION_ROLES), + None, + ) + if last_conversation_index is None or roles[last_conversation_index] != "assistant": + return len(roles) + return last_conversation_index + 1 + + +def hoisted_instruction_indices(roles: Sequence[str | None]) -> tuple[int, ...]: + """ + Indices of the leading instruction block plus every developer item after it, except + a developer block that closes the conversation right after an assistant turn, which + stays in place so the request does not end on the assistant's turn. Items without a + role (tool calls and their outputs) count as conversation. + """ + leading_length: Final = _leading_instruction_block_length(roles) + closing_start: Final = _closing_instruction_block_start(roles, leading_length) + moved: Final = tuple(index for index in range(leading_length, closing_start) if roles[index] == "developer") + return (*range(leading_length), *moved) + + +def closing_instruction_indices(roles: Sequence[str | None]) -> frozenset[int]: + closing_start: Final = _closing_instruction_block_start(roles, _leading_instruction_block_length(roles)) + return frozenset(range(closing_start, len(roles))) + + +def _role_name(role: object) -> str | None: + return role if isinstance(role, str) else None + + +def _message_roles(messages: Sequence[AllMessageValues]) -> tuple[str | None, ...]: + return tuple(_role_name(message["role"]) for message in messages) + + +def _hoisted_message_order(messages: Sequence[AllMessageValues]) -> tuple[int, ...]: + hoisted: Final = hoisted_instruction_indices(_message_roles(messages)) + hoisted_set: Final = frozenset(hoisted) + return (*hoisted, *(index for index in range(len(messages)) if index not in hoisted_set)) + + +def _developer_message_as_user(message: AllMessageValues) -> AllMessageValues: + if message["role"] != "developer": + return message + copied: Final = copy.deepcopy(message) + if "cache_control" not in copied: + as_user: Final[ChatCompletionUserMessage] = {"role": "user", "content": copied["content"]} + return as_user + cached_as_user: Final[ChatCompletionUserMessage] = { + "role": "user", + "content": copied["content"], + "cache_control": copied["cache_control"], + } + return cached_as_user + + +def merge_system_message_runs(messages: Sequence[AllMessageValues]) -> Sequence[AllMessageValues]: + """ + Translate `developer` role to `system` role in place and fold each run of + consecutive system messages into one, for chat templates that render a single + system message per run. + """ + return tuple(_merged_system_runs(tuple(map(_as_system_message, messages)))) + + +def hoist_developer_messages_into_leading_system_message( + messages: Sequence[AllMessageValues], +) -> Sequence[AllMessageValues]: + """ + Place instructions for an OpenAI-compatible model whose chat template allows a single + system message and only at the start: developer messages after the first user turn move + into the leading system block, a developer message that closes the conversation right + after an assistant turn becomes a user turn so the request does not end on the + assistant's turn, and each run of consecutive system messages folds into one. + """ + order: Final = _hoisted_message_order(messages) + closing: Final = closing_instruction_indices(_message_roles(messages)) + placed: Final = tuple( + _developer_message_as_user(messages[index]) if index in closing else messages[index] for index in order + ) + return merge_system_message_runs(placed) diff --git a/litellm/llms/base_llm/chat/transformation.py b/litellm/llms/base_llm/chat/transformation.py index 207a4705c92..b87a783e244 100644 --- a/litellm/llms/base_llm/chat/transformation.py +++ b/litellm/llms/base_llm/chat/transformation.py @@ -4,7 +4,7 @@ Common base config for all LLM providers import types from abc import ABC, abstractmethod -from collections.abc import AsyncIterator, Iterator, Mapping +from collections.abc import AsyncIterator, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, Final, Union import httpx @@ -199,12 +199,19 @@ class BaseConfig(ABC): def translate_developer_role_to_system_role( self, - messages: list[AllMessageValues], - ) -> list[AllMessageValues]: + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: """ Translate `developer` role to `system` role for non-OpenAI providers. - Overriden by OpenAI/Azure + Overridden by OpenAI, which also places the instructions in one leading system + message for an OpenAI-compatible model declared to reject a later system + message, and by the o-series and Azure configs. """ return map_developer_role_to_system_role(messages=messages) diff --git a/litellm/llms/base_llm/responses/transformation.py b/litellm/llms/base_llm/responses/transformation.py index 01baeed9bef..792b14090b8 100644 --- a/litellm/llms/base_llm/responses/transformation.py +++ b/litellm/llms/base_llm/responses/transformation.py @@ -69,6 +69,9 @@ class BaseResponsesAPIConfig(ABC): def supports_encrypted_agent_messages(self) -> bool: return False + def supports_streamed_retrieval(self) -> bool: + return self.custom_llm_provider in (LlmProviders.OPENAI, LlmProviders.AZURE, LlmProviders.AZURE_AI) + def sign_request( self, headers: dict, @@ -209,7 +212,7 @@ class BaseResponsesAPIConfig(ABC): api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, - ) -> tuple[str, dict]: + ) -> tuple[str, dict[str, str]]: pass @abstractmethod @@ -268,6 +271,9 @@ class BaseResponsesAPIConfig(ABC): """Returns True if litellm should fake a stream for the given model and stream value""" return False + def may_answer_stream_with_json(self) -> bool: + return False + def supports_native_websocket(self) -> bool: """ Returns True if the provider has a native WebSocket endpoint for Responses API. diff --git a/litellm/llms/bedrock/audio_transcription/__init__.py b/litellm/llms/bedrock/audio_transcription/__init__.py index 7f7a138398b..bc74e9c38ab 100644 --- a/litellm/llms/bedrock/audio_transcription/__init__.py +++ b/litellm/llms/bedrock/audio_transcription/__init__.py @@ -1,7 +1,11 @@ import base64 +import math +from collections.abc import Mapping +from dataclasses import dataclass from typing import Final import httpx +from pydantic import BaseModel, ConfigDict from litellm.litellm_core_utils.audio_utils.utils import process_audio_file from litellm.rust_bridge import runtime @@ -14,7 +18,72 @@ from litellm.rust_bridge.transcription.native import ( RustAtranscription, RustTranscription, ) -from litellm.types.utils import FileTypes, TranscriptionResponse +from litellm.secret_managers.main import get_secret_str +from litellm.types.utils import FileTypes, TranscriptionResponse, TranscriptionUsageTokensObject + +SUPPORTED_OPENAI_TRANSCRIPTION_PARAMS: Final = ("language", "prompt", "temperature", "response_format") +TEXT_RESPONSE_FORMATS: Final = frozenset({"json", "text"}) + + +def _as_float(value: object) -> float | None: + if isinstance(value, bool) or not isinstance(value, (int, float, str)): + return None + try: + number: Final = float(value) + except ValueError: + return None + return number if math.isfinite(number) else None + + +def _rejection(key: str, value: object) -> str | None: + if key not in SUPPORTED_OPENAI_TRANSCRIPTION_PARAMS: + return f"bedrock transcription does not support {key}." + if key == "response_format" and value not in TEXT_RESPONSE_FORMATS: + return f"bedrock transcription returns only the transcript text, so response_format={value!r} is not supported." + if key == "temperature" and _as_float(value) is None: + return f"bedrock transcription needs a number for temperature, got {value!r}." + return None + + +@dataclass(frozen=True, slots=True) +class UnsupportedTranscriptionParams: + message: str + + +def map_openai_transcription_params( + non_default_params: Mapping[str, object], drop_params: bool +) -> Mapping[str, object] | UnsupportedTranscriptionParams: + rejections: Final = { + key: reason for key, value in non_default_params.items() if (reason := _rejection(key, value)) is not None + } + if rejections and not drop_params: + return UnsupportedTranscriptionParams( + message=f"{' '.join(rejections.values())} To drop it from the call, set `litellm.drop_params = True`." + ) + return { + key: _as_float(value) if key == "temperature" else value + for key, value in non_default_params.items() + if key not in rejections + } + + +class _RustTranscriptionResult(BaseModel): + model_config = ConfigDict(frozen=True) + + text: str + usage: TranscriptionUsageTokensObject | None = None + + +def _transcription_response( + result: Mapping[str, object], optional_params: Mapping[str, object] +) -> TranscriptionResponse: + parsed: Final = _RustTranscriptionResult.model_validate(result) + response: Final = TranscriptionResponse(text=parsed.text).model_copy(update={"usage": parsed.usage}) + region_name: Final = ( + optional_params.get("aws_region_name") or get_secret_str("AWS_REGION_NAME") or get_secret_str("AWS_REGION") + ) + response.hidden_params = {**response.hidden_params, "region_name": region_name} + return response class BedrockAudioTranscriptionRustDispatch: @@ -64,7 +133,7 @@ class BedrockAudioTranscriptionRustDispatch: "timeout_seconds": timeout_to_seconds(timeout), } call: Final = NativeCall(args=(), kwargs=fields, base={}) - return TranscriptionResponse(**rust(call)) + return _transcription_response(rust(call), optional_params) return runtime.run( RouteContext(Route.TRANSCRIPTION, provider=custom_llm_provider, model=model), @@ -97,7 +166,7 @@ class BedrockAudioTranscriptionRustDispatch: "timeout_seconds": timeout_to_seconds(timeout), } call: Final = NativeCall(args=(), kwargs=fields, base={}) - return TranscriptionResponse(**await rust(call)) + return _transcription_response(await rust(call), optional_params) return await runtime.arun( RouteContext(Route.TRANSCRIPTION, provider=custom_llm_provider, model=model), diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index abf81368e0a..462f5dfe7e8 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -118,6 +118,7 @@ from litellm.utils import ( ) from ..common_utils import ( + DROP_DISABLED_REASONING_EFFORT_WARNING, BedrockError, BedrockModelInfo, bedrock_cache_point_ttl_supported, @@ -1196,11 +1197,7 @@ class AmazonConverseConfig(BaseConfig): ), status_code=400, ) - verbose_logger.debug( - "Dropping unsupported `reasoning_effort=%s` for Bedrock model=%s.", - value, - model, - ) + verbose_logger.warning(DROP_DISABLED_REASONING_EFFORT_WARNING, "reasoning_effort", value, model) elif param == "reasoning_effort" and isinstance(value, str): self._handle_reasoning_effort_parameter( model=model, reasoning_effort=value, optional_params=optional_params diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index fde995c7907..4175479f068 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -15,7 +15,11 @@ from dataclasses import dataclass from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias -from typing_extensions import ReadOnly, TypedDict +from typing_extensions import ( + ReadOnly, + TypedDict, + TypeIs, # noqa: TID251 # narrows the caller's tools without copying them +) if TYPE_CHECKING: from botocore.eventstream import EventStreamMessage @@ -573,6 +577,26 @@ def strip_unsupported_bedrock_invoke_output_config_keys( request_body["output_config"] = {"format": preserved_format} # rebind-ok: out-param +_TOOL_ITEMS_ADAPTER: Final = TypeAdapter(tuple[object, ...]) + + +def _is_str_keyed_dict( + value: object, +) -> TypeIs[dict[str, object]]: # guard-ok: JSON tool objects # mutable-ok: fallbacks reuse tools edited in place + return isinstance(value, dict) + + +def _normalize_custom_field_on_tool(tool: object) -> None: + if not _is_str_keyed_dict(tool): + return + custom: Final = tool.pop("custom", None) + if not _is_str_keyed_dict(custom) or "defer_loading" in tool: + return + deferred: Final = custom.get("defer_loading") + if isinstance(deferred, bool): + tool["defer_loading"] = deferred # rebind-ok: a fallback reuses the caller's tool, as on main + + def normalize_custom_field_on_tools(request_body: dict) -> None: """ Drop the ``custom`` field from each tool, first hoisting a boolean @@ -590,15 +614,8 @@ def normalize_custom_field_on_tools(request_body: dict) -> None: tools: Final = request_body.get("tools") if not tools or not isinstance(tools, list): return - for tool in tools: - if not isinstance(tool, dict): - continue - custom: dict[str, object] | None = tool.pop("custom", None) - if not isinstance(custom, dict) or "defer_loading" in tool: - continue - deferred: object = custom.get("defer_loading") - if isinstance(deferred, bool): - tool["defer_loading"] = deferred + for tool in _TOOL_ITEMS_ADAPTER.validate_python(tools): + _normalize_custom_field_on_tool(tool) _TOOL_DICTS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...]) @@ -1314,6 +1331,12 @@ def _mantle_api_base_from_env() -> str | None: return next((base[: -len(suffix)] for suffix in _MANTLE_OPENAI_BASE_SUFFIXES if base.endswith(suffix)), base) +DROP_DISABLED_REASONING_EFFORT_WARNING: Final = ( + "Dropping %s=%r for Bedrock model=%r (drop_params=True) because the model map marks that level unsupported. " + "Send a level the model supports." +) + + def bedrock_reasoning_effort_disabled(model: str, effort: str) -> bool: from litellm.utils import is_explicitly_disabled_factory diff --git a/litellm/llms/bedrock/count_tokens/mantle_handler.py b/litellm/llms/bedrock/count_tokens/mantle_handler.py index 34e622b5e8d..5cd26235ba2 100644 --- a/litellm/llms/bedrock/count_tokens/mantle_handler.py +++ b/litellm/llms/bedrock/count_tokens/mantle_handler.py @@ -8,6 +8,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.anthropic.common_utils import tools_without_custom_field from litellm.llms.anthropic.count_tokens.transformation import COUNT_TOKEN_OPTION_NAMES, AnthropicCountTokensConfig from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, run_aws_signing from litellm.llms.bedrock.common_utils import BedrockError, build_mantle_messages_url @@ -26,6 +27,7 @@ class MantleCountTokensRequest(TypedDict): _COUNT_REQUEST: Final = TypeAdapter(MantleCountTokensRequest) _COUNT_RESPONSE: Final = TypeAdapter(dict[str, JsonValue]) _COUNT_OPTIONS: Final[TypeAdapter[Mapping[str, JsonValue]]] = TypeAdapter(Mapping[str, JsonValue]) +_TOOLS: Final = TypeAdapter(list[dict[str, JsonValue]]) class BedrockMantleCountTokensHandler(BaseAWSLLM): @@ -59,10 +61,15 @@ class BedrockMantleCountTokensHandler(BaseAWSLLM): self.get_aws_region_name, optional_params=litellm_params, model=resolved_model, model_id=None ) endpoint_url: Final = await run_aws_signing(self.get_mantle_count_tokens_endpoint, aws_region_name) + tools: Final = request.get("tools") body: Final = self._anthropic_config.transform_request_to_count_tokens( model=resolved_model, messages=request["messages"], - tools=request.get("tools"), + tools=None + if tools is None + else _TOOLS.validate_python( + tools_without_custom_field(tools, hoist_defer_loading=True, drop_custom=True) + ), system=request.get("system"), optional_params=_COUNT_OPTIONS.validate_python( {name: value for name, value in request_data.items() if name in COUNT_TOKEN_OPTION_NAMES} diff --git a/litellm/llms/bedrock/count_tokens/transformation.py b/litellm/llms/bedrock/count_tokens/transformation.py index bf9c119f6f2..df5a3827c1f 100644 --- a/litellm/llms/bedrock/count_tokens/transformation.py +++ b/litellm/llms/bedrock/count_tokens/transformation.py @@ -12,6 +12,7 @@ from typing import Final, Literal from pydantic import JsonValue from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.anthropic.common_utils import tools_without_custom_field from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.common_utils import get_bedrock_base_model @@ -194,7 +195,15 @@ class BedrockCountTokensConfig(BaseAWSLLM): # For InvokeModel, we need to provide the raw body that would be sent to the model # Remove the 'model' field from the body as it's not part of the model input - body_data: Final = {k: v for k, v in request_data.items() if k != "model"} + tools: Final = request_data.get("tools") + body_data: Final = { + **{k: v for k, v in request_data.items() if k != "model"}, + **( + {"tools": tools_without_custom_field(tools, hoist_defer_loading=True, drop_custom=True)} + if isinstance(tools, list) + else {} + ), + } if "messages" in body_data: # Bedrock validates the body against the model's InvokeModel schema; diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py index e1e15861a90..60ce73abd82 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py @@ -102,6 +102,9 @@ class AmazonAnthropicClaudeMessagesConfig( def should_strip_billing_metadata(self) -> bool: return True + def tools_for_upstream(self, tools: Sequence[object], model: str, drop_params: object) -> Sequence[object]: + return tools + def drops_dangerous_tool_use_beta_without_safeguards(self) -> bool: return True diff --git a/litellm/llms/bedrock/responses/transformation.py b/litellm/llms/bedrock/responses/transformation.py index 7cdd7b08f99..ff44d973d97 100644 --- a/litellm/llms/bedrock/responses/transformation.py +++ b/litellm/llms/bedrock/responses/transformation.py @@ -52,6 +52,7 @@ from litellm.llms.base_llm.responses.codex_compat import drop_unsupported_tools, from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.common_utils import ( BEDROCK_CHAT_COMPLETIONS_ROUTE_PREFIX, + DROP_DISABLED_REASONING_EFFORT_WARNING, BedrockError, bedrock_reasoning_effort_disabled, bedrock_supports_openai_responses, @@ -173,7 +174,7 @@ def _without_disabled_reasoning_effort( ), status_code=400, ) - verbose_logger.debug("Dropping unsupported `reasoning.effort=%s` for Bedrock model=%s.", effort, model) + verbose_logger.warning(DROP_DISABLED_REASONING_EFFORT_WARNING, "reasoning.effort", effort, model) rest: Final = {key: value for key, value in reasoning.items() if key != "effort"} without_reasoning: Final = {key: value for key, value in params.items() if key != "reasoning"} return {**without_reasoning, "reasoning": rest} if rest else without_reasoning diff --git a/litellm/llms/bigmodel/__init__.py b/litellm/llms/bigmodel/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/bigmodel/chat/__init__.py b/litellm/llms/bigmodel/chat/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/bigmodel/chat/transformation.py b/litellm/llms/bigmodel/chat/transformation.py new file mode 100644 index 00000000000..12d5e912c15 --- /dev/null +++ b/litellm/llms/bigmodel/chat/transformation.py @@ -0,0 +1,59 @@ +from collections.abc import Mapping +from typing import Final + +from litellm.llms.zai.chat.transformation import ZAIChatConfig + +from ..common_utils import get_bigmodel_api_key, get_bigmodel_url, is_unsupported_disabled_thinking + +_SUPPORTED_PARAMS: Final = ( + "max_tokens", + "max_completion_tokens", + "stream", + "stream_options", + "temperature", + "top_p", + "stop", + "tools", + "tool_choice", + "response_format", + "thinking", + "reasoning_effort", +) + + +class BigModelChatConfig(ZAIChatConfig): + @property + def custom_llm_provider(self) -> str: + return "bigmodel" + + def get_openai_compatible_provider_info(self, api_base: str | None, api_key: str | None) -> tuple[str, str | None]: + return get_bigmodel_url(api_base, "chat"), get_bigmodel_api_key(api_key) + + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: inherited interface requires a list + return list(_SUPPORTED_PARAMS) + + def map_openai_params( + self, + non_default_params: Mapping[str, object], + optional_params: Mapping[str, object], + model: str, + drop_params: bool, + ) -> dict[str, object]: # mutable-ok: request pipeline adds parameters to the returned body + mapped: Final = { + **optional_params, + **{ + ("max_tokens" if key == "max_completion_tokens" else key): value + for key, value in non_default_params.items() + if key in _SUPPORTED_PARAMS and key != "thinking" + }, + } + if "thinking" not in non_default_params: + return mapped + thinking: Final = non_default_params["thinking"] + if is_unsupported_disabled_thinking(model, thinking): + return { + **mapped, + "reasoning_effort": mapped.get("reasoning_effort", "low"), + "extra_body": {"thinking": {"type": "enabled"}}, + } + return {**mapped, "extra_body": {"thinking": thinking}} diff --git a/litellm/llms/bigmodel/common_utils.py b/litellm/llms/bigmodel/common_utils.py new file mode 100644 index 00000000000..0c9b6da5c8f --- /dev/null +++ b/litellm/llms/bigmodel/common_utils.py @@ -0,0 +1,92 @@ +import re +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final, Literal +from urllib.parse import urlsplit, urlunsplit + +import httpx +from pydantic import JsonValue, TypeAdapter, ValidationError + +import litellm +from litellm.exceptions import ContentPolicyViolationError, ContextWindowExceededError +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.secret_managers.main import get_secret_str + +BIGMODEL_API_BASE: Final = "https://open.bigmodel.cn" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_BUSINESS_CODE_IN_ERROR: Final = re.compile(r"""['"]code['"]\s*:\s*['"]?(\d{4})\b""") +_CODING_CHAT_PATH: Final = "/api/coding/paas/v4" +_API_PATHS: Final = ( + "/api/coding/paas/v4/chat/completions", + _CODING_CHAT_PATH, + "/api/paas/v4/chat/completions", + "/api/anthropic/v1/messages", + "/api/anthropic/v1", + "/api/v1/responses", + "/api/paas/v4", + "/api/anthropic", + "/api/v1", + "/api", +) +_ENDPOINT_PATHS: Final = MappingProxyType( + { + "chat": "/api/paas/v4", + "responses": "/api/v1/responses", + "messages": "/api/anthropic/v1/messages", + } +) + + +def get_bigmodel_api_key(api_key: str | None) -> str | None: + return api_key or get_secret_str("BIGMODEL_API_KEY") + + +def get_bigmodel_url(api_base: str | None, endpoint: Literal["chat", "responses", "messages"]) -> str: + resolved_base: Final = api_base or get_secret_str("BIGMODEL_API_BASE") or BIGMODEL_API_BASE + parsed: Final = urlsplit(resolved_base) + path: Final = parsed.path.rstrip("/") + suffix: Final = next((candidate for candidate in _API_PATHS if path.endswith(candidate)), "") + endpoint_path: Final = ( + _CODING_CHAT_PATH if endpoint == "chat" and suffix.startswith(_CODING_CHAT_PATH) else _ENDPOINT_PATHS[endpoint] + ) + return urlunsplit( + (parsed.scheme, parsed.netloc, f"{path.removesuffix(suffix)}{endpoint_path}", parsed.query, parsed.fragment) + ) + + +def bigmodel_json_body(response: httpx.Response) -> Mapping[str, JsonValue]: + try: + return _JSON_OBJECT.validate_json(response.content) + except ValidationError: + raise BaseLLMException(message=response.text, status_code=response.status_code, headers=response.headers) + + +def is_unsupported_disabled_thinking(model: str, thinking: object) -> bool: + return thinking == {"type": "disabled"} and litellm.utils.supports_factory( + model=model, custom_llm_provider="bigmodel", key="thinking_always_on" + ) + + +def bigmodel_business_error( + code: JsonValue, message: str, model: str, response: httpx.Response | None +) -> ContextWindowExceededError | ContentPolicyViolationError | None: + match code: + case 1261 | "1261": + return ContextWindowExceededError(message=message, model=model, llm_provider="bigmodel", response=response) + case 1301 | "1301": + return ContentPolicyViolationError(message=message, model=model, llm_provider="bigmodel", response=response) + case _: + return None + + +def raise_bigmodel_business_error(model: object, error_str: str, original_exception: object) -> None: + found: Final = _BUSINESS_CODE_IN_ERROR.search(error_str) + response: Final = getattr(original_exception, "response", None) + error: Final = bigmodel_business_error( + found.group(1) if found else None, + error_str, + str(model), + response if isinstance(response, httpx.Response) else None, + ) + if error is not None: + raise error diff --git a/litellm/llms/bigmodel/messages/__init__.py b/litellm/llms/bigmodel/messages/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/bigmodel/messages/transformation.py b/litellm/llms/bigmodel/messages/transformation.py new file mode 100644 index 00000000000..82ce59631f1 --- /dev/null +++ b/litellm/llms/bigmodel/messages/transformation.py @@ -0,0 +1,206 @@ +import json +from collections.abc import AsyncIterator, Mapping, Sequence +from typing import Final + +import httpx +from openai._streaming import ServerSentEvent, SSEDecoder +from pydantic import JsonValue, TypeAdapter, ValidationError + +from litellm.exceptions import AuthenticationError +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.llms.openai_like.messages.transformation import ( + DEFAULT_ANTHROPIC_API_VERSION, + OpenAILikeAnthropicMessagesConfig, +) +from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse +from litellm.types.router import GenericLiteLLMParams + +from ..common_utils import ( + bigmodel_json_body, + get_bigmodel_api_key, + get_bigmodel_url, + is_unsupported_disabled_thinking, +) + +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_OUTPUT_CONFIG: Final = TypeAdapter(dict[str, object]) + + +def _with_lowest_effort_thinking( + params: Mapping[str, object], +) -> dict[str, object]: # mutable-ok: the parent transform takes a request dict + requested_output_config: Final = params.get("output_config") + output_config: Final = { + **({} if "reasoning_effort" in params else {"effort": "low"}), + **( + _OUTPUT_CONFIG.validate_python(requested_output_config) if isinstance(requested_output_config, dict) else {} + ), + } + return {**params, "thinking": {"type": "adaptive"}, **({"output_config": output_config} if output_config else {})} + + +def _normalize_message(body: Mapping[str, JsonValue]) -> dict[str, JsonValue]: # mutable-ok: native JSON response + usage: Final = body.get("usage") + if not isinstance(usage, dict): + return dict(body) + server_tools: Final = usage.get("server_tool_use") + if not isinstance(server_tools, dict) or "web_fetch_requests" in server_tools: + return dict(body) + normalized: Final[JsonValue] = { + **body, + "usage": {**usage, "server_tool_use": {"web_fetch_requests": 0, **server_tools}}, + } + return normalized + + +class _BigModelMessagesStream(httpx.AsyncByteStream): + def __init__(self, response: httpx.Response) -> None: + self._response = response + + async def __aiter__(self) -> AsyncIterator[bytes]: + async for event in SSEDecoder().aiter_bytes(self._response.aiter_bytes()): + yield _normalize_event(event) + + async def aclose(self) -> None: + await self._response.aclose() + + +def _json_object(data: str) -> Mapping[str, JsonValue] | None: + try: + return _JSON_OBJECT.validate_json(data) + except ValidationError: + return None + + +def _normalized_data(data: str) -> str: + body: Final = _json_object(data) + if body is None: + return data + message: Final = body.get("message") + normalized: Final[JsonValue] = ( + {**body, "message": _normalize_message(message)} + if body.get("type") == "message_start" and isinstance(message, dict) + else _normalize_message(body) + ) + return data if normalized == body else json.dumps(normalized, ensure_ascii=False) + + +def _normalize_event(event: ServerSentEvent) -> bytes: + metadata: Final = "".join( + f"{name}: {value}\n" + for name, value in (("event", event.event), ("id", event.id), ("retry", event.retry)) + if value is not None + ) + data_lines: Final = "".join(f"data: {line}\n" for line in _normalized_data(event.data).split("\n")) + return f"{metadata}{data_lines}\n".encode() + + +class BigModelMessagesConfig(OpenAILikeAnthropicMessagesConfig): + @property + def custom_llm_provider(self) -> str: + return "bigmodel" + + def is_native_messages_route(self) -> bool: + return True + + def should_strip_billing_metadata(self) -> bool: + return True + + def transform_anthropic_messages_request( + self, + model: str, + messages: list[dict[str, object]], # mutable-ok: matches dict-typed base signature + anthropic_messages_optional_request_params: dict[str, object], # mutable-ok: matches dict-typed base signature + litellm_params: GenericLiteLLMParams, + headers: dict[str, str], # mutable-ok: matches dict-typed base signature + ) -> dict[str, object]: # mutable-ok: matches dict-typed base signature + request_params: Final = ( + _with_lowest_effort_thinking(anthropic_messages_optional_request_params) + if is_unsupported_disabled_thinking(model, anthropic_messages_optional_request_params.get("thinking")) + else anthropic_messages_optional_request_params + ) + return super().transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the parent transform returns a bare dict + model, messages, request_params, litellm_params, headers + ) + + def transform_anthropic_messages_response( + self, model: str, raw_response: httpx.Response, logging_obj: Logging + ) -> AnthropicMessagesResponse: + normalized: Final = httpx.Response( + raw_response.status_code, + json=_normalize_message(bigmodel_json_body(raw_response)), + request=raw_response.request, + ) + return super().transform_anthropic_messages_response(model, normalized, logging_obj) + + def get_async_streaming_response_iterator( + self, + model: str, + httpx_response: httpx.Response, + request_body: dict[str, object], # mutable-ok: inherited native stream API takes a request dict + litellm_logging_obj: Logging, + ) -> AsyncIterator[bytes]: + normalized: Final = httpx.Response( + httpx_response.status_code, + headers={ + key: value + for key, value in httpx_response.headers.items() + if key.lower() not in ("content-encoding", "content-length") + }, + stream=_BigModelMessagesStream(httpx_response), + request=httpx_response.request, + ) + return super().get_async_streaming_response_iterator( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # native parent yields bytes but its iterator annotation omits the element type + model, normalized, request_body, litellm_logging_obj + ) + + @staticmethod + def _translate_reasoning_effort_to_anthropic( + model: str, optional_params: Mapping[str, object], max_tokens: int | None, custom_llm_provider: str + ) -> None: + return None + + @staticmethod + def _translate_adaptive_effort_for_non_adaptive_model( + model: str, optional_params: Mapping[str, object], max_tokens: int | None, custom_llm_provider: str + ) -> None: + return None + + @staticmethod + def _drop_incompatible_temperature_for_thinking( + model: str, optional_params: Mapping[str, object], custom_llm_provider: str + ) -> None: + return None + + def validate_anthropic_messages_environment( + self, + headers: Mapping[str, str], + model: str, + messages: Sequence[object], + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + api_key: str | None = None, + api_base: str | None = None, + ) -> tuple[dict[str, str], str | None]: # mutable-ok: HTTP handler merges forwarded headers + resolved_key: Final = get_bigmodel_api_key(api_key) + if resolved_key is None: + raise AuthenticationError( + message="Set BIGMODEL_API_KEY or pass api_key", llm_provider="bigmodel", model=model + ) + return { + **{key: value for key, value in headers.items() if key.lower() not in ("x-api-key", "authorization")}, + "x-api-key": resolved_key, + "anthropic-version": DEFAULT_ANTHROPIC_API_VERSION, + "content-type": "application/json", + }, api_base + + def get_complete_url( + self, + api_base: str | None, + api_key: str | None, + model: str, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + stream: bool | None = None, + ) -> str: + return get_bigmodel_url(api_base, "messages") diff --git a/litellm/llms/bigmodel/responses/__init__.py b/litellm/llms/bigmodel/responses/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/bigmodel/responses/transformation.py b/litellm/llms/bigmodel/responses/transformation.py new file mode 100644 index 00000000000..80c53682d16 --- /dev/null +++ b/litellm/llms/bigmodel/responses/transformation.py @@ -0,0 +1,263 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final + +import httpx +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError +from typing_extensions import Never + +from litellm.exceptions import AuthenticationError, UnsupportedParamsError +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.types.llms.openai import ( + ResponseInputParam, + ResponsesAPIOptionalRequestParams, + ResponsesAPIResponse, + ResponsesAPIStreamingResponse, +) +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders + +from ..common_utils import bigmodel_business_error, bigmodel_json_body, get_bigmodel_api_key, get_bigmodel_url + +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_SUPPORTED_PARAMS: Final = ( + "input", + "model", + "instructions", + "stream", + "temperature", + "top_p", + "max_output_tokens", + "tools", + "tool_choice", + "parallel_tool_calls", + "include", + "reasoning", + "text", + "prompt_cache_key", + "previous_response_id", + "store", + "extra_headers", + "extra_query", + "extra_body", + "timeout", +) + + +_STATUS_BY_BUSINESS_CODE: Final = MappingProxyType( + { + **dict.fromkeys((1000, 1001, 1003, 1005), 401), + **dict.fromkeys((1210, 1211, 1212, 1213, 1214, 1215, 1221, 1222, 1261, 1301), 400), + 1220: 403, + **dict.fromkeys( + (1113, 1302, 1305, 1308, 1309, 1310, 1311, 1313, 1314, 1315, 1316, 1317, 1318, 1319, 1320, 1321), + 429, + ), + } +) + + +class _RequestedTools(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + tools: JsonValue = None + tool_choice: JsonValue = None + parallel_tool_calls: JsonValue = None + + +def _requested_tools(logging_obj: Logging) -> _RequestedTools: + try: + return _RequestedTools.model_validate(logging_obj.model_call_details.get("optional_params") or {}) + except ValidationError: + return _RequestedTools() + + +def _envelope_status_code(code: JsonValue) -> int: + if not isinstance(code, int) or isinstance(code, bool): + return 500 + if 400 <= code <= 599: + return code + return _STATUS_BY_BUSINESS_CODE.get(code, 500) + + +def _normalize_output_item(item: JsonValue) -> JsonValue: + if not isinstance(item, dict) or item.get("type") not in ("message", "reasoning"): + return item + content: Final = item.get("content") + normalized: Final[JsonValue] = {"summary": [], **item} if item["type"] == "reasoning" else item + if not isinstance(content, dict): + return normalized + return {**normalized, "content": [content]} + + +def _normalize_response( + body: Mapping[str, JsonValue], + requested: _RequestedTools, +) -> dict[str, JsonValue]: # mutable-ok: JSON serialization boundary + usage: Final = body.get("usage") + output: Final = body.get("output") + tools: Final = body.get("tools") + tool_choice: Final = body.get("tool_choice") + parallel_tool_calls: Final = body.get("parallel_tool_calls") + input_tokens: Final = usage.get("input_tokens") if isinstance(usage, dict) else None + output_tokens: Final = usage.get("output_tokens") if isinstance(usage, dict) else None + normalized_usage: Final = ( + { + "total_tokens": input_tokens + output_tokens, + **usage, + } + if isinstance(usage, dict) and isinstance(input_tokens, int) and isinstance(output_tokens, int) + else usage + ) + normalized: Final[JsonValue] = { + **body, + "parallel_tool_calls": ( + requested.parallel_tool_calls is not False if parallel_tool_calls is None else parallel_tool_calls + ), + "tools": (requested.tools or []) if tools is None else tools, + "tool_choice": (requested.tool_choice or "auto") if tool_choice is None else tool_choice, + **({"usage": normalized_usage} if "usage" in body else {}), + **({"output": [_normalize_output_item(item) for item in output]} if isinstance(output, list) else {}), + } + return normalized + + +def _normalize_stream_part(part: JsonValue) -> JsonValue: + if not isinstance(part, dict): + return part + if part.get("type") == "output_text": + return { + "annotations": [], + "logprobs": None, + **part, + } + if part.get("type") == "reasoning_text": + return {"reasoning": part.get("text", ""), **part} + return part + + +class BigModelResponsesAPIConfig(OpenAIResponsesAPIConfig): + @property + def custom_llm_provider(self) -> LlmProviders: + return LlmProviders.BIGMODEL + + def supports_native_file_search(self) -> bool: + return False + + def supports_native_websocket(self) -> bool: + return False + + def should_fake_stream( + self, model: str | None, stream: bool | None, custom_llm_provider: str | None = None + ) -> bool: + return False + + def may_answer_stream_with_json(self) -> bool: + return True + + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: inherited API requires a list + return list(_SUPPORTED_PARAMS) + + def map_openai_params( + self, + response_api_optional_params: ResponsesAPIOptionalRequestParams, + model: str, + drop_params: bool, + ) -> dict[str, object]: # mutable-ok: request pipeline adds parameters to the returned body + return {key: value for key, value in response_api_optional_params.items() if key in _SUPPORTED_PARAMS} + + def validate_environment( + self, headers: Mapping[str, str], model: str, litellm_params: GenericLiteLLMParams | None + ) -> dict[str, str]: # mutable-ok: HTTP handler merges forwarded headers + resolved_key: Final = get_bigmodel_api_key(litellm_params.api_key if litellm_params else None) + if resolved_key is None: + raise AuthenticationError( + message="Set BIGMODEL_API_KEY or pass api_key", llm_provider="bigmodel", model=model + ) + return { + **headers, + "Authorization": f"Bearer {resolved_key}", + "Content-Type": "application/json", + } + + def get_complete_url(self, api_base: str | None, litellm_params: Mapping[str, object]) -> str: + return get_bigmodel_url(api_base, "responses") + + def transform_response_api_response( + self, model: str, raw_response: httpx.Response, logging_obj: Logging + ) -> ResponsesAPIResponse: + body: Final = bigmodel_json_body(raw_response) + if body.get("success") is False: + message: Final = str(body.get("msg") or raw_response.text) + raise bigmodel_business_error(body.get("code"), message, model, None) or BaseLLMException( + message=message, + status_code=_envelope_status_code(body.get("code")), + headers=raw_response.headers, + ) + normalized: Final = _normalize_response(body, _requested_tools(logging_obj)) + normalized_response: Final = httpx.Response( + status_code=raw_response.status_code, + headers={ + key: value + for key, value in raw_response.headers.items() + if key.lower() not in ("content-encoding", "content-length") + }, + json=normalized, + request=raw_response.request, + ) + return super().transform_response_api_response(model, normalized_response, logging_obj) + + def transform_get_response_api_response( + self, raw_response: httpx.Response, logging_obj: Logging + ) -> ResponsesAPIResponse: + return self.transform_response_api_response("", raw_response, logging_obj) + + def transform_streaming_response( + self, model: str, parsed_chunk: Mapping[str, object], logging_obj: Logging + ) -> ResponsesAPIStreamingResponse: + chunk: Final = _JSON_OBJECT.validate_python(parsed_chunk) + response: Final = chunk.get("response") + item: Final = chunk.get("item") + normalized: Final[JsonValue] = { + **chunk, + **( + {"response": _normalize_response(response, _requested_tools(logging_obj))} + if isinstance(response, dict) + else {} + ), + **({"item": _normalize_output_item(item)} if "item" in chunk else {}), + } + event_model: Final = self.get_event_model_class(str(normalized.get("type"))) + match chunk.get("type"): + case "response.output_text.done": + return event_model.model_validate({"logprobs": [], **normalized}) + case "response.content_part.done": + return event_model.model_validate({**normalized, "part": _normalize_stream_part(chunk.get("part"))}) + case "error": + error: Final = chunk.get("error") + error_fields: Final = error if isinstance(error, dict) else chunk + normalized_error: Final[JsonValue] = { + **normalized, + "error": {"type": "api_error", **error_fields, "code": error_fields.get("code") or "unknown_error"}, + } + return event_model.model_validate(normalized_error) + case _: + return event_model.model_validate(normalized) + + def transform_cancel_response_api_request( + self, response_id: str, api_base: str, litellm_params: GenericLiteLLMParams, headers: Mapping[str, str] + ) -> Never: + raise UnsupportedParamsError(message="BigModel does not support cancelling responses", status_code=400) + + def transform_compact_response_api_request( + self, + model: str, + input: str | ResponseInputParam, + response_api_optional_request_params: Mapping[str, object], + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: Mapping[str, str], + ) -> Never: + raise UnsupportedParamsError(message="BigModel does not support compacting responses", status_code=400) diff --git a/litellm/llms/cohere/embed/transformation.py b/litellm/llms/cohere/embed/transformation.py index 590164ffa0f..1477e45997b 100644 --- a/litellm/llms/cohere/embed/transformation.py +++ b/litellm/llms/cohere/embed/transformation.py @@ -10,11 +10,13 @@ Convers Docs - https://docs.cohere.com/v2/reference/embed """ -from collections.abc import Iterable, Mapping, Sized -from typing import Final, Protocol, cast +from collections.abc import Iterable, Iterator, Mapping, Sequence, Sized +from enum import Enum +from itertools import chain, starmap +from typing import Final, Protocol, TypeAlias, cast import httpx -from pydantic import ConfigDict, TypeAdapter +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter import litellm from litellm import COHERE_DEFAULT_EMBEDDING_INPUT_TYPE @@ -32,14 +34,117 @@ from litellm.utils import is_base64_encoded from ..common_utils import CohereError _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) -_EMBEDDINGS_BY_TYPE: Final = TypeAdapter(Mapping[str, Iterable[object]], config=ConfigDict(hide_input_in_errors=True)) _DECODED_JSON: Final = TypeAdapter(object) class _SupportsEncode(Protocol): """Tokenizer handle: the embedding usage path only encodes text to measure its token length.""" - def encode(self, text: str, /) -> Sized: ... + def encode_ordinary(self, text: str, /) -> Sized: ... + + +class CohereImageTokenBilling(Enum): + SEPARATE_FROM_INPUT_TOKENS = "separate_from_input_tokens" + FOLDED_INTO_INPUT_TOKENS = "folded_into_input_tokens" + + +class CohereEmbeddingRowShape(Enum): + OPENAI = "openai" + TYPED = "typed" + + +class _CohereBilledUnits(BaseModel): + model_config = ConfigDict(frozen=True, hide_input_in_errors=True) + + input_tokens: int | None = None + image_tokens: int | None = None + images: int | None = None + + +class _CohereEmbedMeta(BaseModel): + model_config = ConfigDict(frozen=True, hide_input_in_errors=True) + + billed_units: _CohereBilledUnits = Field(default_factory=_CohereBilledUnits) + + +_CohereEmbeddings: TypeAlias = Mapping[str, Sequence[object]] | Sequence[object] + + +class _CohereEmbedBody(BaseModel): + model_config = ConfigDict(frozen=True, hide_input_in_errors=True) + + embeddings: _CohereEmbeddings + meta: _CohereEmbedMeta = Field(default_factory=_CohereEmbedMeta) + + +def _embedding_rows(vectors: Sequence[object]) -> Iterator[Mapping[str, object]]: + return ({"object": "embedding", "index": index, "embedding": vector} for index, vector in enumerate(vectors)) + + +def _typed_embedding_rows(embedding_type: str, vectors: Sequence[object]) -> Iterator[Mapping[str, object]]: + return ({**row, "type": embedding_type} for row in _embedding_rows(vectors)) + + +def _response_rows(embeddings: _CohereEmbeddings, row_shape: CohereEmbeddingRowShape) -> Iterable[Mapping[str, object]]: + if not isinstance(embeddings, Mapping): + return _embedding_rows(embeddings) + match row_shape: + case CohereEmbeddingRowShape.OPENAI: + return chain.from_iterable(map(_embedding_rows, embeddings.values())) + case CohereEmbeddingRowShape.TYPED: + return chain.from_iterable(starmap(_typed_embedding_rows, embeddings.items())) + + +def _prompt_usage(prompt_tokens: int, prompt_tokens_details: PromptTokensDetailsWrapper | None) -> Usage: + return Usage( + prompt_tokens=prompt_tokens, + completion_tokens=0, + total_tokens=prompt_tokens, + prompt_tokens_details=prompt_tokens_details, + ) + + +def _billed_usage(billed_units: _CohereBilledUnits, image_billing: CohereImageTokenBilling) -> Usage: + match image_billing: + case CohereImageTokenBilling.SEPARATE_FROM_INPUT_TOKENS: + image_units: Final = (billed_units.image_tokens or 0) + (billed_units.images or 0) + return _prompt_usage( + (billed_units.input_tokens or 0) + image_units, + PromptTokensDetailsWrapper( + text_tokens=billed_units.input_tokens, + image_tokens=image_units or None, + image_count=billed_units.images, + ), + ) + case CohereImageTokenBilling.FOLDED_INTO_INPUT_TOKENS: + return _prompt_usage(billed_units.input_tokens or 0, None) + + +def _cohere_embed_usage( + billed_units: _CohereBilledUnits, + input_texts: Iterable[str], + encoding: _SupportsEncode, + image_billing: CohereImageTokenBilling, +) -> Usage: + if billed_units.input_tokens is None and billed_units.image_tokens is None and billed_units.images is None: + return _prompt_usage(sum(len(encoding.encode_ordinary(text)) for text in input_texts), None) + return _billed_usage(billed_units, image_billing) + + +def cohere_embedding_response( + body: Mapping[str, object], + model: str | None, + encoding: _SupportsEncode, + input_texts: Iterable[str], + image_billing: CohereImageTokenBilling, + row_shape: CohereEmbeddingRowShape, +) -> EmbeddingResponse: + cohere_body: Final = _CohereEmbedBody.model_validate(body) + return EmbeddingResponse( + model=model, + data=list(_response_rows(cohere_body.embeddings, row_shape)), + usage=_cohere_embed_usage(cohere_body.meta.billed_units, input_texts, encoding, image_billing), + ) class CohereEmbeddingConfig(BaseEmbeddingConfig): @@ -145,44 +250,16 @@ class CohereEmbeddingConfig(BaseEmbeddingConfig): ), ) - def _calculate_usage(self, input: list[str], encoding: _SupportsEncode, meta: dict) -> Usage: - input_tokens = 0 - - text_tokens: Final[int | None] = meta.get("billed_units", {}).get("input_tokens") - - image_tokens: Final[int | None] = meta.get("billed_units", {}).get("images") - - prompt_tokens_details: PromptTokensDetailsWrapper | None = None - if image_tokens is None and text_tokens is None: - for text in input: - input_tokens += len(encoding.encode(text)) - else: - prompt_tokens_details = PromptTokensDetailsWrapper( - image_tokens=image_tokens, - text_tokens=text_tokens, - ) - if image_tokens: - input_tokens += image_tokens - if text_tokens: - input_tokens += text_tokens - - return Usage( - prompt_tokens=input_tokens, - completion_tokens=0, - total_tokens=input_tokens, - prompt_tokens_details=prompt_tokens_details, - ) - - def _transform_response( + def transform_native_embed_response( self, response: httpx.Response, api_key: str | None, logging_obj: LiteLLMLoggingObj, - data: dict | CohereEmbeddingRequest, - model_response: EmbeddingResponse, + data: Mapping[str, object] | CohereEmbeddingRequest, model: str, encoding: _SupportsEncode, - input: list, + input: Sequence[str], + row_shape: CohereEmbeddingRowShape, ) -> EmbeddingResponse: response_json: Final = _DECODED_JSON.validate_python(response.json()) ## LOGGING @@ -192,38 +269,15 @@ class CohereEmbeddingConfig(BaseEmbeddingConfig): additional_args={"complete_input_dict": data}, original_response=response_json, ) - """ - response - { - 'object': "list", - 'data': [ - - ] - 'model', - 'usage' - } - """ - payload: Final = _JSON_OBJECT.validate_python(response_json) - embeddings: Final = _EMBEDDINGS_BY_TYPE.validate_python(payload["embeddings"]) - output_data: Final = [] - for k, embedding_list in embeddings.items(): - for idx, embedding in enumerate(embedding_list): - output_data.append({"object": "embedding", "index": idx, "embedding": embedding}) - model_response.object = "list" - model_response.data = output_data - model_response.model = model - input_tokens = 0 - for text in input: - input_tokens += len(encoding.encode(text)) - - setattr( - model_response, - "usage", - self._calculate_usage(input, encoding, dict(_JSON_OBJECT.validate_python(payload.get("meta", {})))), + return cohere_embedding_response( + _JSON_OBJECT.validate_python(response_json), + model, + encoding, + input, + CohereImageTokenBilling.SEPARATE_FROM_INPUT_TOKENS, + row_shape, ) - return model_response - def transform_embedding_response( self, model: str, @@ -235,15 +289,15 @@ class CohereEmbeddingConfig(BaseEmbeddingConfig): optional_params: dict, litellm_params: dict, ) -> EmbeddingResponse: - return self._transform_response( + return self.transform_native_embed_response( response=raw_response, api_key=api_key, logging_obj=logging_obj, data=request_data, - model_response=model_response, model=model, encoding=litellm.encoding, input=logging_obj.model_call_details["input"], + row_shape=CohereEmbeddingRowShape.OPENAI, ) def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException: diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 0b64885df45..54adfb5c692 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -764,18 +764,23 @@ class AsyncHTTPHandler: follow_redirects: bool | None = None, timeout: float | httpx.Timeout | None = None, max_response_bytes: int | None = None, + stream: bool = False, ): # Set follow_redirects to UseClientDefault if None _follow_redirects: Final = follow_redirects if follow_redirects is not None else USE_CLIENT_DEFAULT + query: Final = httpx.QueryParams(params).merge(HTTPHandler.extract_query_params(url)) + request_headers: Final = httpx.Headers(headers) - params = params or {} - params.update(HTTPHandler.extract_query_params(url)) + if stream: + return await self._send_streaming_get( + url, params=query, headers=request_headers, follow_redirects=follow_redirects, timeout=timeout + ) if max_response_bytes is not None: return await self._get_with_response_limit( url, - params=httpx.QueryParams(params), - headers=httpx.Headers(headers), + params=query, + headers=request_headers, max_bytes=max_response_bytes, follow_redirects=self.client.follow_redirects if follow_redirects is None else follow_redirects, timeout=self.client.timeout if timeout is None else httpx.Timeout(timeout), @@ -783,13 +788,41 @@ class AsyncHTTPHandler: response: Final = await self.client.get( url, - params=params, - headers=headers, + params=query, + headers=request_headers, follow_redirects=_follow_redirects, timeout=timeout if timeout is not None else USE_CLIENT_DEFAULT, ) return response + async def _send_streaming_get( + self, + url: str, + *, + params: httpx.QueryParams, + headers: httpx.Headers, + follow_redirects: bool | None, + timeout: float | httpx.Timeout | None, + ) -> httpx.Response: + start_time: Final = time.time() + request: Final = self.client.build_request( + "GET", url, params=params, headers=headers, timeout=USE_CLIENT_DEFAULT if timeout is None else timeout + ) + try: + response: Final = await self.client.send( + request, + stream=True, + follow_redirects=USE_CLIENT_DEFAULT if follow_redirects is None else follow_redirects, + ) + _anchor_handler_to(response, self) + response.raise_for_status() + except httpx.TimeoutException as e: + _raise_timeout(e, start_time) + except httpx.HTTPStatusError as e: + await _raise_masked_async_error(e, True) + else: + return response + async def _get_with_response_limit( self, url: str, @@ -1468,15 +1501,20 @@ class HTTPHandler: headers: Mapping[str, Any] | None = None, follow_redirects: bool | None = None, timeout: float | httpx.Timeout | None = None, + stream: bool = False, ): # Set follow_redirects to UseClientDefault if None _follow_redirects: Final = follow_redirects if follow_redirects is not None else USE_CLIENT_DEFAULT - params = params or {} - params.update(self.extract_query_params(url)) + query: Final = httpx.QueryParams(params).merge(self.extract_query_params(url)) + + if stream: + return self._send_streaming_get( + url, params=query, headers=headers, follow_redirects=follow_redirects, timeout=timeout + ) response: Final = self.client.get( url, - params=params, + params=query, headers=headers, follow_redirects=_follow_redirects, timeout=timeout if timeout is not None else USE_CLIENT_DEFAULT, @@ -1484,6 +1522,34 @@ class HTTPHandler: return response + def _send_streaming_get( + self, + url: str, + *, + params: httpx.QueryParams, + headers: Mapping[str, str] | None, + follow_redirects: bool | None, + timeout: float | httpx.Timeout | None, + ) -> httpx.Response: + start_time: Final = time.time() + request: Final = self.client.build_request( + "GET", url, params=params, headers=headers, timeout=USE_CLIENT_DEFAULT if timeout is None else timeout + ) + try: + response: Final = self.client.send( + request, + stream=True, + follow_redirects=USE_CLIENT_DEFAULT if follow_redirects is None else follow_redirects, + ) + _anchor_handler_to(response, self) + response.raise_for_status() + except httpx.TimeoutException as e: + _raise_timeout(e, start_time) + except httpx.HTTPStatusError as e: + _raise_masked_sync_error(e, True) + else: + return response + @staticmethod def extract_query_params(url: str) -> dict[str, str]: """ diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index e7da6586b97..4c0f84a003d 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -50,7 +50,11 @@ from litellm.litellm_core_utils.audio_utils.subtitle_utils import ( SUBTITLE_RESPONSE_FORMATS, synthesize_subtitle_document, ) -from litellm.litellm_core_utils.core_helpers import as_str_mapping, set_provider_response_headers_in_hidden_params +from litellm.litellm_core_utils.core_helpers import ( + as_str_mapping, + qualify_agentic_followup_model, + set_provider_response_headers_in_hidden_params, +) from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header @@ -367,12 +371,37 @@ class _SignedBodyKwargs(TypedDict, total=False): json: ReadOnly[dict[str, object]] +def _is_sized_json_body(response: httpx.Response) -> bool: + headers: Final = httpx.Headers(response.headers) + media_types: Final = (value.split(";")[0].strip().lower() for value in headers.get_list("content-type")) + return "content-length" in headers and "application/json" in media_types + + +def _is_json_document(content: bytes) -> bool: + try: + return isinstance(json.loads(content), dict) + except ValueError: + return False + + def _signed_body_kwargs(*, signed_body: bytes | None, data: dict[str, object]) -> _SignedBodyKwargs: if signed_body is not None: return {"data": signed_body} return {"json": data} +def _answered_json(response: httpx.Response) -> bool: + return "application/json" in response.headers.get("content-type", "").lower() + + +def _stream_query_params(stream: bool, starting_after: int | None) -> Mapping[str, object]: + if not stream: + return {} + if starting_after is None: + return {"stream": "true"} + return {"stream": "true", "starting_after": starting_after} + + def _google_genai_streaming_hidden_params( *, api_base: str, @@ -2552,6 +2581,7 @@ class BaseLLMHTTPHandler: "preset_cache_key": None, "stream_response": {}, "model_info": kwargs.get("model_info"), + **litellm_params.per_second_pricing(), **vertex_location_params, **anthropic_messages_optional_request_params, }, @@ -3025,6 +3055,17 @@ class BaseLLMHTTPHandler: }, ) + replay_as_stream: Final = partial( + MockResponsesAPIStreamingIterator, + model=model, + logging_obj=logging_obj, + responses_api_provider_config=responses_api_provider_config, + litellm_metadata=litellm_metadata, + custom_llm_provider=custom_llm_provider, + request_data=request_context, + call_type=CallTypes.responses.value, + ) + try: if is_stream_request: response = sync_httpx_client.post( @@ -3036,7 +3077,13 @@ class BaseLLMHTTPHandler: ) if caller_requested_stream: if fake_stream is True: - return MockResponsesAPIStreamingIterator( + return replay_as_stream(response=response) + if not ( + responses_api_provider_config.may_answer_stream_with_json() + and _is_sized_json_body(response) + and _is_json_document(response.read()) + ): + return SyncResponsesAPIStreamingIterator( response=response, model=model, logging_obj=logging_obj, @@ -3046,17 +3093,6 @@ class BaseLLMHTTPHandler: request_data=request_context, call_type=CallTypes.responses.value, ) - - return SyncResponsesAPIStreamingIterator( - response=response, - model=model, - logging_obj=logging_obj, - responses_api_provider_config=responses_api_provider_config, - litellm_metadata=litellm_metadata, - custom_llm_provider=custom_llm_provider, - request_data=request_context, - call_type=CallTypes.responses.value, - ) response.read() else: response = sync_httpx_client.post( @@ -3071,6 +3107,9 @@ class BaseLLMHTTPHandler: provider_config=responses_api_provider_config, ) + if is_stream_request and caller_requested_stream: + return replay_as_stream(response=response) + initial_response: Final = responses_api_provider_config.transform_response_api_response( model=model, raw_response=response, @@ -3226,6 +3265,17 @@ class BaseLLMHTTPHandler: }, ) + replay_as_stream: Final = partial( + MockResponsesAPIStreamingIterator, + model=model, + logging_obj=logging_obj, + responses_api_provider_config=responses_api_provider_config, + litellm_metadata=litellm_metadata, + custom_llm_provider=custom_llm_provider, + request_data=request_context, + call_type=CallTypes.responses.value, + ) + try: if is_stream_request: response = await async_httpx_client.post( @@ -3239,7 +3289,13 @@ class BaseLLMHTTPHandler: if caller_requested_stream: if fake_stream is True: - return MockResponsesAPIStreamingIterator( + return replay_as_stream(response=response) + if not ( + responses_api_provider_config.may_answer_stream_with_json() + and _is_sized_json_body(response) + and _is_json_document(await response.aread()) + ): + return ResponsesAPIStreamingIterator( response=response, model=model, logging_obj=logging_obj, @@ -3249,17 +3305,6 @@ class BaseLLMHTTPHandler: request_data=request_context, call_type=CallTypes.responses.value, ) - - return ResponsesAPIStreamingIterator( - response=response, - model=model, - logging_obj=logging_obj, - responses_api_provider_config=responses_api_provider_config, - litellm_metadata=litellm_metadata, - custom_llm_provider=custom_llm_provider, - request_data=request_context, - call_type=CallTypes.responses.value, - ) await response.aread() else: response = await async_httpx_client.post( @@ -3276,6 +3321,9 @@ class BaseLLMHTTPHandler: provider_config=responses_api_provider_config, ) + if is_stream_request and caller_requested_stream: + return replay_as_stream(response=response) + initial_response: Final = responses_api_provider_config.transform_response_api_response( model=model, raw_response=response, @@ -3493,12 +3541,20 @@ class BaseLLMHTTPHandler: logging_obj: LiteLLMLoggingObj, custom_llm_provider: str | None = None, extra_headers: dict[str, object] | None = None, + extra_query: Mapping[str, object] | None = None, extra_body: dict[str, object] | None = None, + stream: bool = False, + starting_after: int | None = None, + litellm_metadata: Mapping[str, object] = MappingProxyType({}), timeout: float | httpx.Timeout | None = None, client: HTTPHandler | AsyncHTTPHandler | None = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> ResponsesAPIResponse | Coroutine[object, object, ResponsesAPIResponse]: + ) -> ( + ResponsesAPIResponse + | SyncResponsesAPIStreamingIterator + | Coroutine[object, object, ResponsesAPIResponse | ResponsesAPIStreamingIterator] + ): """ Get a response by ID Uses GET /v1/responses/{response_id} endpoint in the responses API @@ -3511,7 +3567,11 @@ class BaseLLMHTTPHandler: logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, extra_headers=extra_headers, + extra_query=extra_query, extra_body=extra_body, + stream=stream, + starting_after=starting_after, + litellm_metadata=litellm_metadata, timeout=timeout, client=client, shared_session=shared_session, @@ -3540,20 +3600,23 @@ class BaseLLMHTTPHandler: litellm_params=litellm_params, headers=headers, ) + query: Final = {**data, **(extra_query or {}), **_stream_query_params(stream, starting_after)} ## LOGGING logging_obj.pre_call( input="", api_key="", additional_args={ - "complete_input_dict": data, + "complete_input_dict": query, "api_base": api_base, "headers": headers, }, ) try: - response: Final = sync_httpx_client.get(url=url, headers=headers, params=data) + response: Final = sync_httpx_client.get( + url=url, headers=headers, params=query, stream=stream, timeout=timeout if stream else None + ) response.raise_for_status() except Exception as e: raise self._handle_error( @@ -3561,6 +3624,20 @@ class BaseLLMHTTPHandler: provider_config=responses_api_provider_config, ) + if stream and not _answered_json(response): + return SyncResponsesAPIStreamingIterator( + response=response, + model="", + logging_obj=logging_obj, + responses_api_provider_config=responses_api_provider_config, + litellm_metadata=dict(litellm_metadata), + custom_llm_provider=custom_llm_provider, + request_data={"litellm_params": dict(litellm_params)}, + replays_stored_stream=True, + ) + + if stream: + response.read() return responses_api_provider_config.transform_get_response_api_response( raw_response=response, logging_obj=logging_obj, @@ -3574,11 +3651,15 @@ class BaseLLMHTTPHandler: logging_obj: LiteLLMLoggingObj, custom_llm_provider: str | None = None, extra_headers: dict[str, object] | None = None, + extra_query: Mapping[str, object] | None = None, extra_body: dict[str, object] | None = None, + stream: bool = False, + starting_after: int | None = None, + litellm_metadata: Mapping[str, object] = MappingProxyType({}), timeout: float | httpx.Timeout | None = None, client: HTTPHandler | AsyncHTTPHandler | None = None, shared_session: Optional["ClientSession"] = None, - ) -> ResponsesAPIResponse: + ) -> ResponsesAPIResponse | ResponsesAPIStreamingIterator: """ Async version of get_responses """ @@ -3616,20 +3697,23 @@ class BaseLLMHTTPHandler: litellm_params=litellm_params, headers=headers, ) + query: Final = {**data, **(extra_query or {}), **_stream_query_params(stream, starting_after)} ## LOGGING logging_obj.pre_call( input="", api_key="", additional_args={ - "complete_input_dict": data, + "complete_input_dict": query, "api_base": api_base, "headers": headers, }, ) try: - response: Final = await async_httpx_client.get(url=url, headers=headers, params=data) + response: Final = await async_httpx_client.get( + url=url, headers=headers, params=query, stream=stream, timeout=timeout if stream else None + ) response.raise_for_status() except Exception as e: verbose_logger.debug("Error retrieving response: %s", e) @@ -3638,6 +3722,20 @@ class BaseLLMHTTPHandler: provider_config=responses_api_provider_config, ) + if stream and not _answered_json(response): + return ResponsesAPIStreamingIterator( + response=response, + model="", + logging_obj=logging_obj, + responses_api_provider_config=responses_api_provider_config, + litellm_metadata=dict(litellm_metadata), + custom_llm_provider=custom_llm_provider, + request_data={"litellm_params": dict(litellm_params)}, + replays_stored_stream=True, + ) + + if stream: + await response.aread() return responses_api_provider_config.transform_get_response_api_response( raw_response=response, logging_obj=logging_obj, @@ -6182,9 +6280,7 @@ class BaseLLMHTTPHandler: if patch.messages is None: raise ValueError("Agentic loop plan missing patched messages") - full_model_name = patch.model or model - if "/" not in full_model_name: - full_model_name = f"{custom_llm_provider}/{full_model_name}" + full_model_name: Final = qualify_agentic_followup_model(patch.model, model, custom_llm_provider) optional_params_for_followup: Final = dict(optional_params) optional_params_for_followup.update(patch.optional_params) diff --git a/litellm/llms/fireworks_ai/responses/transformation.py b/litellm/llms/fireworks_ai/responses/transformation.py index db234c09930..86d9df9618a 100644 --- a/litellm/llms/fireworks_ai/responses/transformation.py +++ b/litellm/llms/fireworks_ai/responses/transformation.py @@ -1,10 +1,9 @@ -from collections.abc import Mapping, Sequence +from collections.abc import Mapping from types import MappingProxyType from typing import TYPE_CHECKING, Final from urllib.parse import unquote import httpx -from openai.types.responses import EasyInputMessageParam, ResponseInputContentParam, ResponseInputItemParam from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.fireworks_ai.common_utils import ( @@ -15,12 +14,18 @@ from litellm.llms.fireworks_ai.common_utils import ( with_fireworks_session_affinity, without_caller_user, ) +from litellm.llms.openai.responses.instruction_items import developer_items_as_system, instruction_items_folded from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ResponseInputParam from litellm.types.responses.main import DeleteResponseResult from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders +from litellm.utils import ( + declared_mid_conversation_system_support, + deployment_declared_mid_conversation_system_support, + deployment_model_info, +) if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -45,90 +50,6 @@ def _forwarded_user_params(litellm_params: GenericLiteLLMParams) -> Mapping[str, ) -_INSTRUCTION_ROLES: Final = frozenset({"system", "developer"}) - - -def _role(item: ResponseInputItemParam) -> str | None: - match item: - case {"role": str(role)}: - return role - case _: - return None - - -def _developer_item_as_system(item: ResponseInputItemParam) -> ResponseInputItemParam: - if "role" not in item or item["role"] != "developer": - return item - return EasyInputMessageParam(role="system", content=item["content"], type="message") - - -def _developer_items_as_system(input: str | ResponseInputParam) -> str | ResponseInputParam: - if isinstance(input, str): - return input - return [_developer_item_as_system(item) for item in input] - - -def _text_part(part: ResponseInputContentParam) -> str | None: - match part: - case {"type": "input_text", "text": str(text)}: - return text - case _: - return None - - -def _text_only_content(item: ResponseInputItemParam) -> str | None: - match item: - case {"role": "system" | "developer", "content": str(text)}: - return text - case {"role": "system" | "developer", "content": [*parts]}: - texts: Final = tuple(map(_text_part, parts)) - return None if any(text is None for text in texts) else "\n\n".join(text for text in texts if text) - case _: - return None - - -def _leading_instruction_block_length(roles: Sequence[str | None]) -> int: - return next((index for index, role in enumerate(roles) if role not in _INSTRUCTION_ROLES), len(roles)) - - -def _closing_instruction_block_start(roles: Sequence[str | None], leading_length: int) -> int: - last_conversation_index: Final = next( - (index for index in range(len(roles) - 1, leading_length - 1, -1) if roles[index] not in _INSTRUCTION_ROLES), - None, - ) - if last_conversation_index is None or roles[last_conversation_index] != "assistant": - return len(roles) - return last_conversation_index + 1 - - -def _hoisted_indices(roles: Sequence[str | None]) -> tuple[int, ...]: - leading_length: Final = _leading_instruction_block_length(roles) - closing_start: Final = _closing_instruction_block_start(roles, leading_length) - return tuple( - index for index, role in enumerate(roles[:closing_start]) if index < leading_length or role == "developer" - ) - - -def _with_instruction_items_folded( - input: str | ResponseInputParam, instructions: str | None -) -> tuple[str | None, str | ResponseInputParam]: - if isinstance(input, str): - return instructions, input - items: Final = tuple(input) - folded: Final = MappingProxyType( - { - index: text - for index in _hoisted_indices(tuple(map(_role, items))) - if (text := _text_only_content(items[index])) is not None - } - ) - joined: Final = "\n\n".join(chunk for chunk in (instructions, *folded.values()) if chunk) - return ( - instructions if not folded else joined or None, - [_developer_item_as_system(item) for index, item in enumerate(items) if index not in folded], - ) - - class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig): @property def custom_llm_provider(self) -> LlmProviders: @@ -158,29 +79,28 @@ class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig): self, model: str, input: str | ResponseInputParam, - response_api_optional_request_params: dict, # mutable-ok: overrides the base class signature + response_api_optional_request_params: dict[str, object], # mutable-ok: overrides the base class signature litellm_params: GenericLiteLLMParams, - headers: dict, # mutable-ok: overrides the base class signature - ) -> dict: # mutable-ok: overrides the base class signature - instructions_param: Final[object] = response_api_optional_request_params.get("instructions") + headers: dict[str, str], # mutable-ok: overrides the base class signature + ) -> dict[str, object]: # mutable-ok: overrides the base class signature validated_input: Final = self._validate_input_param(input) - instructions, folded_input = ( - _with_instruction_items_folded(validated_input, instructions_param) - if isinstance(instructions_param, str | None) - else (instructions_param, _developer_items_as_system(validated_input)) - ) - instruction_entries: Final = () if instructions is None else (("instructions", instructions),) - folded_params: Final = { - key: value - for key, value in ( - *((key, value) for key, value in response_api_optional_request_params.items() if key != "instructions"), - *instruction_entries, + model_info: Final = deployment_model_info(litellm_params) + declared: Final = declared_mid_conversation_system_support(model, self.custom_llm_provider.value, model_info) + folded: Final = ( + None + if deployment_declared_mid_conversation_system_support(model_info) is True + else instruction_items_folded( + validated_input, response_api_optional_request_params, "user" if declared is False else "system" ) - } + ) + placed_input, placed_params = folded or ( + developer_items_as_system(validated_input), + response_api_optional_request_params, + ) request: Final = super().transform_responses_api_request( model=resolve_fireworks_resource_name(model), - input=folded_input, - response_api_optional_request_params=folded_params, + input=placed_input, + response_api_optional_request_params=dict(placed_params), litellm_params=litellm_params, headers=headers, ) diff --git a/litellm/llms/github_copilot/authenticator.py b/litellm/llms/github_copilot/authenticator.py index 59e1804f453..ee0e7af7e9b 100644 --- a/litellm/llms/github_copilot/authenticator.py +++ b/litellm/llms/github_copilot/authenticator.py @@ -1,11 +1,12 @@ import json import os import time +from dataclasses import dataclass from datetime import datetime -from typing import Any, Final +from typing import Any, Final, TypeVar import httpx -from pydantic import ConfigDict, TypeAdapter, with_config +from pydantic import BaseModel, ConfigDict, TypeAdapter, with_config from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm._logging import verbose_logger @@ -13,7 +14,6 @@ from litellm.litellm_core_utils.asyncify import can_block_current_thread from litellm.llms.custom_httpx.http_handler import get_httpx_client from .common_utils import ( - APIKeyExpiredError, GetAccessTokenError, GetAPIKeyError, GetDeviceCodeError, @@ -27,6 +27,44 @@ DEFAULT_GITHUB_ACCESS_TOKEN_URL: Final = "https://github.com/login/oauth/access_ DEFAULT_GITHUB_API_KEY_URL: Final = "https://api.github.com/copilot_internal/v2/token" +class CopilotEndpoints(BaseModel): + model_config = ConfigDict(frozen=True, strict=True, extra="ignore", hide_input_in_errors=True) + + api: str | None = None + + +class CopilotCachedKey(BaseModel): + model_config = ConfigDict(frozen=True, strict=True, extra="ignore", hide_input_in_errors=True) + + token: str + expires_at: float + endpoints: CopilotEndpoints | None = None + + +class CopilotCachedEndpoints(BaseModel): + model_config = ConfigDict(frozen=True, strict=True, extra="ignore", hide_input_in_errors=True) + + endpoints: CopilotEndpoints | None = None + + +ApiKeyFileView = TypeVar("ApiKeyFileView", CopilotCachedKey, CopilotCachedEndpoints) + + +@dataclass(frozen=True, slots=True) +class _UnsavedApiKey: + access_token: str + key: CopilotCachedKey + + +_UNSAVED_API_KEYS: Final[dict[str, _UnsavedApiKey]] = {} # mutable-ok: refreshed keys api-key.json could not hold + + +def _read_token_file(path: str) -> str: + with open(path, "rb") as f: + raw: Final = f.read() + return raw.decode(json.detect_encoding(raw)) + + @with_config(ConfigDict(extra="allow", strict=True)) class _DeviceCode(TypedDict): device_code: ReadOnly[str] @@ -87,12 +125,13 @@ class Authenticator: GetAccessTokenError: If unable to obtain an access token after retries. """ try: - with open(self.access_token_file, "r") as f: - access_token = f.read().strip() - if access_token: - return access_token + saved_token: Final = _read_token_file(self.access_token_file).strip() + if saved_token: + return saved_token except OSError: verbose_logger.warning("No existing access token found or error reading file") + except ValueError as e: + verbose_logger.warning("Ignoring %s, not a text access token: %s", self.access_token_file, e) if not can_block_current_thread(): raise GetAccessTokenError( @@ -135,64 +174,83 @@ class Authenticator: Raises: GetAPIKeyError: If unable to obtain an API key. """ - try: - with open(self.api_key_file, "r") as f: - api_key_info = json.load(f) - if api_key_info.get("expires_at", 0) > datetime.now().timestamp(): - return api_key_info.get("token") - else: - verbose_logger.warning("API key expired, refreshing") - raise APIKeyExpiredError( - message="API key expired", - status_code=401, - ) - except OSError: - verbose_logger.warning("No API key file found or error opening file") - except (json.JSONDecodeError, KeyError) as e: - verbose_logger.warning("Error reading API key from file: %s", e) - except APIKeyExpiredError: - pass # Already logged in the try block + unsaved: Final = self._unsaved_api_key() + if unsaved is not None: + return unsaved.token + cached: Final = self._read_api_key_file(CopilotCachedKey) + if cached is not None and cached.expires_at > datetime.now().timestamp(): + return cached.token + if cached is not None: + verbose_logger.warning("API key expired, refreshing") try: - api_key_info = self._refresh_api_key() - with open(self.api_key_file, "w") as f: - json.dump(api_key_info, f) - token: Final = api_key_info.get("token") - if token: - return token - else: - raise GetAPIKeyError( - message="API key response missing token", - status_code=401, - ) - except OSError as e: - verbose_logger.error("Error saving API key to file: %s", e) - raise GetAPIKeyError( - message=f"Failed to save API key: {e}", - status_code=500, - ) + api_key_info: Final = self._refresh_api_key() except RefreshAPIKeyError as e: raise GetAPIKeyError( message=f"Failed to refresh API key: {e}", status_code=401, + ) from e + serialized: Final = json.dumps(api_key_info) + try: + with open(self.api_key_file, "w") as f: + f.write(serialized) + except OSError as e: + verbose_logger.warning( + "Keeping the refreshed API key in memory, it could not be saved to %s: %s", self.api_key_file, e ) + self._keep_unsaved_api_key(serialized) + token: Final = api_key_info.get("token") + if not token: + raise GetAPIKeyError( + message="API key response missing token", + status_code=401, + ) + return token def get_api_base(self) -> str | None: """ - Get the API endpoint from the api-key.json file. + Get the API endpoint of the current API key. Returns: Optional[str]: The GitHub Copilot API endpoint, or None if not found. """ - try: - with open(self.api_key_file, "r") as f: - api_key_info: Final = json.load(f) - endpoints: Final = api_key_info.get("endpoints", {}) - api_endpoint: Final = endpoints.get("api") - return api_endpoint - except (OSError, json.JSONDecodeError, KeyError) as e: - verbose_logger.warning("Error reading API endpoint from file: %s", e) + cached: Final = self._unsaved_api_key() or self._read_api_key_file(CopilotCachedEndpoints) + if cached is None or cached.endpoints is None: return None + return cached.endpoints.api + + def _read_api_key_file(self, view: type[ApiKeyFileView]) -> ApiKeyFileView | None: + try: + return view.model_validate_json(_read_token_file(self.api_key_file)) + except OSError: + verbose_logger.warning("No API key file found or error opening file") + return None + except ValueError as e: + verbose_logger.warning("Ignoring %s, not a Copilot API key file: %s", self.api_key_file, e) + return None + + def _stored_access_token(self) -> str | None: + try: + access_token: Final = _read_token_file(self.access_token_file).strip() + except (OSError, ValueError): + return None + return access_token or None + + def _unsaved_api_key(self) -> CopilotCachedKey | None: + unsaved: Final = _UNSAVED_API_KEYS.get(self.api_key_file) + if unsaved is None or unsaved.key.expires_at <= time.time(): + return None + return unsaved.key if unsaved.access_token == self._stored_access_token() else None + + def _keep_unsaved_api_key(self, serialized: str) -> None: + access_token: Final = self._stored_access_token() + if access_token is None: + return + try: + key: Final = CopilotCachedKey.model_validate_json(serialized) + except ValueError: + return + _UNSAVED_API_KEYS[self.api_key_file] = _UnsavedApiKey(access_token=access_token, key=key) def _refresh_api_key(self) -> dict[str, Any]: """ diff --git a/litellm/llms/hosted_vllm/responses/transformation.py b/litellm/llms/hosted_vllm/responses/transformation.py index 8489bc0da84..0831ed3b37a 100644 --- a/litellm/llms/hosted_vllm/responses/transformation.py +++ b/litellm/llms/hosted_vllm/responses/transformation.py @@ -9,10 +9,13 @@ the chat completions → responses conversion pipeline. from typing import Final from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.openai.responses.instruction_items import instruction_items_folded from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import ResponseInputParam from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders +from litellm.utils import declared_mid_conversation_system_support, deployment_model_info class HostedVLLMResponsesAPIConfig(OpenAIResponsesAPIConfig): @@ -69,6 +72,31 @@ class HostedVLLMResponsesAPIConfig(OpenAIResponsesAPIConfig): return join_api_base(api_base, "v1/responses") + def transform_responses_api_request( + self, + model: str, + input: str | ResponseInputParam, + response_api_optional_request_params: dict[str, object], # mutable-ok: overrides the base class signature + litellm_params: GenericLiteLLMParams, + headers: dict[str, str], # mutable-ok: overrides the base class signature + ) -> dict[str, object]: # mutable-ok: overrides the base class signature + declared: Final = declared_mid_conversation_system_support( + model, self.custom_llm_provider.value, deployment_model_info(litellm_params) + ) + folded: Final = ( + instruction_items_folded(self._validate_input_param(input), response_api_optional_request_params, "user") + if declared is False + else None + ) + placed_input, placed_params = folded or (input, response_api_optional_request_params) + return super().transform_responses_api_request( + model=model, + input=placed_input, + response_api_optional_request_params=dict(placed_params), + litellm_params=litellm_params, + headers=headers, + ) + def supports_native_websocket(self) -> bool: """Hosted vLLM does not support native WebSocket for Responses API""" return False diff --git a/litellm/llms/manus/responses/transformation.py b/litellm/llms/manus/responses/transformation.py index ed06391a260..91832899141 100644 --- a/litellm/llms/manus/responses/transformation.py +++ b/litellm/llms/manus/responses/transformation.py @@ -241,7 +241,7 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig): api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, - ) -> tuple[str, dict]: + ) -> tuple[str, dict[str, str]]: """ Transform the get response API request into a URL and data. @@ -252,7 +252,7 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig): """ encoded_response_id: Final = encode_url_path_segment(response_id, field_name="response_id") url: Final = f"{api_base}/{encoded_response_id}" - data: Final[dict] = {} + data: Final[dict[str, str]] = {} return url, data def transform_get_response_api_response( diff --git a/litellm/llms/openai/chat/gpt_reasoning_transformation.py b/litellm/llms/openai/chat/gpt_reasoning_transformation.py index 0311f6c6590..8c9551d01c0 100644 --- a/litellm/llms/openai/chat/gpt_reasoning_transformation.py +++ b/litellm/llms/openai/chat/gpt_reasoning_transformation.py @@ -5,8 +5,10 @@ from collections.abc import Collection from typing import ClassVar, Final import litellm +from litellm._logging import verbose_logger from litellm.utils import ( declared_value_factory, + has_catalog_entry, is_explicitly_disabled_factory, supports_factory, ) @@ -16,6 +18,25 @@ from .gpt_transformation import OpenAIGPTConfig _GPT_SERIES_VERSION: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?(?=[.-]|$)") +def refused_reasoning_effort_warning(*, suggest_declared_levels: bool, suggest_base_model: bool) -> str: + return "".join( + ( + "Dropping %s=%r for model=%r (drop_params=True) because the model map does not mark that level " + "supported for %r.", + ( + " If the deployment supports it, list every level it supports in model_info.reasoning_effort_levels." + if suggest_declared_levels + else "" + ), + ( + " If the deployment name is not its underlying model, set model_info.base_model to that model." + if suggest_base_model + else "" + ), + ) + ) + + def _catalogue_declares_default_effort() -> bool: """Whether the loaded cost map carries default_reasoning_effort for ANY entry. @@ -93,6 +114,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig): OPT_IN_REASONING_EFFORTS: ClassVar[frozenset[str]] = frozenset(("xhigh",)) ROUTE_SPECIFIC_REASONING_EFFORTS: ClassVar[frozenset[str]] = frozenset(("max",)) + LOOKS_UP_BASE_MODEL: ClassVar[bool] = False @classmethod def is_model_gpt_5_model(cls, model: str) -> bool: @@ -307,7 +329,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig): model: str, drop_params: bool, *, - declared_reasoning_efforts: Collection[str] = (), + declared_reasoning_efforts: Collection[str] | None = None, ) -> dict: if self.is_model_gpt_5_search_model(model): if "max_tokens" in non_default_params: @@ -338,10 +360,21 @@ class OpenAIGPT5Config(OpenAIGPTConfig): if ( requested_effort is not None - and requested_effort not in declared_reasoning_efforts + and requested_effort not in (declared_reasoning_efforts or ()) and self.refuses_reasoning_effort_level(model, requested_effort) ): if litellm.drop_params or drop_params: + verbose_logger.warning( + refused_reasoning_effort_warning( + suggest_declared_levels=declared_reasoning_efforts is not None, + suggest_base_model=self.LOOKS_UP_BASE_MODEL + and not has_catalog_entry(self._model_map_lookup_name(model), None), + ), + "reasoning_effort", + requested_effort, + model, + self._model_map_lookup_name(model), + ) non_default_params.pop("reasoning_effort", None) optional_params.pop("reasoning_effort", None) else: diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index b7396e8fb03..22af51aaabe 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -36,7 +36,11 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import ( ) from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator -from litellm.llms.base_llm.base_utils import BaseLLMModelInfo +from litellm.llms.base_llm.base_utils import ( + BaseLLMModelInfo, + hoist_developer_messages_into_leading_system_message, + map_developer_role_to_system_role, +) from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( @@ -58,7 +62,7 @@ from litellm.types.utils import ( ModelResponseStream, chat_completion_tool_call_from_dict, ) -from litellm.utils import convert_to_model_response_object +from litellm.utils import convert_to_model_response_object, declared_mid_conversation_system_support from ..common_utils import OpenAIError from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config @@ -75,6 +79,45 @@ else: _NO_TOOLS_UPDATE: Final[Mapping[str, object]] = MappingProxyType({}) + + +def targets_openai_hosted_endpoint(custom_llm_provider: str | None, api_base: str | None) -> bool: + if custom_llm_provider != "openai": + return False + resolved_api_base: Final = ( + api_base or litellm.api_base or os.getenv("OPENAI_BASE_URL") or os.getenv("OPENAI_API_BASE") + ) + if not resolved_api_base: + return True + hostname: Final = urlparse(resolved_api_base).hostname + if hostname is None: + return True + return hostname == "openai.com" or hostname.endswith(".openai.com") + + +def _roles(messages: Sequence[AllMessageValues]) -> tuple[str, ...]: + return tuple(message["role"] for message in messages) + + +def translate_developer_role_for_openai_compatible_endpoint( + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, +) -> Sequence[AllMessageValues]: + in_place: Final = map_developer_role_to_system_role(messages=messages) + if targets_openai_hosted_endpoint(custom_llm_provider, api_base): + return in_place + placed: Final = hoist_developer_messages_into_leading_system_message(messages=messages) + if _roles(placed) == _roles(in_place): + return in_place + if declared_mid_conversation_system_support(model, custom_llm_provider, model_info) is not False: + return in_place + return placed + + _OBJECT_MAPPING: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) _CHOICE_MAPPINGS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) _DECODED_JSON: Final = TypeAdapter(object) @@ -437,20 +480,30 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): ) return messages, tools + def translate_developer_role_to_system_role( + self, + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: + """ + A developer message only changes role, except on a model whose deployment or cost + map declares `supports_mid_conversation_system` false, where the instructions are + placed in one leading system message for its chat template. + """ + return translate_developer_role_for_openai_compatible_endpoint( + messages, model=model, custom_llm_provider=custom_llm_provider, api_base=api_base, model_info=model_info + ) + def _targets_openai_hosted_endpoint( self, custom_llm_provider: str | None, api_base: str | None, ) -> bool: - if custom_llm_provider != "openai": - return False - resolved_api_base = api_base or litellm.api_base or os.getenv("OPENAI_BASE_URL") or os.getenv("OPENAI_API_BASE") - if not resolved_api_base: - return True - hostname: Final = urlparse(resolved_api_base).hostname - if hostname is None: - return True - return hostname == "openai.com" or hostname.endswith(".openai.com") + return targets_openai_hosted_endpoint(custom_llm_provider, api_base) def _should_preserve_cache_control_for_endpoint( self, diff --git a/litellm/llms/openai/chat/o_series_transformation.py b/litellm/llms/openai/chat/o_series_transformation.py index c7552d47985..64285aafdb0 100644 --- a/litellm/llms/openai/chat/o_series_transformation.py +++ b/litellm/llms/openai/chat/o_series_transformation.py @@ -11,7 +11,7 @@ Translations handled by LiteLLM: - Logprobs => drop param (if user opts in to dropping param) """ -from collections.abc import Coroutine +from collections.abc import Coroutine, Sequence from typing import Final, Literal, cast, overload import litellm @@ -37,7 +37,15 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig): def get_config(cls): return super().get_config() - def translate_developer_role_to_system_role(self, messages: list[AllMessageValues]) -> list[AllMessageValues]: + def translate_developer_role_to_system_role( + self, + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: """ O-series models support `developer` role. """ diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index 12ead582975..a9d2dcdffb8 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -31,7 +31,7 @@ if TYPE_CHECKING: import litellm from litellm.litellm_core_utils.completion_timeout import CompletionTimeout from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object -from litellm.litellm_core_utils.token_counter import token_counter +from litellm.litellm_core_utils.token_counter import token_counter, usage_tokenizer_model from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.custom_httpx.http_handler import ( _DEFAULT_TTL_FOR_HTTPX_CLIENTS, @@ -104,6 +104,19 @@ def is_openai_backed_api_base(api_base: str) -> bool: return hostname is not None and (hostname == _OPENAI_API_HOST or hostname.endswith(f".{_OPENAI_API_HOST}")) +_OPENAI_AGENTS_API_RESOURCES: Final = frozenset({"agents", "vaults"}) + + +def openai_api_resource(path: str) -> str | None: + segments: Final = tuple(segment for segment in path.split("/") if segment) + resource_segments: Final = segments[1:] if segments[:1] == ("v1",) else segments + return next(iter(resource_segments), None) + + +def is_openai_agents_api_path(path: str) -> bool: + return openai_api_resource(path) in _OPENAI_AGENTS_API_RESOURCES + + class OpenAIError(BaseLLMException): def __init__( self, @@ -242,7 +255,7 @@ async def _aiter_once(chunk: ChatCompletionChunk) -> AsyncIterator[ChatCompletio def build_output_token_limit_response( - e: openai.BadRequestError, data: Mapping[str, object], is_async: bool + e: openai.BadRequestError, data: Mapping[str, object], is_async: bool, base_model: str | None ) -> tuple[httpx.Headers, ChatCompletion | Iterator[ChatCompletionChunk] | AsyncIterator[ChatCompletionChunk]]: """Synthesize the length-truncated response the provider itself returns for slightly larger output budgets. @@ -251,7 +264,11 @@ def build_output_token_limit_response( """ model: Final[str] = str(data.get("model", "")) messages: Final = data.get("messages") - prompt_tokens: Final = token_counter(model=model, messages=messages) if isinstance(messages, list) else 0 + prompt_tokens: Final = ( + token_counter(model=usage_tokenizer_model(model, base_model), messages=messages) + if isinstance(messages, list) + else 0 + ) if not data.get("stream"): return e.response.headers, _output_token_limit_completion(model, prompt_tokens) chunk: Final = _output_token_limit_chunk(model) diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 9d4df354a30..af2e85c5f20 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -1,6 +1,6 @@ import time import types -from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Iterator, Mapping +from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Final, Literal, Optional, cast import httpx @@ -44,12 +44,17 @@ from litellm.utils import ( CustomStreamWrapper, ProviderConfigManager, convert_to_model_response_object, + get_base_model_from_metadata, ) from ...types.llms.openai import * from ..base import BaseLLM from .chat.gpt_reasoning_transformation import OpenAIGPT5Config -from .chat.gpt_transformation import OpenAIGPTConfig, OpenAIUnknownModelConfig +from .chat.gpt_transformation import ( + OpenAIGPTConfig, + OpenAIUnknownModelConfig, + translate_developer_role_for_openai_compatible_endpoint, +) from .chat.o_series_transformation import OpenAIOSeriesConfig from .common_utils import ( BaseOpenAILLM, @@ -205,6 +210,19 @@ class OpenAIConfig(BaseConfig): else: return self._gpt_config_for_model(model).get_supported_openai_params(model=model) + def translate_developer_role_to_system_role( + self, + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: + return translate_developer_role_for_openai_compatible_endpoint( + messages, model=model, custom_llm_provider=custom_llm_provider, api_base=api_base, model_info=model_info + ) + def _gpt_config_for_model(self, model: str) -> OpenAIGPTConfig: if type(self) is OpenAIConfig and not OpenAIGPTConfig.is_openai_catalog_model(model): return OpenAIUnknownModelConfig() @@ -557,7 +575,12 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): except openai.BadRequestError as e: if not is_output_token_limit_error(e): raise - return build_output_token_limit_response(e=e, data=data, is_async=True) + return build_output_token_limit_response( + e=e, + data=data, + is_async=True, + base_model=get_base_model_from_metadata(logging_obj.model_call_details), + ) except Exception as e: raise e @@ -604,7 +627,12 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): except openai.BadRequestError as e: if not is_output_token_limit_error(e): raise - return build_output_token_limit_response(e=e, data=data, is_async=False) + return build_output_token_limit_response( + e=e, + data=data, + is_async=False, + base_model=get_base_model_from_metadata(logging_obj.model_call_details), + ) except Exception as e: if raw_response is not None: raise Exception( diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index b7e4cc6868f..5be488f0f6e 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -4,11 +4,17 @@ This file contains the calling OpenAI's `/v1/realtime` endpoint. This requires websockets, and is currently only supported on LiteLLM Proxy. """ +import asyncio import ssl -from typing import TYPE_CHECKING, Final, cast +from collections.abc import Mapping +from typing import TYPE_CHECKING, Final, Literal, cast + +from pydantic import TypeAdapter from litellm._logging import redact_string, verbose_logger from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES +from litellm.exceptions import AuthenticationError +from litellm.responses.streaming_iterator import map_stream_error_to_exception from litellm.types.realtime import RealtimeQueryParams from ....litellm_core_utils.litellm_logging import Logging as LiteLLMLogging @@ -25,6 +31,54 @@ from ..openai import OpenAIChatCompletion if TYPE_CHECKING: from fastapi import WebSocket + from openai.types.realtime import RealtimeError + from websockets.asyncio.client import ClientConnection + +_SERVER_EVENT_FIELDS: Final = TypeAdapter(Mapping[str, object]) +_CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL: Final = "invalid_model" +_CREDENTIALS_REJECTED: Final = "invalid_api_key" + + +def first_event_error(first_event: str | bytes) -> "RealtimeError | None": + event: Final = _SERVER_EVENT_FIELDS.validate_json(first_event) + if event.get("type") != "error": + return None + from openai.types.realtime import RealtimeErrorEvent + + return RealtimeErrorEvent.model_validate(event).error + + +def _confirm_first_event( + first_event: str | bytes, model: str, sent_credentials: bool, model_listed_outside_realtime: bool +) -> Literal[True]: + error: Final = first_event_error(first_event) + if error is None or ( + error.code == _CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL and not model_listed_outside_realtime + ): + return True + if error.code == _CREDENTIALS_REJECTED or not sent_credentials: + raise AuthenticationError(message=error.message, llm_provider="openai", model=model) + raise map_stream_error_to_exception(error, model=model, custom_llm_provider="openai") + + +async def confirm_session_started( + connection: "ClientConnection", model: str, sent_credentials: bool, model_listed_outside_realtime: bool +) -> Literal[True]: + return _confirm_first_event(await connection.recv(), model, sent_credentials, model_listed_outside_realtime) + + +async def confirm_gateway_session_started( + connection: "ClientConnection", + model: str, + sent_credentials: bool, + model_listed_outside_realtime: bool, + first_event_wait_seconds: float, +) -> Literal[True]: + try: + first_event: Final = await asyncio.wait_for(connection.recv(), first_event_wait_seconds) + except asyncio.TimeoutError: + return True + return _confirm_first_event(first_event, model, sent_credentials, model_listed_outside_realtime) class OpenAIRealtime(OpenAIChatCompletion): diff --git a/litellm/llms/openai/responses/count_tokens/transformation.py b/litellm/llms/openai/responses/count_tokens/transformation.py index b4e152ff406..a45a0448326 100644 --- a/litellm/llms/openai/responses/count_tokens/transformation.py +++ b/litellm/llms/openai/responses/count_tokens/transformation.py @@ -22,7 +22,7 @@ from litellm.llms.anthropic.messages.responses_adapters.transformation import ( ) from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.anthropic import AnthropicMessagesRequest, AnthropicOutputSchema -from litellm.types.llms.openai import ResponsesAPIRequestParams +from litellm.types.llms.openai import ResponsesAPIRequestParams, replayable_responses_input from litellm.types.utils import all_litellm_params @@ -387,10 +387,13 @@ def _forwards(field: str, policy: DropPolicy) -> bool: return field in _COUNT_REQUEST_FIELDS or (not policy.drop_params and field in _INFERENCE_REQUEST_FIELDS) -def _restored(field: str, value: object) -> object: +def _restored(field: str, value: object, policy: DropPolicy) -> object: match field, value: case "input", _: - return ResponsesAPIRequestUtils.restore_encrypted_content_item_ids_in_input(copy.deepcopy(value)) + return replayable_responses_input( + ResponsesAPIRequestUtils.restore_encrypted_content_item_ids_in_input(copy.deepcopy(value)), + drops_set_created_by=policy.drop_params, + ) case "previous_response_id", str(): return ResponsesAPIRequestUtils.decode_previous_response_id_to_original_previous_response_id(value) case _: @@ -398,5 +401,7 @@ def _restored(field: str, value: object) -> object: def native_count_request(request: Mapping[str, object], model: str, policy: DropPolicy) -> Mapping[str, object]: - forwarded: Final = {field: _restored(field, value) for field, value in request.items() if _forwards(field, policy)} + forwarded: Final = { + field: _restored(field, value, policy) for field, value in request.items() if _forwards(field, policy) + } return {**forwarded, "model": model} diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index d37a12a91d6..559daf767c2 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -32,11 +32,11 @@ import copy import json import time import uuid -from collections.abc import Mapping, MutableSequence, Sequence +from collections.abc import Iterable, Mapping, MutableSequence, Sequence from dataclasses import dataclass -from itertools import accumulate, chain, repeat +from itertools import accumulate, chain, groupby, repeat from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, Union, cast +from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, TypeVar, Union, cast from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict @@ -55,7 +55,9 @@ from litellm.llms.base_llm.guardrail_translation.base_translation import ( ) from litellm.llms.base_llm.guardrail_translation.utils import ( blocked_responses_stream_usage, + effective_scan_only_tool_results_for_guardrail, effective_skip_system_message_for_guardrail, + effective_skip_tool_message_for_guardrail, list_field, merge_guardrailed_scoped_messages, role_out_of_guardrail_scope, @@ -65,7 +67,10 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( stream_item_items, unappliable_request_rewrite, ) -from litellm.llms.openai.responses.guardrail_translation.tool_merge import merge_guardrailed_tools +from litellm.llms.openai.responses.guardrail_translation.tool_merge import ( + append_returned_tools, + merge_guardrailed_tools, +) from litellm.responses.litellm_completion_transformation.compaction_items import ( compaction_summary, decode_compaction_item, @@ -74,7 +79,9 @@ from litellm.responses.litellm_completion_transformation.compaction_items import from litellm.responses.litellm_completion_transformation.custom_tools import without_malformed_tool_flags from litellm.responses.litellm_completion_transformation.tool_search import is_server_tool_search_tool from litellm.responses.litellm_completion_transformation.transformation import ( + TOOL_CALL_OUTPUT_ITEM_TYPES, LiteLLMCompletionResponsesConfig, + tool_output_to_chat_content, ) from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import ( @@ -270,6 +277,7 @@ _TOOL_CALL_PAYLOAD_EVENT_TYPES: Final = _TOOL_CALL_PAYLOAD_DELTA_EVENT_TYPES | f ) _OUTPUT_ITEM_EVENT_TYPES: Final = frozenset({"response.output_item.added", "response.output_item.done"}) _OUTPUT_TEXT_EVENT_TYPES: Final = frozenset({"response.output_text.delta", "response.output_text.done"}) +_TOOL_OUTPUT_TEXT_PART_TYPES: Final = frozenset({"input_text", "output_text", "text"}) _PATCHABLE_ITEM_FIELDS: Final[Mapping[str, str]] = MappingProxyType( {"function_call_output": "output", "custom_tool_call_output": "output", "message": "content"} ) @@ -519,10 +527,109 @@ def scannable_instructions(data: Mapping[str, object], *, skip_system: bool = Fa def _input_item_role(item: object) -> str: + if stream_item_field(item, "type") in TOOL_CALL_OUTPUT_ITEM_TYPES: + return "tool" role: Final = item.get("role") if isinstance(item, Mapping) else None return role.lower() if isinstance(role, str) else "" +def _scanned_text_field(item: Mapping[str, object]) -> str: + return "output" if item.get("type") in TOOL_CALL_OUTPUT_ITEM_TYPES else "content" + + +def _tool_output_part_text(part: object) -> str | None: + if not isinstance(part, dict): + return None + fields: Final = _item_fields_adapter.validate_python(part) + text: Final = fields.get("text") + return text if fields.get("type") in _TOOL_OUTPUT_TEXT_PART_TYPES and isinstance(text, str) else None + + +def tool_output_texts(item: object) -> tuple[tuple[int | None, str], ...]: + if not stream_item_field(item, "call_id") or stream_item_field(item, "type") == "computer_call_output": + return () + parts: Final = stream_item_items(item, "output") + part_texts: Final = tuple( + (index, text) for index, part in enumerate(parts) if (text := _tool_output_part_text(part)) + ) + if part_texts: + return part_texts + tool_content: Final = tool_output_to_chat_content(stream_item_field(item, "output")) + return ((None, tool_content),) if isinstance(tool_content, str) and tool_content else () + + +_Grouped: Final = TypeVar("_Grouped") + + +def _pair_call_id(pair: tuple[str, object]) -> str: + return pair[0] + + +def _grouped_by_call_id(pairs: Iterable[tuple[str, _Grouped]]) -> Mapping[str, tuple[_Grouped, ...]]: + ordered: Final = sorted(pairs, key=_pair_call_id) + return MappingProxyType( + {call_id: tuple(value for _, value in group) for call_id, group in groupby(ordered, key=_pair_call_id)} + ) + + +def _texts_by_tool_row( + messages: Sequence[Mapping[str, object]], input_data: "str | ResponseInputParam" +) -> Mapping[int, tuple[str, ...]]: + if isinstance(input_data, str): + return MappingProxyType({}) + output_texts_by_call_id: Final = _grouped_by_call_id( + (str(stream_item_field(item, "call_id")), tuple(text for _, text in tool_output_texts(item))) + for item in input_data + if stream_item_field(item, "type") in TOOL_CALL_OUTPUT_ITEM_TYPES + ) + row_indices_by_call_id: Final = _grouped_by_call_id( + (str(message.get("tool_call_id")), index) + for index, message in enumerate(messages) + if message.get("role") == "tool" + ) + return MappingProxyType( + dict( + chain.from_iterable( + zip(indices, output_texts_by_call_id[call_id], strict=True) + for call_id, indices in row_indices_by_call_id.items() + if len(indices) == len(output_texts_by_call_id.get(call_id, ())) + ) + ) + ) + + +def _is_string_tool_row(message: Mapping[str, object]) -> bool: + return message.get("role") == "tool" and isinstance(message.get("content"), str) + + +def _split_into_text_parts(message: AllMessageValues, texts: tuple[str, ...]) -> AllMessageValues: + if not _is_string_tool_row(message) or len(texts) < 2: + return message + return cast( # cast-ok: a tool row whose content is its text parts as chat text blocks + "AllMessageValues", {**message, "content": [{"type": "text", "text": text} for text in texts]} + ) + + +def _with_split_rows_restored( + rows: Sequence[AllMessageValues], split_rows: Sequence[tuple[AllMessageValues, AllMessageValues]] +) -> Sequence[AllMessageValues]: + if not isinstance(rows, list): + return rows + return [next((original for split, original in split_rows if row == split), row) for row in rows] + + +def _is_textless_tool_row(message: Mapping[str, object], texts: tuple[str, ...] | None) -> bool: + return _is_string_tool_row(message) and (not message.get("content") or texts == ()) + + +def _rewritten_whole_text(item: object, field: str, guardrail_response: str) -> object | None: + if isinstance(stream_item_field(item, field), str): + return guardrail_response + if tool_output_texts(item) == ((None, guardrail_response),): + return None + return guardrail_response + + def _patched_request_fields( raw_input: object, instructions: object, @@ -636,17 +743,28 @@ class OpenAIResponsesHandler(BaseTranslation): Handles both string input and list of message objects. """ - input_data: Final[str | ResponseInputParam | None] = data.get("input") - if not isinstance(input_data, (str, list)): + raw_input: Final[str | ResponseInputParam | None] = data.get("input") + if not isinstance(raw_input, (str, list)): return data + input_data: Final = cast("str | ResponseInputParam", raw_input) # cast-ok: fields read via stream_item_field skip_system: Final = effective_skip_system_message_for_guardrail(guardrail_to_apply) + skip_tool: Final = effective_skip_tool_message_for_guardrail(guardrail_to_apply) + scan_only: Final = effective_scan_only_tool_results_for_guardrail(guardrail_to_apply) structured_messages: Final = self.get_structured_messages(data) - scoped_indices: Final = scoped_structured_message_indices( - structured_messages or [], scan_only_tool_results=False, skip_system=skip_system, skip_tool=False + all_messages: Final = structured_messages or [] + texts_by_tool_row: Final = _texts_by_tool_row(all_messages, input_data) + scoped_indices: Final = tuple( + index + for index in scoped_structured_message_indices( + all_messages, scan_only_tool_results=scan_only, skip_system=skip_system, skip_tool=skip_tool + ) + if not _is_textless_tool_row(all_messages[index], texts_by_tool_row.get(index)) ) - scoped_structured_messages: Final = ( - [structured_messages[index] for index in scoped_indices] if structured_messages else None + split_rows: Final = tuple( + (_split_into_text_parts(all_messages[index], texts_by_tool_row.get(index, ())), all_messages[index]) + for index in scoped_indices ) + scoped_structured_messages: Final = [split for split, _ in split_rows] if structured_messages else None raw_tools: Final = data.get("tools") original_tools: Final[tuple[Mapping[str, object], ...]] = ( tuple(raw_tools) if isinstance(raw_tools, list) else () @@ -661,7 +779,12 @@ class OpenAIResponsesHandler(BaseTranslation): ) ) extracted: Final = self._extract_guardrail_inputs( - data, input_data, flattened_tool_groups, skip_system=skip_system + data, + input_data, + () if scan_only else flattened_tool_groups, + skip_system=skip_system, + skip_tool=skip_tool, + scan_only=scan_only, ) if not extracted.inputs.get("texts"): return data @@ -674,13 +797,14 @@ class OpenAIResponsesHandler(BaseTranslation): logging_obj=litellm_logging_obj, ) self._apply_guardrailed_tools_to_data( - data, original_tools, flattened_tool_groups, guardrailed_inputs.get("tools") + data, original_tools, flattened_tool_groups, guardrailed_inputs.get("tools"), scan_only=scan_only ) written_back: Final = self._written_back_request_fields( data, structured_messages or (), scoped_indices, scoped_structured_messages, + tuple((split, original) for split, original in split_rows if split is not original), guardrail_to_apply, guardrailed_inputs, ) @@ -730,8 +854,10 @@ class OpenAIResponsesHandler(BaseTranslation): flattened_tool_groups: Sequence[Sequence[Mapping[str, object]]], *, skip_system: bool = False, + skip_tool: bool = False, + scan_only: bool = False, ) -> _ExtractedInputs: - instructions: Final = scannable_instructions(data, skip_system=skip_system) + instructions: Final = scannable_instructions(data, skip_system=skip_system or scan_only) texts_to_check: Final[list[str]] = [] if instructions is None else [instructions] images_to_check: Final[list[str]] = [] task_mappings: Final[list[tuple[int, int | None]]] = [] @@ -745,11 +871,15 @@ class OpenAIResponsesHandler(BaseTranslation): ) ) if isinstance(input_data, str): - texts_to_check.append(input_data) + if not scan_only: + texts_to_check.append(input_data) else: for msg_idx, message in enumerate(input_data): if role_out_of_guardrail_scope( - _input_item_role(message), skip_system_message=skip_system, skip_tool_message=False + _input_item_role(message), + skip_system_message=skip_system, + skip_tool_message=skip_tool, + scan_only_tool_results=scan_only, ): continue self._extract_input_text_and_images( @@ -775,12 +905,14 @@ class OpenAIResponsesHandler(BaseTranslation): structured_messages: Sequence[AllMessageValues], scoped_indices: Sequence[int], scoped_structured_messages: Sequence[AllMessageValues] | None, + split_rows: Sequence[tuple[AllMessageValues, AllMessageValues]], guardrail_to_apply: "CustomGuardrail", guardrailed_inputs: GenericGuardrailAPIInputs, ) -> _RequestFields | None: - guardrailed: Final = guardrailed_inputs.get("structured_messages") - if guardrailed is None or guardrailed is scoped_structured_messages: + returned: Final = guardrailed_inputs.get("structured_messages") + if returned is None or returned is scoped_structured_messages: return None + guardrailed: Final = _with_split_rows_restored(returned, split_rows) covers_full_request: Final = len(scoped_indices) == len(structured_messages) or ( guardrail_to_apply.structured_messages_cover_full_request() and len(guardrailed) == len(structured_messages) ) @@ -814,11 +946,14 @@ class OpenAIResponsesHandler(BaseTranslation): original_tools: Sequence[Mapping[str, object]], flattened_tool_groups: Sequence[Sequence[Mapping[str, object]]], guardrailed_tools: Sequence[ChatCompletionToolParam] | None, + *, + scan_only: bool, ) -> None: if guardrailed_tools is None: return + merge: Final = append_returned_tools if scan_only else merge_guardrailed_tools data["tools"] = list( # rebind-ok: in-place request rewrite - merge_guardrailed_tools(original_tools, flattened_tool_groups, guardrailed_tools) + merge(original_tools, flattened_tool_groups, guardrailed_tools) ) def _extract_input_text_and_images( @@ -839,6 +974,11 @@ class OpenAIResponsesHandler(BaseTranslation): texts_to_check.append(summary) task_mappings.append((msg_idx, None)) return + if message.get("type") in TOOL_CALL_OUTPUT_ITEM_TYPES: + for content_idx, text in tool_output_texts(message): + texts_to_check.append(text) + task_mappings.append((msg_idx, content_idx)) + return content: Final = message.get("content", None) if content is None: return @@ -890,18 +1030,20 @@ class OpenAIResponsesHandler(BaseTranslation): ) continue - content = messages[msg_idx].get("content", None) + field = _scanned_text_field(messages[msg_idx]) + content = messages[msg_idx].get(field, None) if content is None: continue - if isinstance(content, str) and content_idx_optional is None: - # Replace string content with guardrail response - messages[msg_idx]["content"] = guardrail_response + if content_idx_optional is None: + rewritten = _rewritten_whole_text(messages[msg_idx], field, guardrail_response) + if rewritten is not None: + messages[msg_idx][field] = rewritten - elif isinstance(content, list) and content_idx_optional is not None: + elif isinstance(content, list): # Replace specific text item in list content - if isinstance(messages[msg_idx]["content"][content_idx_optional], dict): - messages[msg_idx]["content"][content_idx_optional]["text"] = guardrail_response + if isinstance(messages[msg_idx][field][content_idx_optional], dict): + messages[msg_idx][field][content_idx_optional]["text"] = guardrail_response async def process_output_response( self, diff --git a/litellm/llms/openai/responses/guardrail_translation/tool_merge.py b/litellm/llms/openai/responses/guardrail_translation/tool_merge.py index f295a864b71..e529cba4edf 100644 --- a/litellm/llms/openai/responses/guardrail_translation/tool_merge.py +++ b/litellm/llms/openai/responses/guardrail_translation/tool_merge.py @@ -6,6 +6,7 @@ from typing import Final, TypeAlias from pydantic import BaseModel, TypeAdapter, ValidationError from litellm._logging import verbose_logger +from litellm.llms.base_llm.guardrail_translation.utils import merge_returned_tools_into_request_tools, openai_tool_name from litellm.responses.litellm_completion_transformation.custom_tools import custom_tool_grammar_suffix from litellm.responses.litellm_completion_transformation.transformation import ( NAMESPACE_DESCRIPTION_SEPARATOR, @@ -183,3 +184,18 @@ def merge_guardrailed_tools( tuple(tool for key, tool in zip(guardrailed_keys, guardrailed) if key not in owned_keys) ) return tuple(chain(merged_originals, appended)) + + +def append_returned_tools( + original_tools: Sequence[Tool], + flattened_groups: Sequence[Sequence[Tool]], + returned_tools: Iterable[object], +) -> tuple[Tool, ...]: + request_chat_tools: Final = tuple(chain.from_iterable(flattened_groups)) + additions: Final = merge_returned_tools_into_request_tools( + request_tools=request_chat_tools, returned_tools=_validated_tools(returned_tools), tool_name=openai_tool_name + )[len(request_chat_tools) :] + return ( + *original_tools, + *LiteLLMCompletionResponsesConfig.transform_chat_completion_tool_params_to_responses_api_tools(additions), + ) diff --git a/litellm/llms/openai/responses/instruction_items.py b/litellm/llms/openai/responses/instruction_items.py new file mode 100644 index 00000000000..82bf4329b34 --- /dev/null +++ b/litellm/llms/openai/responses/instruction_items.py @@ -0,0 +1,98 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final, Literal, TypeAlias + +from openai.types.responses import EasyInputMessageParam, ResponseInputContentParam, ResponseInputItemParam + +from litellm.llms.base_llm.base_utils import hoisted_instruction_indices +from litellm.types.llms.openai import ResponseInputParam + +UnfoldedDeveloperRole: TypeAlias = Literal["system", "user"] + +_INSTRUCTION_ROLES: Final = frozenset({"system", "developer"}) + + +def _role(item: ResponseInputItemParam) -> str | None: + match item: + case {"role": str(role)}: + return role + case _: + return None + + +def _developer_item_as(item: ResponseInputItemParam, role: UnfoldedDeveloperRole) -> ResponseInputItemParam: + if "role" not in item or item["role"] != "developer" or "content" not in item: + return item + return EasyInputMessageParam(role=role, content=item["content"], type="message") + + +def developer_items_as_system(input: str | ResponseInputParam) -> str | ResponseInputParam: + if isinstance(input, str): + return input + return [_developer_item_as(item, "system") for item in input] + + +def _text_part(part: ResponseInputContentParam) -> str | None: + match part: + case {"type": "input_text", "text": str(text)}: + return text + case _: + return None + + +def _text_only_content(item: ResponseInputItemParam) -> str | None: + match item: + case {"role": "system" | "developer", "content": str(text)}: + return text + case {"role": "system" | "developer", "content": [*parts]}: + texts: Final = tuple(map(_text_part, parts)) + return None if any(text is None for text in texts) else "\n\n".join(text for text in texts if text) + case _: + return None + + +def _folded_instructions_and_items( + items: tuple[ResponseInputItemParam, ...], instructions: str | None, unfolded_developer_role: UnfoldedDeveloperRole +) -> tuple[str | None, ResponseInputParam]: + folded: Final = MappingProxyType( + { + index: text + for index in hoisted_instruction_indices(tuple(map(_role, items))) + if (text := _text_only_content(items[index])) is not None + } + ) + joined: Final = "\n\n".join(chunk for chunk in (instructions, *folded.values()) if chunk) + return ( + instructions if not folded else joined or None, + [_developer_item_as(item, unfolded_developer_role) for index, item in enumerate(items) if index not in folded], + ) + + +def instruction_items_folded( + input: str | ResponseInputParam, + params: Mapping[str, object], + unfolded_developer_role: UnfoldedDeveloperRole, +) -> tuple[ResponseInputParam, Mapping[str, object]] | None: + """ + For a model whose chat template takes a single leading system message, the + leading system and developer items and every later text developer item folded into + top-level `instructions`, the developer items left over sent with + `unfolded_developer_role`, or None when the request needs no folding. + """ + instructions: Final = params.get("instructions") + if isinstance(input, str) or not isinstance(instructions, str | None): + return None + items: Final = tuple(input) + if not any(_role(item) in _INSTRUCTION_ROLES for item in items): + return None + folded_instructions, folded_items = _folded_instructions_and_items(items, instructions, unfolded_developer_role) + instruction_entries: Final = () if folded_instructions is None else (("instructions", folded_instructions),) + return folded_items, MappingProxyType( + { + key: value + for key, value in ( + *((key, value) for key, value in params.items() if key != "instructions"), + *instruction_entries, + ) + } + ) diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 9cb074a177e..9b8515a9ecd 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -1,4 +1,4 @@ -from collections.abc import Callable, Collection, Mapping, Sequence +from collections.abc import Callable, Collection, Iterable, Mapping, Sequence from functools import lru_cache from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Protocol, cast, get_type_hints @@ -10,7 +10,11 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger -from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook, targets_openai_api +from litellm.integrations.anthropic_cache_control_hook import ( + AnthropicCacheControlHook, + openai_prompt_cache_breakpoint_support, + targets_openai_api, +) from litellm.litellm_core_utils.core_helpers import normalize_drop_params, process_response_headers from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( @@ -20,6 +24,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( cache_control_as_prompt_cache_breakpoint, drop_non_python_regex_patterns, flatten_combinators_and_drop_non_python_regex_patterns, + prompt_cache_breakpoint_for_wire, with_prompt_cache_breakpoint, ) from litellm.litellm_core_utils.safe_json_loads import safe_json_loads @@ -27,6 +32,7 @@ from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_a from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.llms.openai.chat.gpt_reasoning_transformation import ( is_gpt_reasoning_series_name, + refused_reasoning_effort_warning, unsupported_reasoning_effort_message, ) from litellm.responses.litellm_completion_transformation.custom_tools import TOOL_CALL_ITEM_ID_PREFIX_BY_TYPE @@ -50,6 +56,9 @@ from ..common_utils import OpenAIError from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS: Final = 16 +DROP_UNSUPPORTED_REASONING_WARNING: Final = ( + "Dropping reasoning=%r for model=%r (drop_params=True) because its model cost map entry lacks supports_reasoning." +) _RAW_RESPONSE_JSON: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True)) if TYPE_CHECKING: @@ -66,10 +75,41 @@ _PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS: Final = frozenset({LlmProviders.AZURE, _PROVIDERS_REPLAYING_ONLY_THEIR_OWN_REASONING: Final = _PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS _ROLES_READING_PROMPT_CACHE_BREAKPOINTS: Final = frozenset({"system", "developer", "user"}) _PART_TYPES_READING_PROMPT_CACHE_BREAKPOINTS: Final = frozenset({"input_text", "input_image", "input_file"}) +_INPUT_PART_LIST_KEYS: Final = frozenset({"content", "output"}) +_COMPACT_TRACKING_PARAMS: Final = frozenset({"metadata", "user"}) +_COMPACT_REJECTED_PARAMS: Final = frozenset( + { + "background", + "context_management", + "max_output_tokens", + "max_tool_calls", + "moderation", + "partial_images", + "prompt", + "safety_identifier", + "store", + "stream_options", + "temperature", + "tool_choice", + "top_logprobs", + "top_p", + "truncation", + } +) +_PROVIDERS_WITH_STRICT_COMPACT_PARAMS: Final = frozenset( + {LlmProviders.OPENAI, LlmProviders.AZURE, LlmProviders.AZURE_AI} +) _INPUT_ITEM: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) _CONTENT_PARTS: Final[TypeAdapter[Sequence[Mapping[str, object]]]] = TypeAdapter(Sequence[Mapping[str, object]]) +class _DeclaredModel(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True) + + base_model: object = None + model_info: object = None + + class _DeploymentHost(BaseModel): """`responses()` builds the params from raw kwargs, where callers may spell the host `base_url`.""" @@ -85,7 +125,7 @@ class _DropParamsSetting(BaseModel): drop_params: object = None -def _drops_params(litellm_params: GenericLiteLLMParams) -> bool: +def drops_params(litellm_params: GenericLiteLLMParams) -> bool: setting: Final = _DropParamsSetting.model_validate(litellm_params) return litellm.drop_params is True or normalize_drop_params(setting.drop_params) is True @@ -193,6 +233,63 @@ def _with_breakpoint_on_last_reading_part( return (*parts[:last], marked_last, *parts[last + 1 :]) +def _json_object_or_none(value: object) -> Mapping[str, object] | None: + return value if isinstance(value, Mapping) else None + + +def _json_array_or_empty(value: object) -> Sequence[object]: + return value if isinstance(value, (list, tuple)) else () + + +def _item_with_wire_safe_prompt_cache_breakpoints(item: object, keeps_markers: bool) -> object: + fields: Final = _json_object_or_none(item) + if fields is None or not _carries_prompt_cache_breakpoint(fields): + return item + wire_safe_fields: Final = { + key: _parts_with_wire_safe_prompt_cache_breakpoints(key, value, keeps_markers) + for key, value in fields.items() + if key != "prompt_cache_breakpoint" + } + unchanged: Final = "prompt_cache_breakpoint" not in fields and _same_objects( + wire_safe_fields.values(), fields.values() + ) + return item if unchanged else wire_safe_fields + + +def _same_objects(rebuilt: Iterable[object], originals: Iterable[object]) -> bool: + return all(new is old for new, old in zip(rebuilt, originals, strict=True)) + + +def _carries_prompt_cache_breakpoint(fields: Mapping[str, object]) -> bool: + return "prompt_cache_breakpoint" in fields or any( + _parts_carry_prompt_cache_breakpoint(fields.get(key)) for key in _INPUT_PART_LIST_KEYS + ) + + +def _parts_carry_prompt_cache_breakpoint(parts: object) -> bool: + return any(isinstance(part, Mapping) and "prompt_cache_breakpoint" in part for part in _json_array_or_empty(parts)) + + +def _parts_with_wire_safe_prompt_cache_breakpoints(key: str, value: object, keeps_markers: bool) -> object: + if key not in _INPUT_PART_LIST_KEYS or not _parts_carry_prompt_cache_breakpoint(value): + return value + parts: Final = _json_array_or_empty(value) + wire_safe_parts: Final = tuple(_part_with_wire_safe_prompt_cache_breakpoint(part, keeps_markers) for part in parts) + return value if _same_objects(wire_safe_parts, parts) else list(wire_safe_parts) + + +def _part_with_wire_safe_prompt_cache_breakpoint(part: object, keeps_markers: bool) -> object: + fields: Final = _json_object_or_none(part) + if fields is None or "prompt_cache_breakpoint" not in fields: + return part + marker: Final = fields["prompt_cache_breakpoint"] + wire_marker: Final = prompt_cache_breakpoint_for_wire(marker, drop_params=True) if keeps_markers else None + if keeps_markers and wire_marker == marker: + return part + unmarked: Final = {key: value for key, value in fields.items() if key != "prompt_cache_breakpoint"} + return with_prompt_cache_breakpoint(unmarked, wire_marker) + + class _ReasoningSupportEntry(LiteLLMBaseModel): litellm_provider: str | None = None supports_reasoning: bool | None = None @@ -324,12 +421,19 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): return declared return info["key"] in _bundled_openai_reasoning_models() + @staticmethod + def _is_known_non_reasoning_model(model: str) -> bool: + from litellm.utils import is_known_non_reasoning_model + + return is_known_non_reasoning_model(model.split("/")[-1], LlmProviders.OPENAI.value) + @staticmethod def _requests_reasoning_effort(reasoning: object) -> bool: - effort: Final = ( - reasoning.get("effort") if isinstance(reasoning, Mapping) else getattr(reasoning, "effort", None) - ) - return effort is not None + return OpenAIResponsesAPIConfig._requested_effort(reasoning) is not None + + @staticmethod + def _requested_effort(reasoning: object) -> object: + return reasoning.get("effort") if isinstance(reasoning, Mapping) else getattr(reasoning, "effort", None) @staticmethod def _enforce_min_max_output_tokens(max_output_tokens: "int | None") -> "int | None": @@ -371,17 +475,20 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): ) -> dict: return dict( self.map_openai_params_for_deployment( - response_api_optional_params, model, drop_params, declared_reasoning_efforts=() + response_api_optional_params, model, drop_params, declared_reasoning_efforts=None ) ) + def honors_declared_reasoning_efforts(self) -> bool: + return type(self).map_openai_params is OpenAIResponsesAPIConfig.map_openai_params + def map_openai_params_for_deployment( self, response_api_optional_params: ResponsesAPIOptionalRequestParams, model: str, drop_params: bool, *, - declared_reasoning_efforts: Collection[str], + declared_reasoning_efforts: Collection[str] | None, ) -> Mapping[str, object]: """No mapping applied since inputs are in OpenAI spec already. @@ -401,7 +508,9 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): and not self._supports_reasoning_param(model=model) ): if drop_params or litellm.drop_params: - params.pop("reasoning", None) + dropped_reasoning: Final = params.pop("reasoning", None) + if self._requested_effort(dropped_reasoning) != "none" or not self._is_known_non_reasoning_model(model): + verbose_logger.warning(DROP_UNSUPPORTED_REASONING_WARNING, dropped_reasoning, model) else: raise litellm.UnsupportedParamsError( message=( @@ -419,7 +528,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): if ( isinstance(reasoning, dict) and isinstance(requested_effort, str) - and requested_effort not in declared_reasoning_efforts + and requested_effort not in (declared_reasoning_efforts or ()) and self._effort_level_is_disabled(lookup_name, requested_effort) ): if not (drop_params or litellm.drop_params): @@ -427,6 +536,15 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): message=unsupported_reasoning_effort_message("reasoning.effort", requested_effort, model), status_code=400, ) + verbose_logger.warning( + refused_reasoning_effort_warning( + suggest_declared_levels=declared_reasoning_efforts is not None, suggest_base_model=False + ), + "reasoning.effort", + requested_effort, + model, + lookup_name, + ) remaining: Final = reasoning.copy() remaining.pop("effort", None) if remaining: @@ -518,13 +636,14 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): litellm_params: GenericLiteLLMParams, prompt_cache_options: object = None, ) -> tuple[str | ResponseInputParam, Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None]: - validated_input: Final = self._validate_input_param(self._drop_bridge_minted_reasoning_items(input)) - marked_input: Final = ( - self._input_with_prompt_cache_breakpoints(validated_input) - if self._reads_prompt_cache_breakpoints( - model=model, litellm_params=litellm_params, prompt_cache_options=prompt_cache_options - ) - else validated_input + validated_input: Final = self._validate_input_param( + self._drop_bridge_minted_reasoning_items(input), drops_set_created_by=drops_params(litellm_params) + ) + reads_breakpoints: Final = self._reads_prompt_cache_breakpoints( + model=model, litellm_params=litellm_params, prompt_cache_options=prompt_cache_options + ) + marker_safe_input: Final = self._input_with_wire_safe_prompt_cache_breakpoints( + model=model, input=validated_input, litellm_params=litellm_params, reads_breakpoints=reads_breakpoints ) maps_anthropic_tool_search: Final = self._targets_openai_api(litellm_params) and any( is_anthropic_tool_search_chat_tool(tool) for tool in tools or () @@ -534,20 +653,71 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): if maps_anthropic_tool_search else tools ) - replayable_input: Final = ( - input_without_unanswered_anthropic_tool_search_calls(marked_input) - if maps_anthropic_tool_search - else marked_input - ) stripped_input, stripped_tools = self.remove_cache_control_flag_from_input_and_tools( - model=model, input=replayable_input, tools=mapped_tools + model=model, + input=self._replayable_input(marker_safe_input, reads_breakpoints, maps_anthropic_tool_search), + tools=mapped_tools, ) + if marker_safe_input is not validated_input: + self._strip_caller_input_as_if_unmarked( + model=model, + caller_input=validated_input, + reads_breakpoints=reads_breakpoints, + maps_anthropic_tool_search=maps_anthropic_tool_search, + ) object_schema_tools: Final = self._tools_with_object_parameters(model=model, tools=stripped_tools) sanitized_tools: Final = self._sanitized_tool_schemas_for_openai( model=model, tools=object_schema_tools, litellm_params=litellm_params ) return self._drop_foreign_tool_call_item_ids(stripped_input), sanitized_tools + def _replayable_input( + self, input: str | ResponseInputParam, reads_breakpoints: bool, maps_anthropic_tool_search: bool + ) -> str | ResponseInputParam: + marked_input: Final = self._input_with_prompt_cache_breakpoints(input) if reads_breakpoints else input + if not maps_anthropic_tool_search: + return marked_input + return input_without_unanswered_anthropic_tool_search_calls(marked_input) + + def _strip_caller_input_as_if_unmarked( + self, + model: str, + caller_input: str | ResponseInputParam, + reads_breakpoints: bool, + maps_anthropic_tool_search: bool, + ) -> None: + self.remove_cache_control_flag_from_input_and_tools( + model=model, input=self._replayable_input(caller_input, reads_breakpoints, maps_anthropic_tool_search) + ) + + def _input_with_wire_safe_prompt_cache_breakpoints( + self, + model: str, + input: str | ResponseInputParam, + litellm_params: GenericLiteLLMParams, + reads_breakpoints: bool, + ) -> str | ResponseInputParam: + if not isinstance(input, list) or not drops_params(litellm_params): + return input + keeps_markers: Final = reads_breakpoints or self._keeps_prompt_cache_breakpoints( + model=model, litellm_params=litellm_params + ) + wire_safe_items: Final = tuple( + _item_with_wire_safe_prompt_cache_breakpoints(item, keeps_markers) for item in input + ) + if _same_objects(wire_safe_items, input): + return input + wire_safe_input: Final = list(wire_safe_items) + return cast("ResponseInputParam", wire_safe_input) # cast-ok: items keep their shape, minus a rejected marker + + def _keeps_prompt_cache_breakpoints(self, model: str, litellm_params: GenericLiteLLMParams) -> bool: + provider: Final = _provider_name(self.custom_llm_provider) + base_model: Final = self._declared_base_model(model=model, litellm_params=litellm_params) + known_support: Final = ( + openai_prompt_cache_breakpoint_support(name, provider) for name in dict.fromkeys((base_model, model)) + ) + return next((support for support in known_support if support is not None), None) is not False + def _targets_openai_api(self, litellm_params: GenericLiteLLMParams) -> bool: host: Final = _DeploymentHost.model_validate(litellm_params) return self.custom_llm_provider == LlmProviders.OPENAI and targets_openai_api(host.api_base or host.base_url) @@ -559,7 +729,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): if tools is None or not any(is_anthropic_tool_search_chat_tool(tool) for tool in tools): return tools callers_own_search: Final = any(is_openai_tool_search_tool(tool) for tool in tools) - if callers_own_search or not _drops_params(litellm_params) or _runs_tool_search(model, tools): + if callers_own_search or not drops_params(litellm_params) or _runs_tool_search(model, tools): return tools_with_anthropic_tool_search_mapped(tools) verbose_logger.warning( "Dropping tool search under drop_params: OpenAI model '%s' cannot run it or no tool is deferred, so every " @@ -663,14 +833,15 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): @staticmethod def _without_foreign_tool_call_item_id(item: object) -> object: - if not isinstance(item, dict): + fields: Final = _json_object_or_none(item) + if fields is None: return item - item_type: Final = item.get("type") - item_id: Final = item.get("id") + item_type: Final = fields.get("type") + item_id: Final = fields.get("id") genuine_prefix: Final = TOOL_CALL_ITEM_ID_PREFIX_BY_TYPE.get(item_type) if isinstance(item_type, str) else None if genuine_prefix is None or not isinstance(item_id, str) or item_id.startswith(genuine_prefix): return item - return {key: value for key, value in item.items() if key != "id"} + return {key: value for key, value in fields.items() if key != "id"} def _sanitized_tool_schemas_for_openai( self, @@ -714,6 +885,13 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): base_model: Final[object] = model_info.get("base_model") if isinstance(model_info, dict) else None return base_model if isinstance(base_model, str) and base_model else model + @staticmethod + def _declared_base_model(model: str, litellm_params: GenericLiteLLMParams) -> str: + declared: Final = _DeclaredModel.model_validate(litellm_params) + model_info: Final = _json_object_or_none(declared.model_info) + candidates: Final = (declared.base_model, None if model_info is None else model_info.get("base_model")) + return next((name for name in candidates if isinstance(name, str) and name), model) + @staticmethod def _sanitized_tool_entry( entry: Mapping[str, object], @@ -752,7 +930,9 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): ] return tools if all(new is old for new, old in zip(sanitized, tools, strict=True)) else sanitized - def _validate_input_param(self, input: str | ResponseInputParam) -> str | ResponseInputParam: + def _validate_input_param( + self, input: str | ResponseInputParam, *, drops_set_created_by: bool = False + ) -> str | ResponseInputParam: """ Ensure all input fields if pydantic are converted to dict @@ -764,18 +944,20 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): for item in input: # if it's pydantic, convert to dict if isinstance(item, BaseModel): - validated_input.append(item.model_dump(exclude_none=True)) + validated_input.append( + replayable_responses_item( + item.model_dump(exclude_none=True), drops_set_created_by=drops_set_created_by + ) + ) elif isinstance(item, dict): - # Handle reasoning items specifically to filter out status=None - if item.get("type") == "reasoning": - verbose_logger.debug("Handling reasoning item: %s", item) - # Type assertion since we know it's a dict at this point - dict_item = cast(dict[str, object], item) - filtered_item = self._handle_reasoning_item(dict_item) - else: - # For other dict items, just pass through - filtered_item = cast(dict[str, object], item) - validated_input.append(filtered_item) + validated_input.append( + replayable_responses_item( + self._handle_reasoning_item(cast(dict[str, object], item)) + if item.get("type") == "reasoning" + else item, + drops_set_created_by=drops_set_created_by, + ) + ) else: validated_input.append(item) return validated_input @@ -1065,7 +1247,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, - ) -> tuple[str, dict]: + ) -> tuple[str, dict[str, str]]: """ Transform the get response API request into a URL and data @@ -1074,7 +1256,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): """ encoded_response_id: Final = encode_url_path_segment(response_id, field_name="response_id") url: Final = f"{api_base}/{encoded_response_id}" - data: Final[dict] = {} + data: Final[dict[str, str]] = {} return url, data def transform_get_response_api_response( @@ -1182,6 +1364,17 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): ######################################################### ########## COMPACT RESPONSE API TRANSFORMATION ########## ######################################################### + def compact_url(self, api_base: str) -> str: + parsed_url: Final = httpx.URL(api_base) + return str(parsed_url.copy_with(path=parsed_url.path.rstrip("/") + "/compact")) + + def compact_dropped_params(self, litellm_params: GenericLiteLLMParams) -> frozenset[str]: + if self.custom_llm_provider not in _PROVIDERS_WITH_STRICT_COMPACT_PARAMS: + return frozenset() + if drops_params(litellm_params): + return _COMPACT_TRACKING_PARAMS | _COMPACT_REJECTED_PARAMS + return _COMPACT_TRACKING_PARAMS + def transform_compact_response_api_request( self, model: str, @@ -1197,10 +1390,8 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): OpenAI API expects the following request - POST /v1/responses/compact """ - # Preserve query params (e.g., api-version) while appending /compact. - parsed_url: Final = httpx.URL(api_base) - compact_path: Final = parsed_url.path.rstrip("/") + "/compact" - url: Final = str(parsed_url.copy_with(path=compact_path)) + url: Final = self.compact_url(api_base) + dropped_params: Final = self.compact_dropped_params(litellm_params) request_params: Final = cast( # cast-ok: the base signature leaves the params bag untyped "Mapping[str, object]", response_api_optional_request_params @@ -1214,14 +1405,11 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): ) if sanitized_tools is not None: response_api_optional_request_params["tools"] = sanitized_tools - data: Final = cast( # cast-ok: typed dict spread is inferred as Unknown by basedpyright - dict[str, object], - { - "model": model, - "input": replay_safe_input, - **response_api_optional_request_params, - }, - ) + data: Final = { + "model": model, + "input": replay_safe_input, + **{key: value for key, value in request_params.items() if key not in dropped_params}, + } return url, data diff --git a/litellm/llms/openai_like/dynamic_config.py b/litellm/llms/openai_like/dynamic_config.py index 685561f55a5..d7aafe9f06e 100644 --- a/litellm/llms/openai_like/dynamic_config.py +++ b/litellm/llms/openai_like/dynamic_config.py @@ -55,6 +55,15 @@ def create_config_class(provider: SimpleProviderConfig): else: return super()._transform_messages(messages=messages, model=model, is_async=False) + def _should_preserve_cache_control_for_endpoint( + self, + custom_llm_provider: str | None, + api_base: str | None, + ) -> bool: + return provider.special_handling.get( + "preserve_cache_control", False + ) or super()._should_preserve_cache_control_for_endpoint(custom_llm_provider, api_base) + def _get_openai_compatible_provider_info( self, api_base: str | None, api_key: str | None ) -> tuple[str | None, str | None]: @@ -177,7 +186,7 @@ def create_config_class(provider: SimpleProviderConfig): return JSONProviderConfig -_responses_config_cache: Final[dict] = {} +_responses_config_cache: Final[dict[str, "type[OpenAILikeResponsesConfig]"]] = {} def create_responses_config_class(provider: SimpleProviderConfig) -> "type[OpenAILikeResponsesConfig]": diff --git a/litellm/llms/openai_like/json_loader.py b/litellm/llms/openai_like/json_loader.py index 23b63177137..6e99b9e7fec 100644 --- a/litellm/llms/openai_like/json_loader.py +++ b/litellm/llms/openai_like/json_loader.py @@ -3,6 +3,7 @@ JSON-based provider configuration loader for OpenAI-compatible providers. """ import json +from collections.abc import Mapping from pathlib import Path from typing import Final @@ -12,6 +13,7 @@ from litellm._logging import verbose_logger from litellm.types.utils import ProviderSpecificModelInfo _PROVIDER_INFO_ADAPTER: Final = TypeAdapter(ProviderSpecificModelInfo) +_SPECIAL_HANDLING_ADAPTER: Final = TypeAdapter(Mapping[str, bool]) class SimpleProviderConfig: @@ -20,12 +22,14 @@ class SimpleProviderConfig: def __init__(self, slug: str, data: dict): self.slug = slug self.base_url = data["base_url"] - self.api_key_env = data["api_key_env"] - self.api_base_env = data.get("api_base_env") + self.api_key_env: str = data["api_key_env"] + self.api_base_env: str | None = data.get("api_base_env") self.base_class = data.get("base_class", "openai_gpt") self.param_mappings = data.get("param_mappings", {}) self.constraints = data.get("constraints", {}) - self.special_handling = data.get("special_handling", {}) + self.special_handling: Mapping[str, bool] = _SPECIAL_HANDLING_ADAPTER.validate_python( + data.get("special_handling", {}) + ) self.supported_endpoints = data.get("supported_endpoints", []) self.provider_info: ProviderSpecificModelInfo = ( _PROVIDER_INFO_ADAPTER.validate_python(data["provider_info"]) if "provider_info" in data else {} diff --git a/litellm/llms/openai_like/messages/transformation.py b/litellm/llms/openai_like/messages/transformation.py index 8daae2a8e6a..a5d825a39d7 100644 --- a/litellm/llms/openai_like/messages/transformation.py +++ b/litellm/llms/openai_like/messages/transformation.py @@ -37,6 +37,9 @@ class OpenAILikeAnthropicMessagesConfig(AnthropicMessagesConfig): def is_native_messages_route(self) -> bool: return False + def hoists_wrapped_defer_loading(self, model: str) -> bool: + return False + def validate_anthropic_messages_environment( self, headers: dict[str, str], diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index 2b71707239b..ad47b334ed4 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -269,5 +269,28 @@ "/v1/chat/completions", "/v1/responses" ] + }, + "kyma": { + "base_url": "https://kymaapi.com/v1", + "api_key_env": "KYMA_API_KEY", + "provider_info": { + "supports_function_calling": true + } + }, + "streamlake": { + "base_url": "https://vanchin.streamlake.ai/api/gateway/v1/endpoints", + "api_key_env": "STREAMLAKE_API_KEY", + "api_base_env": "STREAMLAKE_API_BASE", + "param_mappings": { + "max_tokens": "max_completion_tokens" + }, + "provider_info": { + "supports_function_calling": true, + "supports_reasoning": true + }, + "special_handling": { + "preserve_cache_control": true + }, + "supported_endpoints": ["/v1/chat/completions"] } } diff --git a/litellm/llms/perplexity/responses/transformation.py b/litellm/llms/perplexity/responses/transformation.py index 77817a5e5bb..58f71028451 100644 --- a/litellm/llms/perplexity/responses/transformation.py +++ b/litellm/llms/perplexity/responses/transformation.py @@ -17,7 +17,7 @@ from pydantic import TypeAdapter from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException -from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig, drops_params from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ResponseInputParam, ResponsesAPIResponse from litellm.types.router import GenericLiteLLMParams @@ -84,7 +84,7 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig): """Handle preset/ model prefix: send as {"preset": name} instead of {"model": name}.""" input = self._ensure_message_type(input) if model.startswith("preset/"): - input = self._validate_input_param(input) + input = self._validate_input_param(input, drops_set_created_by=drops_params(litellm_params)) data: Final[dict] = { "preset": model[len("preset/") :], "input": input, diff --git a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py index 5bff43828d6..884c82fe331 100644 --- a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py +++ b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py @@ -1,4 +1,5 @@ from collections.abc import Sequence +from dataclasses import dataclass from typing import Final, Literal import httpx @@ -8,7 +9,6 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger from litellm.caching.caching import Cache, LiteLLMCacheType -from litellm.constants import MINIMUM_PROMPT_CACHE_TOKEN_COUNT from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, @@ -16,10 +16,11 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, ) from litellm.llms.openai.openai import AllMessageValues -from litellm.utils import is_prompt_caching_valid_prompt +from litellm.types.llms.openai import ChatCompletionToolParam +from litellm.utils import get_prompt_cache_min_tokens, is_prompt_caching_valid_prompt from ..common_utils import VertexAIError, get_vertex_base_url -from ..vertex_llm_base import VertexBase +from ..vertex_llm_base import VertexBase, publisher_model_location from .known_cached_contents import KnownCachedContents, known_cached_contents from .transformation import ( cached_messages_end_on_supported_turn, @@ -65,6 +66,19 @@ class _CreatedCachedContentRoot(RootModel[_CreatedCachedContent]): model_config = ConfigDict(frozen=True, hide_input_in_errors=True) +_TOOL_PARAMS: Final = ("tools", "tool_choice") + + +def _sync_cache_client(timeout: float | httpx.Timeout | None) -> HTTPHandler: + if timeout is None: + return HTTPHandler() + return HTTPHandler(timeout=httpx.Timeout(timeout) if isinstance(timeout, (float, int)) else timeout) + + +class _CachedContentsNotServedError(VertexAIError): + pass + + def _record_cache_creation_tokens(logging_obj: Logging, created: _CreatedCachedContent) -> None: tokens: Final = (created.get("usageMetadata") or {}).get("totalTokenCount") or 0 logging_obj.vertex_ai_cache_creation_input_tokens = max(tokens, 0) # rebind-ok: set provider count @@ -78,6 +92,61 @@ def _wait_seconds(timeout: float | httpx.Timeout | None) -> float | None: return timeout +@dataclass(frozen=True, slots=True) +class _CacheTarget: + model: str + project: str | None + location: str | None + + +def _cache_target( + custom_llm_provider: Literal["vertex_ai", "vertex_ai_beta", "gemini"], + api_base: str | None, + model: str, + vertex_project: str | None, + vertex_location: str | None, +) -> _CacheTarget: + model_location: Final = ( + None if custom_llm_provider == "gemini" or api_base is None else publisher_model_location(api_base) + ) + if model_location is None: + return _CacheTarget(model=model, project=vertex_project, location=vertex_location) + return _CacheTarget(model=model_location.model, project=model_location.project, location=model_location.location) + + +def _prompt_cache_min_tokens(models: Sequence[str], custom_llm_provider: str) -> int: + return max( + get_prompt_cache_min_tokens( + model if model.startswith(custom_llm_provider) else f"{custom_llm_provider}/{model}" + ) + for model in models + ) + + +def _context_caching_skip_reason( + cached_messages: Sequence[AllMessageValues], + tools: Sequence[ChatCompletionToolParam] | None, + deployment_model: str, + cache_model: str, + custom_llm_provider: Literal["vertex_ai", "vertex_ai_beta", "gemini"], +) -> str | None: + if not cached_messages_end_on_supported_turn(cached_messages): + return ( + "Vertex AI context caching: cached message block ends on a model turn once " + "system messages are extracted, which the cachedContents API rejects." + ) + min_tokens: Final = _prompt_cache_min_tokens((deployment_model, cache_model), custom_llm_provider) + if not is_prompt_caching_valid_prompt( + model=cache_model, + messages=list(cached_messages), + tools=None if tools is None else list(tools), + custom_llm_provider=custom_llm_provider, + min_token_count=min_tokens, + ): + return f"Vertex AI context caching: cached content is below minimum token count ({min_tokens})." + return None + + class ContextCachingEndpoints(VertexBase): """ Covers context caching endpoints for Vertex AI + Google AI Studio @@ -135,6 +204,7 @@ class ContextCachingEndpoints(VertexBase): vertex_project=vertex_project, vertex_location=vertex_location, vertex_api_version=("v1beta1" if custom_llm_provider == "vertex_ai_beta" else "v1"), + collection_endpoint=True, ) def check_cache( @@ -206,6 +276,8 @@ class ContextCachingEndpoints(VertexBase): except httpx.HTTPStatusError as e: if e.response.status_code == 403: return None + if e.response.status_code == 404 and custom_llm_provider == "gemini": + raise _CachedContentsNotServedError(status_code=404, message=e.response.text) raise VertexAIError(status_code=e.response.status_code, message=e.response.text) except Exception as e: raise VertexAIError(status_code=500, message=str(e)) @@ -305,6 +377,8 @@ class ContextCachingEndpoints(VertexBase): except httpx.HTTPStatusError as e: if e.response.status_code == 403: return None + if e.response.status_code == 404 and custom_llm_provider == "gemini": + raise _CachedContentsNotServedError(status_code=404, message=e.response.text) raise VertexAIError(status_code=e.response.status_code, message=e.response.text) except Exception as e: raise VertexAIError(status_code=500, message=str(e)) @@ -370,41 +444,25 @@ class ContextCachingEndpoints(VertexBase): if len(cached_messages) == 0: return messages, optional_params, None - if not cached_messages_end_on_supported_turn(cached_messages): - verbose_logger.debug( - "Vertex AI context caching: cached message block ends on a model turn once " - "system messages are extracted, which the cachedContents API rejects. " - "Skipping context caching." - ) + target: Final = _cache_target(custom_llm_provider, api_base, model, vertex_project, vertex_location) + tools: Final = optional_params.get("tools") + tool_choice: Final = optional_params.get("tool_choice") + skip_reason: Final = _context_caching_skip_reason( + cached_messages, tools, model, target.model, custom_llm_provider + ) + if skip_reason is not None: + verbose_logger.debug("%s Skipping context caching.", skip_reason) return messages, optional_params, None - # Gemini requires a minimum of 1024 tokens for context caching. - # Skip caching if the cached content is too small to avoid API errors. - if not is_prompt_caching_valid_prompt( - model=model, - messages=cached_messages, - tools=optional_params.get("tools"), - custom_llm_provider=custom_llm_provider, - ): - verbose_logger.debug( - "Vertex AI context caching: cached content is below minimum token " - "count (%d). Skipping context caching.", - MINIMUM_PROMPT_CACHE_TOKEN_COUNT, - ) - return messages, optional_params, None - - tools: Final = optional_params.pop("tools", None) - tool_choice: Final = optional_params.pop("tool_choice", None) - ## AUTHORIZATION ## token, url = self._get_token_and_url_context_caching( gemini_api_key=api_key, custom_llm_provider=custom_llm_provider, api_base=api_base, - vertex_project=vertex_project, - vertex_location=vertex_location, + vertex_project=target.project, + vertex_location=target.location, vertex_auth_header=vertex_auth_header, - model=model, + model=target.model, ) headers: Final = { @@ -417,44 +475,42 @@ class ContextCachingEndpoints(VertexBase): if extra_headers is not None: headers.update(extra_headers) - if client is None or not isinstance(client, HTTPHandler): - _params: Final = {} - if timeout is not None: - if isinstance(timeout, float) or isinstance(timeout, int): - timeout = httpx.Timeout(timeout) - _params["timeout"] = timeout - client = HTTPHandler(**_params) - else: - client = client + http_client: Final = client if isinstance(client, HTTPHandler) else _sync_cache_client(timeout) ## CHECK IF CACHED ALREADY generated_cache_key: Final = local_cache_obj.get_cache_key( - messages=cached_messages, tools=tools, tool_choice=tool_choice, model=model - ) - google_cache_name: Final = self.check_cache( - cache_key=generated_cache_key, - client=client, - headers=headers, - api_key=api_key, - api_base=api_base, - logging_obj=logging_obj, - custom_llm_provider=custom_llm_provider, - vertex_project=vertex_project, - vertex_location=vertex_location, - vertex_auth_header=vertex_auth_header, - model=model, + messages=cached_messages, tools=tools, tool_choice=tool_choice, model=target.model ) + try: + google_cache_name: Final = self.check_cache( + cache_key=generated_cache_key, + client=http_client, + headers=headers, + api_key=api_key, + api_base=api_base, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + vertex_project=target.project, + vertex_location=target.location, + vertex_auth_header=vertex_auth_header, + model=target.model, + ) + except _CachedContentsNotServedError: + verbose_logger.debug("Gemini context caching: %s answers 404. Skipping context caching.", url) + return messages, optional_params, None + for tool_param in _TOOL_PARAMS: + optional_params.pop(tool_param, None) if google_cache_name: return non_cached_messages, optional_params, google_cache_name ## TRANSFORM REQUEST cached_content_request_body: Final = transform_openai_messages_to_gemini_context_caching( - model=model, + model=target.model, messages=cached_messages, cache_key=generated_cache_key, custom_llm_provider=custom_llm_provider, - vertex_project=vertex_project, - vertex_location=vertex_location, + vertex_project=target.project, + vertex_location=target.location, ) cached_content_request_body["tools"] = tools @@ -473,7 +529,7 @@ class ContextCachingEndpoints(VertexBase): ) try: - response: Final = client.post( + response: Final = http_client.post( url=url, headers=headers, json=cached_content_request_body, @@ -489,7 +545,7 @@ class ContextCachingEndpoints(VertexBase): _record_cache_creation_tokens(logging_obj, raw_response_cached) self._known_caches.remember( KnownCachedContents.scope( - url, custom_llm_provider, headers, vertex_project=vertex_project, vertex_location=vertex_location + url, custom_llm_provider, headers, vertex_project=target.project, vertex_location=target.location ), generated_cache_key, raw_response_cached.get("name"), @@ -536,41 +592,25 @@ class ContextCachingEndpoints(VertexBase): if len(cached_messages) == 0: return messages, optional_params, None - if not cached_messages_end_on_supported_turn(cached_messages): - verbose_logger.debug( - "Vertex AI context caching: cached message block ends on a model turn once " - "system messages are extracted, which the cachedContents API rejects. " - "Skipping context caching." - ) + target: Final = _cache_target(custom_llm_provider, api_base, model, vertex_project, vertex_location) + tools: Final = optional_params.get("tools") + tool_choice: Final = optional_params.get("tool_choice") + skip_reason: Final = _context_caching_skip_reason( + cached_messages, tools, model, target.model, custom_llm_provider + ) + if skip_reason is not None: + verbose_logger.debug("%s Skipping context caching.", skip_reason) return messages, optional_params, None - # Gemini requires a minimum of 1024 tokens for context caching. - # Skip caching if the cached content is too small to avoid API errors. - if not is_prompt_caching_valid_prompt( - model=model, - messages=cached_messages, - tools=optional_params.get("tools"), - custom_llm_provider=custom_llm_provider, - ): - verbose_logger.debug( - "Vertex AI context caching: cached content is below minimum token " - "count (%d). Skipping context caching.", - MINIMUM_PROMPT_CACHE_TOKEN_COUNT, - ) - return messages, optional_params, None - - tools: Final = optional_params.pop("tools", None) - tool_choice: Final = optional_params.pop("tool_choice", None) - ## AUTHORIZATION ## token, url = self._get_token_and_url_context_caching( gemini_api_key=api_key, custom_llm_provider=custom_llm_provider, api_base=api_base, - vertex_project=vertex_project, - vertex_location=vertex_location, + vertex_project=target.project, + vertex_location=target.location, vertex_auth_header=vertex_auth_header, - model=model, + model=target.model, ) headers: Final = { @@ -590,10 +630,10 @@ class ContextCachingEndpoints(VertexBase): ## CHECK IF CACHED ALREADY generated_cache_key: Final = local_cache_obj.get_cache_key( - messages=cached_messages, tools=tools, tool_choice=tool_choice, model=model + messages=cached_messages, tools=tools, tool_choice=tool_choice, model=target.model ) scope: Final = KnownCachedContents.scope( - url, custom_llm_provider, headers, vertex_project=vertex_project, vertex_location=vertex_location + url, custom_llm_provider, headers, vertex_project=target.project, vertex_location=target.location ) async def lookup_or_create() -> str | None: @@ -605,10 +645,10 @@ class ContextCachingEndpoints(VertexBase): api_base=api_base, logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, - vertex_project=vertex_project, - vertex_location=vertex_location, + vertex_project=target.project, + vertex_location=target.location, vertex_auth_header=vertex_auth_header, - model=model, + model=target.model, ) if google_cache_name: @@ -616,12 +656,12 @@ class ContextCachingEndpoints(VertexBase): ## TRANSFORM REQUEST cached_content_request_body: Final = transform_openai_messages_to_gemini_context_caching( - model=model, + model=target.model, messages=cached_messages, cache_key=generated_cache_key, custom_llm_provider=custom_llm_provider, - vertex_project=vertex_project, - vertex_location=vertex_location, + vertex_project=target.project, + vertex_location=target.location, ) cached_content_request_body["tools"] = tools @@ -659,9 +699,15 @@ class ContextCachingEndpoints(VertexBase): ) return raw_response_cached.get("name") - cache_name: Final = await self._known_caches.single_flight( - scope, generated_cache_key, lookup_or_create, _wait_seconds(timeout) - ) + try: + cache_name: Final = await self._known_caches.single_flight( + scope, generated_cache_key, lookup_or_create, _wait_seconds(timeout) + ) + except _CachedContentsNotServedError: + verbose_logger.debug("Gemini context caching: %s answers 404. Skipping context caching.", url) + return messages, optional_params, None + for tool_param in _TOOL_PARAMS: + optional_params.pop(tool_param, None) return non_cached_messages, optional_params, cache_name def get_cache(self): diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py index 6d8eb228786..1f0ee728cd1 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py @@ -1,6 +1,8 @@ from collections.abc import Mapping, Sequence from typing import Final +from litellm.exceptions import UnsupportedParamsError +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.llms.anthropic.common_utils import AnthropicModelInfo from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, @@ -17,6 +19,41 @@ from litellm.types.router import GenericLiteLLMParams from ....vertex_llm_base import VertexBase from ..output_params_utils import sanitize_vertex_anthropic_output_params +_VERSION_HEADER_VERTEX_READS_FROM_BODY: Final = "anthropic-version" +_VERTEX_DEFAULT_VERSION: Final = "vertex-2023-10-16" +_VERSIONS_SERVED_AS_VERTEX_DEFAULT: Final = frozenset({"", "2023-01-01", "2023-06-01"}) +_VERSIONS_VERTEX_ACCEPTS_IN_BODY: Final = frozenset({_VERTEX_DEFAULT_VERSION, "2026-10-21"}) + + +def _requested_version(headers: Mapping[str, object]) -> str: + requested: Final = next( + (value for name, value in headers.items() if name.lower() == _VERSION_HEADER_VERTEX_READS_FROM_BODY), + "", + ) + return requested.strip() if isinstance(requested, str) else "" + + +def _vertex_version(headers: Mapping[str, object], model: str, drop_params: bool) -> str | UnsupportedParamsError: + requested: Final = _requested_version(headers) + if requested in _VERSIONS_SERVED_AS_VERTEX_DEFAULT: + return _VERTEX_DEFAULT_VERSION + if requested in _VERSIONS_VERTEX_ACCEPTS_IN_BODY: + return requested + if drop_params: + return _VERTEX_DEFAULT_VERSION + return UnsupportedParamsError( + message=( + f"{model} does not support {_VERSION_HEADER_VERTEX_READS_FROM_BODY}={requested}. " + "To drop unsupported params, set `litellm.drop_params = True`." + ), + llm_provider="vertex_ai", + model=model, + ) + + +def _without_version_header(headers: Mapping[str, str]) -> dict[str, str]: # mutable-ok: the caller sends these headers + return {name: value for name, value in headers.items() if name.lower() != _VERSION_HEADER_VERTEX_READS_FROM_BODY} + class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, VertexBase): @property @@ -198,7 +235,7 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert messages: list[dict], anthropic_messages_optional_request_params: dict, litellm_params: GenericLiteLLMParams, - headers: dict, + headers: dict[str, object], # mutable-ok: mirrors the provider transform contract ) -> dict: anthropic_messages_request: Final = super().transform_anthropic_messages_request( model=model, @@ -212,10 +249,28 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert self._remove_scope_from_cache_control(anthropic_messages_request) - anthropic_messages_request["anthropic_version"] = "vertex-2023-10-16" + version: Final = _vertex_version( + headers, model=model, drop_params=drops_unsupported_params(litellm_params.drop_params) + ) + if isinstance(version, UnsupportedParamsError): + raise version + anthropic_messages_request["anthropic_version"] = version anthropic_messages_request.pop("model", None) # do not pass model in request body to vertex ai sanitize_vertex_anthropic_output_params(anthropic_messages_request, model) return anthropic_messages_request + + def sign_request( + self, + headers: Mapping[str, str], + optional_params: Mapping[str, object], + request_data: Mapping[str, object], + api_base: str, + api_key: str | None = None, + model: str | None = None, + stream: bool | None = None, + fake_stream: bool | None = None, + ) -> tuple[dict[str, str], bytes | None]: # mutable-ok: the caller sends these headers + return _without_version_header(headers), None diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py index 4be41e26326..180761f15d9 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py @@ -6,9 +6,12 @@ Unlike Gemini models which use Google's token counting API, partner models use their respective publisher-specific count-tokens endpoints. """ +from collections.abc import Mapping from typing import Any, Final +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.anthropic.common_utils import OBJECT_LIST_ADAPTER, is_claude_model_target, tools_without_custom_field from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.llms.vertex_ai.common_utils import get_vertex_base_url from litellm.llms.vertex_ai.vertex_llm_base import VertexBase @@ -20,6 +23,22 @@ _COUNT_TOKENS_PUBLISHERS: Final = ( ) +def count_tokens_request_body( + request_data: Mapping[str, object], model: str, drop_params: object +) -> Mapping[str, object]: + tools: Final = request_data.get("tools") + if not isinstance(tools, list): + return request_data + return { + **request_data, + "tools": tools_without_custom_field( + OBJECT_LIST_ADAPTER.validate_python(tools), + hoist_defer_loading=is_claude_model_target("vertex_ai", model), + drop_custom=drops_unsupported_params(drop_params), + ), + } + + def count_tokens_publisher(model: str) -> str | None: return next((publisher for marker, publisher in _COUNT_TOKENS_PUBLISHERS if marker in model), None) @@ -169,7 +188,7 @@ class VertexAIPartnerModelsTokenCounter(VertexBase): response: Final = await async_client.post( endpoint_url, headers=headers, - json=request_data, + json=dict(count_tokens_request_body(request_data, model, litellm_params.get("drop_params"))), timeout=30.0, ) diff --git a/litellm/llms/vertex_ai/vertex_llm_base.py b/litellm/llms/vertex_ai/vertex_llm_base.py index 0cdd92bb071..7dd98212f76 100644 --- a/litellm/llms/vertex_ai/vertex_llm_base.py +++ b/litellm/llms/vertex_ai/vertex_llm_base.py @@ -6,9 +6,11 @@ Handles Authentication and generating request urls for Vertex AI and Google AI S import asyncio import json +import re import threading from collections import OrderedDict from collections.abc import Mapping +from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Final, Literal, Protocol from urllib.parse import urlparse @@ -46,6 +48,31 @@ def graft_default_vertex_path(api_base: str, default_url: str) -> str: _graft_default_vertex_path = graft_default_vertex_path + +_PUBLISHER_MODEL_LOCATION: Final = re.compile(r".*/projects/[^/]+/locations/[^/]+(?=/publishers/[^/]+/models/[^/]+/?$)") + + +@dataclass(frozen=True, slots=True) +class PublisherModelLocation: + parent_url: str + project: str + location: str + model: str + + +def publisher_model_location(api_base: str) -> PublisherModelLocation | None: + match: Final = _PUBLISHER_MODEL_LOCATION.match(api_base) + if match is None: + return None + _, _, project, _, location = match.group().rsplit("/", 4) + return PublisherModelLocation( + parent_url=match.group(), + project=project, + location=location, + model=api_base.rstrip("/").rsplit("/", 1)[-1], + ) + + GOOGLE_IMPORT_ERROR_MESSAGE: Final = ( "Google Cloud SDK not found. Install it with: pip install 'litellm[google]' or pip install google-cloud-aiplatform" ) @@ -637,15 +664,18 @@ class VertexBase: vertex_location: str | None = None, vertex_api_version: Literal["v1", "v1beta1"] | None = None, use_psc_endpoint_format: bool = False, + collection_endpoint: bool = False, ) -> tuple[str | None, str]: """ for cloudflare ai gateway - https://github.com/BerriAI/litellm/issues/4317 Handles custom api_base for: - 1. Gemini (Google AI Studio) - constructs /models/{model}:{endpoint} + 1. Gemini (Google AI Studio) - constructs /models/{model}:{endpoint}, or {api_base}/{endpoint} + for a collection endpoint 2. Vertex AI with standard proxies - grafts the default vertex URL path onto the api_base when its path is empty or only an API version (/v1, /v1beta1); - otherwise constructs {api_base}:{endpoint} + otherwise constructs {api_base}:{endpoint}, or, for a collection endpoint under an api_base + naming a publisher model, the collection under that model's projects/{project}/locations/{location} 3. Vertex AI with PSC endpoints - constructs full path structure {api_base}/v1/projects/{project}/locations/{location}/endpoints/{model}:{endpoint} (only when use_psc_endpoint_format=True) @@ -653,22 +683,22 @@ class VertexBase: Args: use_psc_endpoint_format: If True, constructs PSC endpoint URL format. If False (default), uses api_base as-is and appends :{endpoint} + collection_endpoint: True when `endpoint` names a collection resource such as + cachedContents rather than a model action such as generateContent ## Returns - (auth_header, url) - Tuple[Optional[str], str] """ if api_base: if custom_llm_provider == "gemini": - # For Gemini (Google AI Studio), construct the full path like other providers - if model is None: + if model is None and not collection_endpoint: raise ValueError("Model parameter is required for Gemini custom API base URLs") - url = join_api_base(api_base, f"models/{model}:{endpoint}") + url = join_api_base(api_base, endpoint if collection_endpoint else f"models/{model}:{endpoint}") if gemini_api_key is None: raise ValueError( "Missing Gemini API key. Set the GEMINI_API_KEY or GOOGLE_API_KEY environment variable." ) - if gemini_api_key is not None: - auth_header = {"x-goog-api-key": gemini_api_key} + auth_header = {"x-goog-api-key": gemini_api_key} else: # For Vertex AI if use_psc_endpoint_format: @@ -692,7 +722,12 @@ class VertexBase: elif urlparse(api_base).path.rstrip("/") in ("/v1", "/v1beta1") and "/projects/" in urlparse(url).path: url = graft_default_vertex_path(api_base=api_base, default_url=url) else: - url = f"{api_base.rstrip('/')}:{endpoint}" + model_location: Final = publisher_model_location(api_base) if collection_endpoint else None + url = ( + f"{api_base.rstrip('/')}:{endpoint}" + if model_location is None + else join_api_base(model_location.parent_url, endpoint) + ) if stream is True: parsed_stream_url: Final = urlparse(url) stream_query: Final = f"{parsed_stream_url.query}&alt=sse" if parsed_stream_url.query else "alt=sse" diff --git a/litellm/llms/volcengine/responses/transformation.py b/litellm/llms/volcengine/responses/transformation.py index 4076ed06fc2..7f2425d0909 100644 --- a/litellm/llms/volcengine/responses/transformation.py +++ b/litellm/llms/volcengine/responses/transformation.py @@ -313,10 +313,10 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig): api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, - ) -> tuple[str, dict]: + ) -> tuple[str, dict[str, str]]: encoded_response_id: Final = encode_url_path_segment(response_id, field_name="response_id") url: Final = f"{api_base}/{encoded_response_id}" - data: Final[dict] = {} + data: Final[dict[str, str]] = {} return url, data def transform_get_response_api_response( diff --git a/litellm/main.py b/litellm/main.py index 5aae80549e1..d1af2de3a13 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -113,7 +113,6 @@ from litellm.litellm_core_utils.request_timeout_resolver import ( get_configured_request_timeout, ) from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer -from litellm.litellm_core_utils.tool_fields import tools_without_hosted_mcp from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm import BaseConfig, BaseImageGenerationConfig from litellm.llms.base_llm.base_model_iterator import ( @@ -156,6 +155,7 @@ from litellm.types.litellm_params import ControlOptions, RetryStrategy from litellm.types.llms.openai import AllMessageValues from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( + AgenticLoopParams, BaseUrlRoutingHint, CustomPricingLiteLLMParams, ModelResponseStream, @@ -235,6 +235,7 @@ from .litellm_core_utils.streaming_chunk_builder_utils import ( ChunkProcessor as ChunkProcessor, # noqa: PLC0414 # public re-export ) from .llms.anthropic.chat import AnthropicChatCompletion +from .llms.anthropic.common_utils import is_claude_tool_target from .llms.azure.audio_transcriptions import AzureAudioTranscription from .llms.azure.azure import AzureChatCompletion, check_dynamic_azure_params from .llms.azure.chat.o_series_handler import AzureOpenAIO1ChatCompletion @@ -1136,17 +1137,6 @@ from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the o _ANTHROPIC_ONLY_TOOL_KEYS: Final = frozenset({"input_examples", "eager_input_streaming"}) -def _is_claude_tool_target(custom_llm_provider: str | None, model: str) -> bool: - if custom_llm_provider == "anthropic": - return True - model_lower: Final = model.lower() - if custom_llm_provider == "bedrock": - return "claude" in model_lower or ("arn:" in model_lower and ":bedrock:" in model_lower) - if custom_llm_provider == "azure_ai" or custom_llm_provider == "vertex_ai": - return "claude" in model_lower - return False - - def _history_without_unanswered_tool_search_calls( messages: Sequence[object], custom_llm_provider: str | None, provider_config: BaseConfig ) -> tuple[object, ...] | None: @@ -1283,6 +1273,16 @@ def _get_router_deployment_id(kwargs: dict) -> str | None: return None +def _record_agentic_loop_params( + logging_obj: object, requested_model: str, custom_llm_provider: str, owns_agentic_loop: bool +) -> None: + if not owns_agentic_loop or not isinstance(logging_obj, LiteLLMLoggingObj): + return + logging_obj.record_agentic_loop_params( + AgenticLoopParams(model=requested_model, custom_llm_provider=custom_llm_provider) + ) + + def _register_custom_pricing_for_request( model: str, custom_llm_provider: str, @@ -2673,10 +2673,13 @@ def _complete_custom_openai( # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there api_base = _resolve_openai_api_base(api_base) + if custom_llm_provider == "bigmodel" and not api_key: + raise litellm.AuthenticationError( + message="Set BIGMODEL_API_KEY or pass api_key", llm_provider="bigmodel", model=model + ) organization = ( organization - or litellm.organization - or get_secret("OPENAI_ORGANIZATION") + or (None if custom_llm_provider == "bigmodel" else litellm.organization or get_secret("OPENAI_ORGANIZATION")) or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 ) openai.organization = organization @@ -5699,6 +5702,7 @@ def completion( **{k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS | {"provider_specific_header"} if k in kwargs}, "extra_headers": extra_headers, } + requested_model: Final = model model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( model=model, custom_llm_provider=custom_llm_provider, @@ -5710,16 +5714,23 @@ def completion( else None ), ) + _record_agentic_loop_params( + logging_obj=logging, + requested_model=requested_model, + custom_llm_provider=custom_llm_provider, + owns_agentic_loop=not skip_responses_api_bridge, + ) + deployment_model_info: Final = str_keyed_mapping( + cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg + ) ## RESPONSES API BRIDGE LOGIC ## - check early and normalize model name responses_api_model_info, model = responses_api_bridge_check( model=model, custom_llm_provider=custom_llm_provider, web_search_options=web_search_options, api_base=api_base, - deployment_model_info=str_keyed_mapping( - cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg - ), + deployment_model_info=deployment_model_info, ) routing_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict drop_unsupported_reasoning: Final = ( @@ -5750,7 +5761,7 @@ def completion( } ) - if not _is_claude_tool_target(custom_llm_provider=custom_llm_provider, model=model): + if not is_claude_tool_target(custom_llm_provider=custom_llm_provider, model=model): tools = _drop_anthropic_only_tool_keys(tools=tools) if provider_specific_header is not None: @@ -5816,7 +5827,15 @@ def completion( ) if provider_config is not None: - translated_messages: Final = provider_config.translate_developer_role_to_system_role(messages=messages) + translated_messages: Final = list( + provider_config.translate_developer_role_to_system_role( + messages=messages, + model=model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + model_info=deployment_model_info, + ) + ) history_for_route: Final = _history_without_unanswered_tool_search_calls( translated_messages, custom_llm_provider, provider_config ) @@ -5956,20 +5975,14 @@ def completion( optional_params.pop("functions_unsupported_model") ) request_kwargs: Final = _REQUEST_KWARGS_ADAPTER.validate_python(kwargs) - prompt_drop_params: Final = ( - litellm.drop_params is True or normalize_drop_params(request_kwargs.get("drop_params")) is True + messages = function_call_prompt( + messages=messages, + functions=functions_unsupported_model, + model=model, + llm_provider=custom_llm_provider, + drop_params=litellm.drop_params is True + or normalize_drop_params(request_kwargs.get("drop_params")) is True, ) - prompt_functions: Final = tools_without_hosted_mcp( - functions_unsupported_model, model=model, route=custom_llm_provider, drop_params=prompt_drop_params - ) - if prompt_functions: - messages = function_call_prompt( - messages=messages, - functions=list(prompt_functions), - model=model, - llm_provider=custom_llm_provider, - drop_params=prompt_drop_params, - ) # For logging - save the values of the litellm-specific params passed in requested_litellm_params: Final = get_litellm_params( @@ -6306,6 +6319,7 @@ def completion( or custom_llm_provider == "nebius" or custom_llm_provider == "wandb" or custom_llm_provider == "clarifai" + or custom_llm_provider == "bigmodel" or custom_llm_provider in litellm.openai_compatible_providers or JSONProviderRegistry.exists(custom_llm_provider) # JSON-configured providers or "ft:gpt-3.5-turbo" in model # finetune gpt-3.5-turbo diff --git a/litellm/messages/dispatch.py b/litellm/messages/dispatch.py index 0e2fc2e3dcb..8e6becc60b4 100644 --- a/litellm/messages/dispatch.py +++ b/litellm/messages/dispatch.py @@ -111,6 +111,7 @@ _FEATURES: Final[detectors.Detectors] = MappingProxyType( Feature.CACHE_CONTROL_INJECTION: detectors.cache_control_injection, Feature.STREAM_COST_INJECTION: route_detectors.stream_cost_injection, Feature.THINKING_SIGNATURE_RETRY: route_detectors.thinking_signature_retry, + Feature.TOOL_CUSTOM_FIELD: route_detectors.tool_custom_field, } ) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 8719565a96f..4fad4857c60 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -6101,6 +6101,7 @@ "text", "audio" ], + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, @@ -6133,6 +6134,7 @@ "text", "audio" ], + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, @@ -6164,6 +6166,7 @@ "text", "audio" ], + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, @@ -6230,6 +6233,7 @@ "text", "audio" ], + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, @@ -6293,6 +6297,7 @@ "text", "audio" ], + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, @@ -6369,6 +6374,7 @@ "text", "audio" ], + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, @@ -28065,7 +28071,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": true }, "fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct-hf": { "input_cost_per_token": 1.2e-06, @@ -28339,7 +28345,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": true }, "fireworks_ai/qwen3p7-plus": { "cache_read_input_token_cost": 8e-08, @@ -41933,6 +41939,120 @@ "supports_system_messages": true, "supports_native_structured_output": true }, + "bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 4.12e-08, + "input_cost_per_token_batches": 2.06e-08, + "input_cost_per_token_flex": 2.06e-08, + "input_cost_per_token_priority": 7.21e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 4.12e-08, + "output_cost_per_token_batches": 2.06e-08, + "output_cost_per_token_flex": 2.06e-08, + "output_cost_per_token_priority": 7.21e-08, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/ap-southeast-2/mistral.voxtral-small-24b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 1.03e-07, + "input_cost_per_token_batches": 5.15e-08, + "input_cost_per_token_flex": 5.15e-08, + "input_cost_per_token_priority": 1.8025e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 3.09e-07, + "output_cost_per_token_batches": 1.545e-07, + "output_cost_per_token_flex": 1.545e-07, + "output_cost_per_token_priority": 5.4075e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/ap-northeast-1/mistral.voxtral-mini-3b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 5e-08, + "input_cost_per_token_batches": 3e-08, + "input_cost_per_token_flex": 3e-08, + "input_cost_per_token_priority": 9e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 5e-08, + "output_cost_per_token_batches": 3e-08, + "output_cost_per_token_flex": 3e-08, + "output_cost_per_token_priority": 9e-08, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-northeast-1/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/ap-northeast-1/mistral.voxtral-small-24b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 1.2e-07, + "input_cost_per_token_batches": 6e-08, + "input_cost_per_token_flex": 6e-08, + "input_cost_per_token_priority": 2.1e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 3.6e-07, + "output_cost_per_token_batches": 1.8e-07, + "output_cost_per_token_flex": 1.8e-07, + "output_cost_per_token_priority": 6.3e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-northeast-1/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/eu-west-2/mistral.voxtral-mini-3b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 6e-08, + "input_cost_per_token_batches": 3e-08, + "input_cost_per_token_flex": 3e-08, + "input_cost_per_token_priority": 1.1e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 6e-08, + "output_cost_per_token_batches": 3e-08, + "output_cost_per_token_flex": 3e-08, + "output_cost_per_token_priority": 1.1e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/eu-west-2/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/eu-west-2/mistral.voxtral-small-24b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 1.6e-07, + "input_cost_per_token_batches": 8e-08, + "input_cost_per_token_flex": 8e-08, + "input_cost_per_token_priority": 2.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 4.7e-07, + "output_cost_per_token_batches": 2.3e-07, + "output_cost_per_token_flex": 2.3e-07, + "output_cost_per_token_priority": 8.2e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/eu-west-2/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, "mistral/codestral-2508": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 3e-07, @@ -61269,6 +61389,359 @@ "supports_tool_choice": true, "supports_vision": true }, + "streamlake/deepseek-v4-flash-0731": { + "cache_read_input_token_cost": 1.4e-08, + "input_cost_per_token": 4.4e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 1.32e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/DeepSeek-V4-Pro-0813": { + "cache_read_input_token_cost": 4.4e-08, + "input_cost_per_token": 1.32e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 1048566, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 3.96e-06, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/DeepSeek-V3": { + "cache_read_input_token_cost": 1.14e-07, + "input_cost_per_token": 2.86e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 56000, + "max_output_tokens": 8000, + "mode": "chat", + "output_cost_per_token": 1.143e-06, + "supports_function_calling": true, + "supports_reasoning": false, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/GLM-5.3": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_assistant_prefill": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/GLM-5.3-Flash": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_assistant_prefill": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/Kimi-K2-Instruct-0905": { + "cache_read_input_token_cost": 1.43e-07, + "input_cost_per_token": 5.71e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 224000, + "max_output_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 2.286e-06, + "supports_function_calling": true, + "supports_reasoning": false, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-r1": { + "cache_read_input_token_cost": 2.29e-07, + "input_cost_per_token": 5.71e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 98304, + "max_output_tokens": 16000, + "mode": "chat", + "output_cost_per_token": 2.286e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-v3.1": { + "cache_read_input_token_cost": 2.29e-07, + "input_cost_per_token": 5.71e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 98304, + "max_output_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 1.714e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-v4-flash": { + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 1.4e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 2.8e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-v4-pro": { + "cache_read_input_token_cost": 1.45e-07, + "input_cost_per_token": 1.74e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 3.48e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-v4.1-flash-1017": { + "cache_read_input_token_cost": 6e-09, + "input_cost_per_token": 3e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-v4.1-flash-260910": { + "cache_read_input_token_cost": 6e-09, + "input_cost_per_token": 3e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 1048568, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/glm-5": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 202752, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3.2e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/glm-5.1": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 202752, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/glm-5.2": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 1048566, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/kimi-k2.6": { + "cache_read_input_token_cost": 1.6e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 262144, + "max_output_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 4e-06, + "supports_function_calling": true, + "supports_vision": true, + "supports_reasoning": false, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/kimi-k2.7-code": { + "cache_read_input_token_cost": 1.9e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 262144, + "max_output_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 4e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/mimo-v2.6-flash": { + "cache_read_input_token_cost": 2.8e-09, + "input_cost_per_token": 1.4e-07, + "litellm_provider": "streamlake", + "mode": "chat", + "output_cost_per_token": 2.8e-07, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/mimo-v2.6-pro": { + "cache_read_input_token_cost": 3.6e-09, + "input_cost_per_token": 4.35e-07, + "litellm_provider": "streamlake", + "mode": "chat", + "output_cost_per_token": 8.7e-07, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/minimax-m2.5": { + "cache_creation_input_token_cost": 3.75e-07, + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 196608, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/minimax-m3": { + "cache_read_input_token_cost": 1.2e-07, + "cache_read_input_token_cost_above_512k_tokens": 2.4e-07, + "input_cost_per_token": 6e-07, + "input_cost_per_token_above_512k_tokens": 1.2e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 1000000, + "max_output_tokens": 512000, + "mode": "chat", + "output_cost_per_token": 2.4e-06, + "output_cost_per_token_above_512k_tokens": 4.8e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/qwen3.5-397b-a17b": { + "cache_read_input_token_cost": 1.2e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 258048, + "max_output_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/qwen3.6-27b": { + "input_cost_per_token": 6e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 229376, + "max_output_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "supports_function_calling": true, + "supports_vision": true, + "supports_reasoning": false, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/qwen3.6-35b-a3b": { + "input_cost_per_token": 3.75e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 229376, + "max_output_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.25e-06, + "supports_function_calling": true, + "supports_vision": true, + "supports_reasoning": false, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/qwen3.8-2.4t-a95b": { + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 983616, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 6e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "cache_creation_input_token_cost": 2.5e-06, + "explicit_cache_read_input_token_cost": 1.7e-07, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/qwen3.8-27b": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 5e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 983616, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "cache_creation_input_token_cost": 6.25e-07, + "explicit_cache_read_input_token_cost": 5e-08, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, "deepseek-v4-flash": { "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 6e-09, @@ -62584,6 +63057,14 @@ "litellm_provider": "anthropic" } }, + { + "name": "typesafe-jev-ids", + "pattern": "^jev-(?:latest|preview|1\\.13\\.0|1\\.14\\.0)$", + "description": "A bare Jev id that has a typesafe/ price entry (jev-latest, jev-preview, jev-1.13.0, jev-1.14.0), the name the /typesafe pass-through forwards to TypeSafe. Listed by name, matched case-insensitively like every rule here, so foo/jev-latest or a model group named like a Jev id TypeSafe does not serve, such as jev-1.0 or jev-internal-router, never resolves to typesafe. A new Jev version goes here together with its price entry. Provider inference feeds the proxy's provider/* wildcard access checks, so this rule is what lets typesafe/* on a key, team or project allowlist cover a jev-latest request.", + "model_info": { + "litellm_provider": "typesafe" + } + }, { "name": "claude-family-baseline", "pattern": "claude-[a-z]+-\\d+(?:[-.]\\d+)?(?:-\\d{8})?", @@ -63174,6 +63655,7 @@ "output_cost_per_token_priority": 9e-06, "source": "https://api.fireworks.ai/v1/serverless/models", "supports_function_calling": true, + "supports_mid_conversation_system": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -63320,6 +63802,7 @@ "output_cost_per_token_priority": 9e-06, "source": "https://api.fireworks.ai/v1/serverless/models", "supports_function_calling": true, + "supports_mid_conversation_system": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -77858,6 +78341,1254 @@ "supports_xhigh_reasoning_effort": true, "use_openai_responses_path": true }, + "kyma/claude-fable-5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-05, + "output_cost_per_token": 6.75e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true + }, + "kyma/claude-fable-5.1": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-05, + "output_cost_per_token": 6.75e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true + }, + "kyma/claude-haiku-4-5": { + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "input_cost_per_token": 1.35e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 6.75e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false, + "prompt_cache_min_tokens": 4096 + }, + "kyma/claude-haiku-5.5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-07, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 6.75e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": false, + "prompt_cache_min_tokens": 512 + }, + "kyma/claude-opus-4-7": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 6.75e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 3.375e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 2048 + }, + "kyma/claude-opus-5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 6.75e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 3.375e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 512 + }, + "kyma/claude-opus-5.5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512 + }, + "kyma/claude-sonnet-4-6": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 4.05e-06, + "cache_read_input_token_cost": 4.05e-07, + "output_cost_per_token": 2.025e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 1024 + }, + "kyma/claude-sonnet-5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.35e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 1024 + }, + "kyma/claude-sonnet-5.5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 1.35e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512 + }, + "kyma/deepseek-r1": { + "max_input_tokens": 64000, + "max_output_tokens": 32768, + "input_cost_per_token": 7.425e-07, + "output_cost_per_token": 2.957e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v3": { + "max_input_tokens": 160000, + "max_output_tokens": 8192, + "input_cost_per_token": 5.18e-07, + "output_cost_per_token": 7.571e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v4-flash": { + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "input_cost_per_token": 1.23e-07, + "cache_read_input_token_cost": 2.43e-08, + "output_cost_per_token": 2.458e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v4-flash-vision-exp": { + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "input_cost_per_token": 3.892e-07, + "output_cost_per_token": 1.168e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v4.1-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 2.7e-07, + "output_cost_per_token": 8.1e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 6.75e-07, + "cache_read_input_token_cost": 6.75e-08, + "output_cost_per_token": 4.05e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3.1-pro": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.62e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemini-3.5-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 2.025e-06, + "cache_read_input_token_cost": 2.025e-07, + "output_cost_per_token": 1.215e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3.5-flash-lite": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 4.05e-07, + "cache_read_input_token_cost": 4.05e-08, + "output_cost_per_token": 3.375e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3.6-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.013e-06, + "cache_read_input_token_cost": 1.0125e-07, + "output_cost_per_token": 5.063e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": false, + "supports_pdf_input": false + }, + "kyma/gemini-3.7-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 2.026e-06, + "cache_read_input_token_cost": 1.0125e-07, + "output_cost_per_token": 1.0126e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemini-3.8-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 2.026e-06, + "output_cost_per_token": 1.0126e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemini-4-argon": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemma-4-31b": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.642e-07, + "cache_read_input_token_cost": 1.642e-07, + "output_cost_per_token": 4.689e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-4.5-air": { + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "input_cost_per_token": 1.797e-07, + "cache_read_input_token_cost": 3.375e-08, + "output_cost_per_token": 1.175e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-4.7-flash": { + "max_input_tokens": 203000, + "max_output_tokens": 65536, + "input_cost_per_token": 8.1e-08, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 5.4e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.1": { + "max_input_tokens": 203000, + "max_output_tokens": 65536, + "input_cost_per_token": 1.89e-06, + "cache_read_input_token_cost": 2.7675e-07, + "output_cost_per_token": 5.94e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.2": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 9.029e-07, + "cache_read_input_token_cost": 2.9835e-07, + "output_cost_per_token": 3.083e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.3": { + "max_input_tokens": 1310720, + "max_output_tokens": 131072, + "input_cost_per_token": 1.178e-06, + "output_cost_per_token": 3.702e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.3-flash": { + "max_input_tokens": 1310720, + "max_output_tokens": 65536, + "input_cost_per_token": 1.72e-07, + "output_cost_per_token": 5.733e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gpt-5.6-luna": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.944e-07, + "cache_read_input_token_cost": 2.7e-08, + "output_cost_per_token": 1.767e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-luna-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.703e-07, + "cache_read_input_token_cost": 2.7e-08, + "output_cost_per_token": 1.621e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-sol": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-sol-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-terra": { + "max_input_tokens": 1050000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.62e-05, + "input_cost_per_token_above_272k_tokens": 5.4e-06, + "output_cost_per_token_above_272k_tokens": 2.43e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gpt-5.6-terra-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.62e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-6-astra": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-05, + "cache_read_input_token_cost": 1.35e-06, + "output_cost_per_token": 6.75e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gpt-6-luna": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.639e-07, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 8.195e-07, + "input_cost_per_token_above_272k_tokens": 2.7e-07, + "output_cost_per_token_above_272k_tokens": 1.0125e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-6.1-sol": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 1.35e-05, + "input_cost_per_token_above_272k_tokens": 5.4e-06, + "output_cost_per_token_above_272k_tokens": 2.025e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-oss-120b": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 4.73e-08, + "output_cost_per_token": 2.303e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/grok-4.20": { + "max_input_tokens": 2000000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.763e-06, + "output_cost_per_token": 3.528e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.20-multi-agent": { + "max_input_tokens": 2000000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.767e-06, + "output_cost_per_token": 3.534e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": false, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.3": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 1.92e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 3.838e-06, + "input_cost_per_token_above_200k_tokens": 3.375e-06, + "output_cost_per_token_above_200k_tokens": 6.75e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/grok-4.5": { + "max_input_tokens": 500000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.73e-06, + "output_cost_per_token": 8.189e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.6": { + "max_input_tokens": 500000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.758e-06, + "output_cost_per_token": 8.274e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.7": { + "max_input_tokens": 500000, + "max_output_tokens": 450000, + "input_cost_per_token": 2.16e-06, + "cache_read_input_token_cost": 5.4e-07, + "output_cost_per_token": 6.48e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-build": { + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "input_cost_per_token": 1.389e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 2.777e-06, + "input_cost_per_token_above_200k_tokens": 2.7e-06, + "output_cost_per_token_above_200k_tokens": 5.4e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/hy3": { + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "input_cost_per_token": 1.594e-07, + "output_cost_per_token": 6.576e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/hy4-preview": { + "max_input_tokens": 1048576, + "max_output_tokens": 64000, + "input_cost_per_token": 1.1259e-06, + "output_cost_per_token": 3.3764e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k2.6": { + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "input_cost_per_token": 5.909e-07, + "output_cost_per_token": 2.758e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k2.7-code": { + "max_input_tokens": 262144, + "max_output_tokens": 16384, + "input_cost_per_token": 1.009e-06, + "output_cost_per_token": 4.774e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k3": { + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "input_cost_per_token": 3.052e-06, + "cache_read_input_token_cost": 4.05e-07, + "output_cost_per_token": 1.5258e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k3.1-pro": { + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.35e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/llama-3.3-70b": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.35e-07, + "output_cost_per_token": 4.32e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/llama-4-maverick": { + "max_input_tokens": 1048576, + "max_output_tokens": 16384, + "input_cost_per_token": 3.344e-07, + "output_cost_per_token": 1.337e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/mimo-v2.5": { + "max_input_tokens": 1050000, + "max_output_tokens": 16384, + "input_cost_per_token": 1.859e-07, + "output_cost_per_token": 3.718e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/mimo-v2.6-flash": { + "max_input_tokens": 1050000, + "max_output_tokens": 131072, + "input_cost_per_token": 1.89e-07, + "output_cost_per_token": 3.78e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/mimo-v2.6-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 131072, + "input_cost_per_token": 5.805e-07, + "output_cost_per_token": 1.1745e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m2.5": { + "max_input_tokens": 196608, + "max_output_tokens": 32768, + "input_cost_per_token": 4.188e-07, + "output_cost_per_token": 1.675e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m2.7": { + "max_input_tokens": 204800, + "max_output_tokens": 32768, + "input_cost_per_token": 4.05e-07, + "output_cost_per_token": 1.62e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m3": { + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "input_cost_per_token": 3.956e-07, + "cache_read_input_token_cost": 7.56e-08, + "output_cost_per_token": 1.582e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m3.1-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 524288, + "input_cost_per_token": 2.025e-07, + "cache_read_input_token_cost": 2.025e-08, + "output_cost_per_token": 8.1e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/muse-glimmer-30b": { + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "input_cost_per_token": 4.554e-07, + "output_cost_per_token": 1.821e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/muse-spark-1.1": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.542e-06, + "output_cost_per_token": 5.243e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/muse-spark-1.2": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.773e-06, + "output_cost_per_token": 6.025e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/muse-spark-1.3": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.688e-06, + "output_cost_per_token": 5.738e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/muse-spark-open": { + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "input_cost_per_token": 1.688e-06, + "output_cost_per_token": 5.738e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/muse-wonder": { + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "input_cost_per_token": 5.4e-06, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/nemotron-3-ultra-550b": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 6.75e-07, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 3.375e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/nemotron-3.5-super-vl": { + "max_input_tokens": 1048576, + "max_output_tokens": 235929, + "input_cost_per_token": 8.1e-08, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 4.725e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3-32b": { + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "input_cost_per_token": 1.08e-07, + "output_cost_per_token": 3.78e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3-coder": { + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "input_cost_per_token": 3.546e-07, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 1.612e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.6-plus": { + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "input_cost_per_token": 4.388e-07, + "output_cost_per_token": 2.633e-06, + "input_cost_per_token_above_256k_tokens": 1.755e-06, + "output_cost_per_token_above_256k_tokens": 5.265e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.7-max": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 2.071e-06, + "cache_read_input_token_cost": 1.6875e-07, + "output_cost_per_token": 6.207e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.7-plus": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 4.325e-07, + "cache_read_input_token_cost": 8.64e-08, + "output_cost_per_token": 1.731e-06, + "input_cost_per_token_above_256k_tokens": 1.296e-06, + "output_cost_per_token_above_256k_tokens": 5.184e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.8-max": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 2.228e-06, + "cache_read_input_token_cost": 2.781e-07, + "output_cost_per_token": 6.684e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen3.7-flash": { + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "input_cost_per_token": 4.35e-08, + "cache_read_input_token_cost": 8.1e-09, + "output_cost_per_token": 1.886e-07, + "input_cost_per_token_above_32k_tokens": 1.35e-07, + "output_cost_per_token_above_32k_tokens": 5.4e-07, + "input_cost_per_token_above_256k_tokens": 2.7e-07, + "output_cost_per_token_above_256k_tokens": 1.08e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen3.8-27b": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 3.826e-07, + "output_cost_per_token": 2.732e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen3.8-flash": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 2.045e-07, + "output_cost_per_token": 6.41e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/sonar": { + "max_input_tokens": 127072, + "max_output_tokens": 4096, + "input_cost_per_token": 1.35e-06, + "output_cost_per_token": 1.35e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": false, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_pdf_input": false + }, + "kyma/sonar-pro": { + "max_input_tokens": 200000, + "max_output_tokens": 8000, + "input_cost_per_token": 4.05e-06, + "output_cost_per_token": 2.025e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": false, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_pdf_input": false + }, + "kyma/step-3.7-flash": { + "max_input_tokens": 256000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.229e-07, + "cache_read_input_token_cost": 5.4e-08, + "output_cost_per_token": 1.282e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/step-5-preview": { + "max_input_tokens": 1048576, + "max_output_tokens": 64000, + "input_cost_per_token": 1.35e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 3.645e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, "databricks/databricks-claude-haiku-5-5": { "cache_creation_input_token_cost": 1.2502e-07, "cache_creation_input_token_cost_above_100k_tokens": 6.2503e-07, @@ -78514,5 +80245,81 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true + }, + "bigmodel/glm-5.3": { + "input_cost_per_token": 1.1428571428571428e-06, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 2.857142857142857e-07, + "litellm_provider": "bigmodel", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "source": "https://docs.bigmodel.cn/cn/guide/start/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "thinking_always_on": true, + "metadata": { + "comment": "Official API list price checked 2026-10-27: input 8, output 28, cached input 2 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded." + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_native_streaming": true + }, + "bigmodel/glm-5.3-flash": { + "input_cost_per_token": 1.142857142857143e-07, + "output_cost_per_token": 4e-07, + "cache_read_input_token_cost": 3.2857142857142855e-08, + "litellm_provider": "bigmodel", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "source": "https://docs.bigmodel.cn/cn/guide/start/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "thinking_always_on": true, + "supports_vision": true, + "supports_video_input": true, + "metadata": { + "comment": "Official API list price checked 2026-10-27: input 0.8, output 2.8, cached input 0.23 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded." + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_native_streaming": true + }, + "bigmodel/glm-5.3-flashx": { + "input_cost_per_token": 2.857142857142857e-07, + "output_cost_per_token": 1e-06, + "cache_read_input_token_cost": 8.142857142857142e-08, + "litellm_provider": "bigmodel", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "source": "https://docs.bigmodel.cn/cn/guide/start/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "thinking_always_on": true, + "supports_vision": true, + "supports_video_input": true, + "metadata": { + "comment": "Official API list price checked 2026-10-27: input 2, output 7, cached input 0.57 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded." + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_native_streaming": true } } diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index 0a6c2bcb2a9..4188866de08 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -3785,6 +3785,32 @@ "generateContent": true } }, + "streamlake": { + "display_name": "StreamLake (`streamlake`)", + "url": "https://docs.litellm.ai/docs/providers/openai_compatible", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, "tencent": { "display_name": "Tencent TokenHub (`tencent`)", "url": "https://docs.litellm.ai/docs/providers/tencent", @@ -5300,6 +5326,58 @@ "a2a": false, "generateContent": false } + }, + "kyma": { + "display_name": "Kyma API (`kyma`)", + "url": "https://docs.litellm.ai/docs/providers/kyma", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, + "bigmodel": { + "display_name": "BigModel (Zhipu AI) (`bigmodel`)", + "url": "https://docs.bigmodel.cn/cn/guide/develop/responses/introduction", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } } }, "endpoints": { diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 39adb99a0d0..dd964a74735 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -7816,7 +7816,7 @@ "paths": { "/claude-code/marketplace.json": { "get": { - "description": "Serve marketplace.json for Claude Code plugin discovery.\n\nThis endpoint is accessed by Claude Code CLI when users run:\n- claude plugin marketplace add \n- claude plugin install @\n\nWithout `key` the catalog holds the enabled (public) plugins. With `?key=sk-...`\nthe key is authenticated and the catalog also holds the disabled plugins granted\nto it through `object_permission.skills` or an access group on the key or its team.\n\nReturns:\n Marketplace catalog with list of available plugins and their git sources.\n\nExample:\n ```bash\n claude plugin marketplace add http://localhost:4000/claude-code/marketplace.json\n claude plugin marketplace add \"http://localhost:4000/claude-code/marketplace.json?key=sk-...\"\n claude plugin install my-plugin@litellm\n ```", + "description": "Serve marketplace.json for Claude Code plugin discovery.\n\nThis endpoint is accessed by Claude Code CLI when users run:\n- claude plugin marketplace add \n- claude plugin install @\n\nWithout a key the catalog holds the enabled (public) plugins. With `?key=sk-...`\nthe key is authenticated and the catalog also holds the disabled plugins granted\nto it through `object_permission.skills` or an access group on the key or its team.\n\nWith `general_settings.claude_code_marketplace_auth_required: true` the key must come\nin the `Authorization` or `x-litellm-api-key` header (Claude Code sends them from the\nmarketplace source's `headers`), `?key=` is ignored, and a caller with no valid key\ngets a 401 with `WWW-Authenticate: Bearer`.\n\nReturns:\n Marketplace catalog with list of available plugins and their git sources.\n\nExample:\n ```bash\n claude plugin marketplace add http://localhost:4000/claude-code/marketplace.json\n claude plugin marketplace add \"http://localhost:4000/claude-code/marketplace.json?key=sk-...\"\n claude plugin install my-plugin@litellm\n ```", "operationId": "get_marketplace_claude_code_marketplace_json_get", "parameters": [ { @@ -33639,7 +33639,7 @@ ] }, "get": { - "description": "Get a response by ID.\n\nSupports both:\n- Polling IDs (litellm_poll_*): Returns cumulative cached content from background responses\n- Provider response IDs: Passes through to provider API\n\nFollows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/get\n\n```bash\n# Get polling response\ncurl -X GET http://localhost:4000/v1/responses/litellm_poll_abc123 -H \"Authorization: Bearer $LITELLM_MASTER_KEY\"\n\n# Get provider response\ncurl -X GET http://localhost:4000/v1/responses/resp_abc123 -H \"Authorization: Bearer $LITELLM_MASTER_KEY\"\n```", + "description": "Get a response by ID.\n\nSupports both:\n- Polling IDs (litellm_poll_*): Returns cumulative cached content from background responses\n- Provider response IDs: Passes through to provider API\n\nFollows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/get\n\n```bash\n# Get polling response\ncurl -X GET http://localhost:4000/v1/responses/litellm_poll_abc123 -H \"Authorization: Bearer $LITELLM_MASTER_KEY\"\n\n# Get provider response\ncurl -X GET http://localhost:4000/v1/responses/resp_abc123 -H \"Authorization: Bearer $LITELLM_MASTER_KEY\"\n\n# Resume a background response's event stream after sequence number 7\ncurl -N \"http://localhost:4000/v1/responses/resp_abc123?stream=true&starting_after=7\" -H \"Authorization: Bearer $LITELLM_MASTER_KEY\"\n```", "operationId": "get_response_openai_v1_responses__response_id__get", "parameters": [ { @@ -33650,6 +33650,46 @@ "title": "Response Id", "type": "string" } + }, + { + "description": "Replay the response's events as server-sent events (OpenAI, Azure, and Azure AI Foundry)", + "in": "query", + "name": "stream", + "required": false, + "schema": { + "type": "boolean" + } + }, + { + "description": "With stream=true, replay only the events after this sequence number", + "in": "query", + "name": "starting_after", + "required": false, + "schema": { + "minimum": 0, + "type": "integer" + } + }, + { + "description": "Additional output data to include, forwarded to the provider", + "in": "query", + "name": "include", + "required": false, + "schema": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + { + "description": "With stream=true, whether delta events carry obfuscation padding", + "in": "query", + "name": "include_obfuscation", + "required": false, + "schema": { + "type": "boolean" + } } ], "responses": { @@ -33659,6 +33699,12 @@ "schema": { "$ref": "#/components/schemas/ResponsesAPIResponse" } + }, + "text/event-stream": { + "schema": { + "description": "Server sent events when stream=true", + "type": "string" + } } }, "description": "Successful Response" diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 5ce6cd93a22..4047fa3f0ad 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2833,6 +2833,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): None, description="If True, stores request messages and responses in spend logs. Default is False.", ) + claude_code_marketplace_auth_required: bool = Field( + False, + description="If True, GET /claude-code/marketplace.json requires a LiteLLM key in the Authorization or x-litellm-api-key header (?key= is not accepted) and answers 401 with WWW-Authenticate: Bearer without one, and GET /public/skill_hub returns an empty list. Default is False, which serves the enabled plugins to anyone.", + ) disable_requester_ip_address_logging: bool | None = Field( None, description="If True, new spend log rows store requester_ip_address as null in the column, the metadata and the stored request. The stored request also drops client-IP forwarding headers such as x-forwarded-for, and IP addresses in its other header values and in stored error messages are masked. Guardrails, metrics and callbacks that read the standard logging payload still receive the IP. Default is False.", @@ -3744,6 +3748,7 @@ class SpendLogsMetadata(TypedDict): internal_call_origin: InternalCallOrigin | None billed_retry_attempt: ReadOnly[NotRequired[bool | None]] litellm_roi_estimator: ReadOnly[NotRequired[bool | None]] + is_silent_experiment: ReadOnly[NotRequired[bool | None]] guardrail_information: list[StandardLoggingGuardrailInformation] | None eval_information: Any | None status: StandardLoggingPayloadStatus @@ -3816,6 +3821,16 @@ class ManagementEndpointLoggingPayload(LiteLLMPydanticObjectBase): end_time: datetime | None = None +def proxy_exception_status_code(message: str, code: str) -> str: + # Litellm router.py returns "No healthy deployment available" when there are no deployments available + # Should map to 429 errors https://github.com/BerriAI/litellm/issues/2487 + if "No healthy deployment available" in message or "No deployments available" in message: + return "429" + if RouterErrors.no_deployments_with_tag_routing.value in message: + return "401" + return code + + class ProxyException(Exception): # NOTE: DO NOT MODIFY THIS # This is used to map exactly to OPENAI Exceptions @@ -3844,13 +3859,7 @@ class ProxyException(Exception): headers[k] = str(v) self.headers = headers or {} self.provider_specific_fields = provider_specific_fields - # rules for proxyExceptions - # Litellm router.py returns "No healthy deployment available" when there are no deployments available - # Should map to 429 errors https://github.com/BerriAI/litellm/issues/2487 - if "No healthy deployment available" in self.message or "No deployments available" in self.message: - self.code = "429" - elif RouterErrors.no_deployments_with_tag_routing.value in self.message: - self.code = "401" + self.code = proxy_exception_status_code(self.message, self.code) def to_dict(self) -> dict: """Converts the ProxyException instance to a dictionary.""" diff --git a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py index b0088d93112..fc82686947d 100644 --- a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py +++ b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py @@ -7,7 +7,8 @@ Actual plugin files are hosted on GitHub/GitLab/Bitbucket or as a zip archive on any HTTPS host (S3, Artifactory, a static file server). Endpoints: -/claude-code/marketplace.json - GET - List plugins for Claude Code discovery (unauthenticated; `?key=` adds the key's granted skills) +/claude-code/marketplace.json - GET - List plugins for Claude Code discovery (anonymous unless + general_settings.claude_code_marketplace_auth_required; a key adds its granted skills) /claude-code/plugins - POST - Register a new plugin (create-only, proxy admin only) /claude-code/plugins - GET - List plugins visible to the key (enabled, plus granted disabled ones) /claude-code/plugins/{name} - GET - Get plugin details (403 on a disabled plugin the key is not granted) @@ -31,14 +32,17 @@ from pydantic import TypeAdapter from typing_extensions import ReadOnly from litellm._logging import verbose_proxy_logger -from litellm.proxy._types import CommonProxyErrors, ProxyException, UserAPIKeyAuth +from litellm.proxy._types import CommonProxyErrors, ProxyErrorTypes, ProxyException, UserAPIKeyAuth from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_skill_access import ( SkillVisibility, resolve_skill_visibility, ) +from litellm.proxy.auth.auth_exception_handler import DB_UNAVAILABLE_FALLBACK_USER_ID from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.claude_code_marketplace_auth import marketplace_auth_required from litellm.proxy.common_utils.resource_ownership import is_proxy_admin from litellm.repositories.table_repositories import ClaudeCodePluginRepository +from litellm.types.proxy.auth.special_headers import SpecialHeaders from litellm.types.proxy.claude_code_endpoints import ( InstallationPreference, ListPluginsResponse, @@ -109,6 +113,41 @@ async def get_prisma_client() -> object: _get_prisma_client: Final = get_prisma_client +async def _header_key_caller(request: Request) -> UserAPIKeyAuth: + try: + caller: Final = await user_api_key_auth( + request=request, + api_key=request.headers.get(SpecialHeaders.openai_authorization.value) or "", + custom_litellm_key_header=request.headers.get(SpecialHeaders.custom_litellm_api_key.value), + ) + except ProxyException as e: + if e.code != "401": + raise + raise ProxyException( + message=e.message, + type=e.type, + param=e.param, + code=401, + headers={**e.headers, "WWW-Authenticate": "Bearer"}, + ) from e + if caller.user_id == DB_UNAVAILABLE_FALLBACK_USER_ID: + raise ProxyException( + message="The LiteLLM key could not be verified because the database is unavailable", + type=ProxyErrorTypes.no_db_connection, + param=None, + code=503, + ) + return caller + + +async def _marketplace_caller(request: Request, key: str | None) -> UserAPIKeyAuth | None: + if marketplace_auth_required(): + return await _header_key_caller(request) + if key: + return await user_api_key_auth(request=request, api_key=f"Bearer {key}") + return None + + @router.get( "/claude-code/marketplace.json", tags=["Claude Code Marketplace"], @@ -121,10 +160,15 @@ async def get_marketplace(request: Request, key: str | None = None): - claude plugin marketplace add - claude plugin install @ - Without `key` the catalog holds the enabled (public) plugins. With `?key=sk-...` + Without a key the catalog holds the enabled (public) plugins. With `?key=sk-...` the key is authenticated and the catalog also holds the disabled plugins granted to it through `object_permission.skills` or an access group on the key or its team. + With `general_settings.claude_code_marketplace_auth_required: true` the key must come + in the `Authorization` or `x-litellm-api-key` header (Claude Code sends them from the + marketplace source's `headers`), `?key=` is ignored, and a caller with no valid key + gets a 401 with `WWW-Authenticate: Bearer`. + Returns: Marketplace catalog with list of available plugins and their git sources. @@ -137,10 +181,7 @@ async def get_marketplace(request: Request, key: str | None = None): """ try: prisma_client: Final = await get_prisma_client() - - caller: Final[UserAPIKeyAuth | None] = ( - await user_api_key_auth(request=request, api_key=f"Bearer {key}") if key else None - ) + caller: Final = await _marketplace_caller(request, key) visibility: Final[SkillVisibility] = await resolve_skill_visibility(caller) plugins: Final[Sequence[_PluginRecord]] = await ClaudeCodePluginRepository(prisma_client).table.find_many( where=visibility.where() @@ -201,7 +242,7 @@ async def get_marketplace(request: Request, key: str | None = None): verbose_proxy_logger.exception("Error generating marketplace: %s", e) raise HTTPException( status_code=500, - detail={"error": f"Failed to generate marketplace: {e}"}, + detail={"error": "Failed to generate marketplace"}, ) diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 1d88e133cf9..1a2e610d5c4 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -2,7 +2,7 @@ Unified /v1/messages endpoint - (Anthropic Spec) """ -from collections.abc import Mapping +from collections.abc import AsyncGenerator, Mapping from types import MappingProxyType from typing import Final @@ -19,12 +19,10 @@ from litellm.anthropic_interface.exceptions import ( ) from litellm.integrations.custom_guardrail import ModifyResponseException from litellm.litellm_core_utils.tool_fields import first_validation_error +from litellm.llms.anthropic.chat.guardrail_translation.handler import AnthropicMessagesHandler from litellm.llms.anthropic.messages.context_management import ( AnthropicContextManagementError, ) -from litellm.llms.base_llm.guardrail_translation.utils import ( - blocked_response_usage as _blocked_response_usage, -) from litellm.proxy._types import * from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_request_processing import ( @@ -44,6 +42,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # leg from litellm.proxy.common_utils.openai_error_payload import ( LITELLM_CALL_ID_HEADER, error_status_code, + litellm_call_id_headers, openai_error_param, openai_error_type, with_litellm_call_id, @@ -58,6 +57,8 @@ from litellm.router import Router router: Final = APIRouter() +_NO_HEADERS: Final[Mapping[str, str]] = MappingProxyType({}) + def _with_provider_specific_fields(exc: ProxyException, detail: AnthropicErrorDetail) -> AnthropicErrorDetail: if not exc.provider_specific_fields: @@ -229,44 +230,22 @@ async def anthropic_response( request_data=_data, ) - # Create Anthropic-formatted response with violation message - import uuid + block_headers: Final = litellm_call_id_headers(base_llm_response_processor.litellm_call_id) or _NO_HEADERS + block_handler: Final = AnthropicMessagesHandler() - from litellm.types.utils import AnthropicMessagesResponse + if data.get("stream") is True: + block_chunks: Final = block_handler.build_block_sse_chunks(e) - # Report the blocked LLM response's real token usage (carried on the - # exception) instead of discarding it; zero for pre-call blocks. - _usage: Final = _blocked_response_usage(e.original_response) - - _anthropic_response: Final = AnthropicMessagesResponse( - id=f"msg_{uuid.uuid4()}", - type="message", - role="assistant", - content=[{"type": "text", "text": e.message}], - model=e.model, - stop_reason="end_turn", - usage=_usage, - ) - - if data.get("stream", None) is not None and data["stream"] is True: - # For streaming, use the standard SSE data generator - async def _passthrough_stream_generator(): - yield _anthropic_response - - selected_data_generator: Final = ProxyBaseLLMRequestProcessing.async_sse_data_generator( - response=_passthrough_stream_generator(), - user_api_key_dict=user_api_key_dict, - request_data=_data, - proxy_logging_obj=proxy_logging_obj, - ) + async def _blocked_stream() -> AsyncGenerator[str, None]: + for chunk in block_chunks: + yield chunk.decode() return await create_response( - generator=selected_data_generator, - media_type="text/event-stream", - headers={}, + generator=_blocked_stream(), media_type="text/event-stream", headers=block_headers ) - return _anthropic_response + fastapi_response.headers.update(block_headers) + return block_handler.build_block_response(e) except AnthropicContextManagementError as e: if e.status_code >= 500: # Server-side polyfill failures hit the failure hook for spend/alert diff --git a/litellm/proxy/anthropic_endpoints/gateway_endpoints.py b/litellm/proxy/anthropic_endpoints/gateway_endpoints.py index 1911cb574ef..6c58f2edc02 100644 --- a/litellm/proxy/anthropic_endpoints/gateway_endpoints.py +++ b/litellm/proxy/anthropic_endpoints/gateway_endpoints.py @@ -45,7 +45,11 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # leg safe_set_request_parsed_body, ) from litellm.proxy.management_endpoints.sso_helper_utils import CLI_SSO_SESSIONS_TARGET -from litellm.proxy.management_endpoints.ui_sso import CliSsoTeamDetail +from litellm.proxy.management_endpoints.ui_sso import ( + CLI_SSO_BROWSER_TEAM_CHOICE, + CLI_SSO_CHOSEN_TEAM_ID, + CliSsoTeamDetail, +) from litellm.proxy.mcp_server.gateway_dcr_flow import ( PROXY_API_AUDIENCE, MintedProxyCredential, @@ -235,6 +239,7 @@ async def device_authorization(request: Request) -> JSONResponse: "sso_complete": False, "user_code_verified": False, "session_data": None, + CLI_SSO_BROWSER_TEAM_CHOICE: True, } _set_cli_sso_flow(login_id=login_id, cache=cli_sso_session_cache, flow=flow) @@ -267,7 +272,15 @@ def _validate_login(flow: Mapping[str, object]) -> _GatewayLogin | _OAuthError: status_code=400, error="invalid_grant", description="The login session is malformed; sign in again" ) - team_id: Final = session_data.teams[0] if session_data.teams else None + chosen: Final = flow.get(CLI_SSO_CHOSEN_TEAM_ID) + chosen_team_id: Final = chosen if isinstance(chosen, str) else None + if chosen_team_id is not None and chosen_team_id not in session_data.teams: + return _OAuthError( + status_code=400, + error="invalid_grant", + description="The team picked at sign-in is no longer one of your teams; sign in again", + ) + team_id: Final = chosen_team_id or next(iter(session_data.teams), None) selected_team: Final = selected_cli_sso_team_detail(team_details=session_data.team_details, team_id=team_id) if selected_team is None: return _OAuthError( diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 448643ef59b..882e98ec097 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -46,6 +46,11 @@ from litellm.constants import ( from litellm.litellm_core_utils.dd_tracing import tracer from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.litellm_core_utils.safe_json_loads import safe_json_loads +from litellm.litellm_core_utils.tool_fields import DropPolicy +from litellm.llms.base_llm.managed_resources.utils import ( + extract_provider_resource_id_from_unified_id, + is_base64_encoded_unified_id, +) from litellm.models.project import LiteLLM_ProjectTable from litellm.proxy._types import ( MODEL_LIST_ROUTES, @@ -165,6 +170,7 @@ from litellm.types.proxy.auth.auth_checks import UserNotFoundError from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget from litellm.types.router import DeploymentTypedDict from litellm.utils import get_utc_datetime +from litellm.vector_stores.vector_store_registry import listed_vector_store_ids, vector_store_ids_in_tool from .auth_checks_organization import ( add_team_org_context_to_request_body, @@ -1063,6 +1069,15 @@ def is_mcp_discovery_request(route: str, request_body: Mapping[str, object]) -> return isinstance(method, str) and method in MCP_ZERO_SPEND_JSONRPC_METHODS +class ModelMaxBudgetHolder(Protocol): + @property + def model_max_budget(self) -> Mapping[str, object] | None: ... + + +def model_max_budget_of(holder: ModelMaxBudgetHolder) -> Mapping[str, object] | None: + return holder.model_max_budget + + def route_skips_budget_checks(route: str) -> bool: return route not in BUDGET_ENFORCED_SIDE_EFFECT_ROUTES and ( route in MODEL_DISCOVERY_ROUTES or not RouteChecks.is_llm_api_route(route=route) @@ -1505,6 +1520,7 @@ async def common_checks( valid_token=valid_token, user_object=user_object, deny_by_default=_vector_store_deny_by_default(_typed_request_body(general_settings)), + llm_router=llm_router, ) # 12. [OPTIONAL] Tool allowlist - key/team allowed_tools (no DB in hot path) @@ -7976,6 +7992,57 @@ def _strict_requested_vector_store_ids(request_body: Mapping[str, object]) -> tu ) +@dataclass(frozen=True, slots=True) +class _UnreadableVectorStoreIds: + pass + + +def _top_level_vector_store_ids( + vector_store_ids: object, drop_policies: Callable[[], tuple[DropPolicy, ...]] +) -> tuple[str, ...] | _UnreadableVectorStoreIds: + listed: Final = listed_vector_store_ids(vector_store_ids) + if vector_store_ids is None or vector_store_ids == [*listed]: + return listed + if any(policy.drops("vector_store_ids") for policy in drop_policies()): + return listed + if isinstance(vector_store_ids, str): + return (vector_store_ids,) + return _UnreadableVectorStoreIds() + + +def _legacy_requested_vector_store_ids( + request_body: Mapping[str, object], drop_policies: Callable[[], tuple[DropPolicy, ...]] +) -> tuple[str, ...] | _UnreadableVectorStoreIds: + """ + Reads tools the way VectorStoreRegistry does and leaves a malformed tool to the proxy's tool + validation. A top-level vector_store_ids string names one store, since a query parameter always + arrives as a string. Any other value that is not a list of strings names no readable stores unless a + drop policy the request can resolve to drops it, and then only the string entries of a list name + stores, as in registry retrieval. + """ + top_level: Final = _top_level_vector_store_ids(request_body.get("vector_store_ids"), drop_policies) + if isinstance(top_level, _UnreadableVectorStoreIds): + return top_level + return tuple( + chain( + top_level, + chain.from_iterable(vector_store_ids_in_tool(tool) for tool in _tools(request_body.get("tools"))), + ) + ) + + +def _restricts_vector_stores(grants: Sequence[tuple[GrantLayer, LiteLLM_ObjectPermissionTable | None]]) -> bool: + return any(grant is not None and bool(grant.vector_stores) for _, grant in grants) + + +def _request_drop_policies( + request_body: Mapping[str, object], llm_router: Router | None, valid_token: UserAPIKeyAuth | None +) -> tuple[DropPolicy, ...]: + from litellm.proxy.common_request_processing import counted_drop_policies + + return counted_drop_policies(request_body, llm_router, valid_token.team_id if valid_token is not None else None) + + def _require_vector_store_grant( object_type: GrantLayer, vector_store_ids_to_run: Sequence[str], @@ -8023,6 +8090,7 @@ async def vector_store_access_check( *, user_object: LiteLLM_UserTable | None = None, deny_by_default: bool = False, + llm_router: Router | None = None, ): """ Checks whether the caller may use every vector store the request names. @@ -8031,11 +8099,19 @@ async def vector_store_access_check( `retrieval_config.vector_store_id`. Grants come from each identity's `object_permission.vector_stores`. - With `deny_by_default=False` (legacy), a key or team only restricts access when its list is - nonempty, and stores are only read from the request when a vector store registry is loaded. + Both modes read the same request fields, with or without a vector store registry, and answer 400 + when the top-level `vector_store_ids` is not a list of strings, which legacy mode does only when the + key or team has an allowlist and the value is not a string, which it checks as one store id. A + LiteLLM-managed store id is compared by the provider store id it decodes to, on both sides. Legacy mode skips that value instead when the request, `litellm_settings`, the router defaults or a deployment of the requested + model drops it (`drop_params`, or `additional_drop_params` naming it), the same settings the proxy's + tool validation reads, and reads tools the way the registry does, where strict mode answers 400 on a + malformed one. - With `deny_by_default=True` (`general_settings.vector_store_deny_by_default`), stores are read - even without a registry, and a missing record, `null` or `[]` grants nothing: + With `deny_by_default=False` (legacy), a key or team only restricts access when its list is + nonempty. + + With `deny_by_default=True` (`general_settings.vector_store_deny_by_default`), a missing record, + `null` or `[]` grants nothing: - virtual key: the key must grant every store, and so must its team when it has one, even if the team failed to load @@ -8047,8 +8123,8 @@ async def vector_store_access_check( rescue nor restrict a key or team request. Raises ProxyException (401, `{key,team,user}_vector_store_access_denied`) on the first identity - that does not grant a requested store, and with the flag on, ProxyException (400, - `invalid_request_error`) when a `vector_store_ids` field is not a list of strings. + that does not grant a requested store, and ProxyException (400, `invalid_request_error`) on a + malformed `vector_store_ids` the request does not drop and, in legacy mode, an allowlist applies to. """ from litellm.proxy.proxy_server import prisma_client, user_api_key_cache @@ -8059,23 +8135,24 @@ async def vector_store_access_check( verbose_proxy_logger.debug("Prisma client not found, skipping vector store access check") return True - registry_ids: Final = ( - _strict_requested_vector_store_ids(_typed_request_body(request_body)) + typed_request_body: Final = _typed_request_body(request_body) + requested_ids: Final = ( + _strict_requested_vector_store_ids(typed_request_body) if deny_by_default - else ( - litellm.vector_store_registry.get_vector_store_ids_to_run( - non_default_params=request_body, tools=request_body.get("tools", None) - ) - if litellm.vector_store_registry is not None - else None + else _legacy_requested_vector_store_ids( + typed_request_body, lambda: _request_drop_policies(typed_request_body, llm_router, valid_token) ) - or () ) - rag_vector_store_id: Final = _get_rag_query_vector_store_id(_typed_request_body(request_body)) + if isinstance(requested_ids, _UnreadableVectorStoreIds): + if _restricts_vector_stores( + await _identity_grants(valid_token, team_object, None, prisma_client, user_api_key_cache) + ): + raise _malformed_vector_store_ids() + return True + rag_vector_store_id: Final = _get_rag_query_vector_store_id(typed_request_body) rag_ids: Final = (rag_vector_store_id,) if rag_vector_store_id is not None else () - vector_store_ids_to_run: Final = tuple(dict.fromkeys((*registry_ids, *rag_ids))) + vector_store_ids_to_run: Final = tuple(dict.fromkeys((*requested_ids, *rag_ids))) if not vector_store_ids_to_run: - verbose_proxy_logger.debug("Vector store to run not found, skipping vector store access check") return True ######################################################### @@ -8101,6 +8178,12 @@ async def vector_store_access_check( return True +def _provider_vector_store_id(vector_store_id: str) -> str: + if not is_base64_encoded_unified_id(vector_store_id): + return vector_store_id + return extract_provider_resource_id_from_unified_id(vector_store_id) or vector_store_id + + def _can_object_call_vector_stores( object_type: Literal["key", "team", "org", "user"], vector_store_ids_to_run: Sequence[str], @@ -8119,8 +8202,9 @@ def _can_object_call_vector_stores( if len(object_permissions.vector_stores) == 0: return True + allowed_stores: Final = frozenset(map(_provider_vector_store_id, object_permissions.vector_stores)) for vector_store_id in vector_store_ids_to_run: - if vector_store_id not in object_permissions.vector_stores: + if _provider_vector_store_id(vector_store_id) not in allowed_stores: raise ProxyException( message=f"User not allowed to access vector store. Tried to access {vector_store_id}. Only allowed to access {object_permissions.vector_stores}", type=ProxyErrorTypes.get_vector_store_access_error_type_for_object(object_type), diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index c4c538083ee..48884d0fedc 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -20,6 +20,8 @@ from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager, get_valid_models +LLM_PROVIDER_PREFIXES: Final = frozenset(llm_provider.value for llm_provider in LlmProviders) + _CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields) @@ -458,23 +460,21 @@ def get_known_models_from_wildcard( # add model prefix to wildcard models wildcard_models = [f"{model_prefix}{model.removeprefix(provider_id_prefix)}" for model in wildcard_models] - known_providers: Final = {provider.value for provider in LlmProviders} - suffix_appended_wildcard_models: Final = [] - for model in wildcard_models: - if not model.startswith(f"{wildcard_provider_prefix}/"): - # `get_provider_models` returns provider-prefixed ids (e.g. "ollama/gemma3:1b"). - # When the wildcard uses a custom prefix (e.g. "ollama_server1/*" to distinguish - # multiple instances), replace that existing provider prefix instead of stacking - # both, which would otherwise yield an uncallable "ollama_server1/ollama/gemma3:1b". - # Only strip the leading segment when it is a known provider, so ids whose first - # segment is an org rather than a provider (e.g. "meta-llama/Llama-3-8B") keep it. - leading, sep, model_suffix = model.partition("/") - if sep and leading in known_providers: - model = f"{wildcard_provider_prefix}/{model_suffix}" - else: - model = f"{wildcard_provider_prefix}/{model}" - suffix_appended_wildcard_models.append(model) - return list(dict.fromkeys(suffix_appended_wildcard_models)) + return list( + dict.fromkeys(_under_wildcard_prefix(model, wildcard_provider_prefix, provider) for model in wildcard_models) + ) + + +def _under_wildcard_prefix(model: str, wildcard_provider_prefix: str, listed_provider: str) -> str: + if model.startswith(f"{wildcard_provider_prefix}/"): + return model + leading, sep, model_suffix = model.partition("/") + leading_is_listed_providers_own_prefix: Final = ( + sep != "" + and leading in LLM_PROVIDER_PREFIXES + and (leading == listed_provider or listed_provider.startswith(f"{leading}_")) + ) + return f"{wildcard_provider_prefix}/{model_suffix if leading_is_listed_providers_own_prefix else model}" def expand_wildcard_deployments_for_model_info( diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 79ad4824c92..292a147d200 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -40,6 +40,7 @@ from litellm.integrations.otel.model.config import is_otel_v2_enabled from litellm.integrations.otel.runtime import phase_event, phase_span, seed_request_identity from litellm.litellm_core_utils.dd_tracing import tracer from litellm.litellm_core_utils.dot_notation_indexing import get_nested_value +from litellm.models.end_user import LiteLLM_EndUserTable from litellm.proxy._types import * from litellm.proxy.agent_endpoints.auth.agent_caller import agent_caller_from_headers from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module exports @@ -85,6 +86,7 @@ from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module expo jwt_key_mapping_cache_key, key_access_group_grants_model, key_model_aliases_for_auth_check, + model_max_budget_of, resolve_and_validate_end_user_id, resolve_default_end_user_budget, route_skips_budget_checks, @@ -384,6 +386,16 @@ class _TeamModelBudgetLimiter(Protocol): ) -> bool: ... +class _EndUserModelBudgetLimiter(Protocol): + async def is_end_user_within_model_budget( + self, + end_user_id: str, + end_user_model_max_budget: Mapping[str, object], + model: str, + team_id: str | None = None, + ) -> bool: ... + + class _EveryModelBudgetLimiter(Protocol): async def is_key_within_model_budget(self, user_api_key_dict: UserAPIKeyAuth, model: str) -> bool: ... @@ -530,6 +542,49 @@ async def _check_team_model_budget( ) +async def _end_user_model_max_budget( + end_user_id: str | None, + end_user_object: LiteLLM_EndUserTable | None, + key_end_user_budget_id: str | None, + prisma_client: PrismaClient | None, + user_api_key_cache: UserApiKeyCache, + parent_otel_span: Span | None, +) -> Mapping[str, object] | None: + budget_table: Final = ( + end_user_object.litellm_budget_table + if end_user_object is not None + else ( + await resolve_default_end_user_budget( + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + key_end_user_budget_id=key_end_user_budget_id, + parent_otel_span=parent_otel_span, + ) + if end_user_id and prisma_client is not None + else None + ) + ) + return model_max_budget_of(budget_table) if budget_table is not None else None + + +async def _check_end_user_model_budget( + end_user_id: str | None, + end_user_model_max_budget: Mapping[str, object] | None, + model_max_budget_limiter: _EndUserModelBudgetLimiter, + models: Sequence[str], + team_id: str | None, +) -> None: + if not end_user_id or not end_user_model_max_budget: + return + for model_name in models: + await model_max_budget_limiter.is_end_user_within_model_budget( + end_user_id=end_user_id, + end_user_model_max_budget=end_user_model_max_budget, + model=model_name, + team_id=team_id, + ) + + async def _is_within_every_model_budget( valid_token: UserAPIKeyAuth, model_max_budget_limiter: _EveryModelBudgetLimiter, @@ -2094,6 +2149,10 @@ async def _resolve_object_permission_for_unresolvable_team( ) +def _custom_auth_token_for_this_request(response: object) -> UserAPIKeyAuth: + return UserAPIKeyAuth.model_validate(response).model_copy() + + async def user_api_key_auth_builder( request: Request, api_key: str, @@ -2172,7 +2231,7 @@ async def user_api_key_auth_builder( request=request, api_key=api_key, user_custom_auth=user_custom_auth ) if response is not None and isinstance(response, UserAPIKeyAuth): - validated = UserAPIKeyAuth.model_validate(response) + validated = _custom_auth_token_for_this_request(response) if getattr(litellm, "enable_post_custom_auth_checks", False): validated = await _run_post_custom_auth_checks( valid_token=validated, @@ -2188,7 +2247,7 @@ async def user_api_key_auth_builder( custom_auth_api_key = True elif user_custom_auth is not None: response = await user_custom_auth(request=request, api_key=api_key) - validated = UserAPIKeyAuth.model_validate(response) + validated = _custom_auth_token_for_this_request(response) if getattr(litellm, "enable_post_custom_auth_checks", False): validated = await _run_post_custom_auth_checks( valid_token=validated, @@ -3129,7 +3188,7 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e ) # Recompute after a potential budget-fallback rewrite so - # the end-user check below validates the final model + # the user check below validates the final model current_model = _get_model_from_request_context( request_data=request_data, route=route, @@ -3147,23 +3206,6 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e models=current_models, ) - # Check 5b. End-user model max budget - end_user_mmb: Final = valid_token.end_user_model_max_budget - if ( - end_user_mmb is not None - and isinstance(end_user_mmb, dict) - and len(end_user_mmb) > 0 - and current_models - and valid_token.end_user_id is not None - ): - for model_name in current_models: - await model_max_budget_limiter.is_end_user_within_model_budget( - end_user_id=valid_token.end_user_id, - end_user_model_max_budget=end_user_mmb, - model=model_name, - team_id=valid_token.team_id, - ) - # Check 6: Additional Common Checks across jwt + key auth if valid_token.team_id is not None: try: @@ -3733,9 +3775,7 @@ async def run_centralized_common_checks( project_object: Final[LiteLLM_ProjectTableCachedObj | None] = ( None if isinstance(project_result, BaseException) else project_result ) - end_user_object: Final[LiteLLM_EndUserTable | None] = ( - None if isinstance(end_user_result, BaseException) else end_user_result - ) + end_user_object: Final = end_user_result if isinstance(end_user_result, LiteLLM_EndUserTable) else None global_proxy_spend: float | None = None if isinstance(global_spend_result, BaseException) else global_spend_result carry_team_and_user_budget_state( valid_token=user_api_key_auth_obj, @@ -3795,6 +3835,21 @@ async def run_centralized_common_checks( parent_otel_span=parent_otel_span, keep_token_limits=user_custom_auth is not None, ) + end_user_model_max_budget: Final = await _end_user_model_max_budget( + end_user_id=end_user_id, + end_user_object=end_user_object, + key_end_user_budget_id=key_end_user_budget_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=parent_otel_span, + ) + if end_user_id and ( + not user_api_key_auth_obj.authenticated_by_custom_auth + or user_api_key_auth_obj.end_user_model_max_budget is None + ): + user_api_key_auth_obj.end_user_model_max_budget = ( # rebind-ok: the spend hook charges from this token + dict(end_user_model_max_budget) if end_user_model_max_budget else None + ) skip_budget_checks: Final = _should_skip_budget_checks( request_data=request_data, @@ -3887,6 +3942,21 @@ async def run_centralized_common_checks( user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, ) + await _check_end_user_model_budget( + end_user_id=end_user_id, + end_user_model_max_budget=user_api_key_auth_obj.end_user_model_max_budget, + model_max_budget_limiter=model_max_budget_limiter, + team_id=user_api_key_auth_obj.team_id, + models=_get_model_names_for_budget_checks( + model=_get_model_from_request_context( + request_data=request_data, + route=route, + request=request, + llm_router=llm_router, + team_id=user_api_key_auth_obj.team_id, + ) + ), + ) if sub_call and RouteChecks.is_llm_api_route(route=route): await virtual_key_max_budget_check( valid_token=user_api_key_auth_obj, diff --git a/litellm/proxy/client/cli/commands/agents.py b/litellm/proxy/client/cli/commands/agents.py index 4d09e4d1e0f..f19fb508c7d 100644 --- a/litellm/proxy/client/cli/commands/agents.py +++ b/litellm/proxy/client/cli/commands/agents.py @@ -5,16 +5,18 @@ import shutil import subprocess import sys import tempfile -from collections.abc import Callable, Mapping, Sequence +from collections import Counter +from collections.abc import Callable, Hashable, Iterable, Mapping, Sequence from dataclasses import dataclass from itertools import chain from pathlib import Path from types import MappingProxyType from typing import Final, TypeAlias, TypeVar +from urllib.parse import urlsplit import click import requests -from pydantic import JsonValue, TypeAdapter, ValidationError +from pydantic import ConfigDict, JsonValue, TypeAdapter, ValidationError from litellm.proxy.common_utils.codex_model_catalog import ( CODEX_BASE_INSTRUCTIONS_PATH, @@ -497,6 +499,113 @@ def _fetch_model_listing( return listing.data +class _DeploymentParams(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + model: str + custom_llm_provider: str | None = None + api_base: str | None = None + litellm_credential_name: str | None = None + reasoning_effort: JsonValue = None + + +class _DeploymentInfo(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + team_public_model_name: str | None = None + blocked: bool | None = None + + +class _Deployment(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + model_name: str + litellm_params: _DeploymentParams + model_info: _DeploymentInfo | None = None + + +class _DeploymentListing(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + data: tuple[_Deployment, ...] + + +@dataclass(frozen=True, slots=True) +class ModelUpstream: + """The OpenAI model every deployment behind a public name routes to, and the reasoning effort they all pin.""" + + model: str + reasoning_effort: str | None = None + + +_NO_UPSTREAMS: Final[Mapping[str, ModelUpstream]] = MappingProxyType({}) +_Pinned: Final = TypeVar("_Pinned", bound=Hashable) + + +def _public_name(deployment: _Deployment) -> str: + team_name: Final = deployment.model_info.team_public_model_name if deployment.model_info is not None else None + return team_name or deployment.model_name + + +def _is_paused(deployment: _Deployment) -> bool: + return deployment.model_info is not None and deployment.model_info.blocked is True + + +def _is_openai_host(params: _DeploymentParams) -> bool: + if params.api_base is None: + return not params.litellm_credential_name + host: Final = urlsplit(params.api_base).hostname or "" + return host == "api.openai.com" or host.endswith(".api.openai.com") + + +def _openai_model(params: _DeploymentParams) -> str | None: + if params.custom_llm_provider not in (None, "openai") or not _is_openai_host(params): + return None + provider, slash, name = params.model.partition("/") + if not slash: + return params.model + return name if provider == "openai" else None + + +def _pinned_effort(params: _DeploymentParams) -> str | None: + return params.reasoning_effort if isinstance(params.reasoning_effort, str) else None + + +def _shared_by_every_deployment(values: Iterable[tuple[str, _Pinned]]) -> Mapping[str, _Pinned]: + distinct: Final = frozenset(values) + values_per_name: Final = Counter(name for name, _ in distinct) + return MappingProxyType({name: value for name, value in distinct if values_per_name[name] == 1}) + + +def _upstreams_by_public_name(deployments: Sequence[_Deployment]) -> Mapping[str, ModelUpstream]: + live: Final = tuple(d for d in deployments if not _is_paused(d)) + models: Final = _shared_by_every_deployment((_public_name(d), _openai_model(d.litellm_params)) for d in live) + efforts: Final = _shared_by_every_deployment((_public_name(d), _pinned_effort(d.litellm_params)) for d in live) + return MappingProxyType( + {name: ModelUpstream(model, efforts.get(name)) for name, model in models.items() if model is not None} + ) + + +def _fetch_model_upstreams( + base_url: str, + api_key: str, + *, + get: Callable[..., requests.Response], +) -> Mapping[str, ModelUpstream]: + url: Final = base_url.rstrip("/") + "/v1/model/info" + try: + resp: Final = get(url, headers=MappingProxyType({"Authorization": f"Bearer {api_key}"}), timeout=10) + except requests.RequestException: + return _NO_UPSTREAMS + if resp.status_code != 200: + return _NO_UPSTREAMS + try: + listing: Final = _DeploymentListing.model_validate_json(resp.content) + except ValidationError: + return _NO_UPSTREAMS + return _upstreams_by_public_name(listing.data) + + class _OpenCodeLimit(LiteLLMBaseModel): context: int output: int @@ -576,8 +685,21 @@ def opencode_model_sync_env( return MappingProxyType({OPENCODE_CONFIG_CONTENT_ENV: opencode_provider_config(base_url, listing)}) +def _codex_row(model: ListedModel, upstream: ModelUpstream | None) -> CodexCatalogRow: + return CodexCatalogRow( + id=model.id, + mode=model.mode, + max_input_tokens=model.max_input_tokens, + upstream_model=upstream.model if upstream is not None else None, + reasoning_effort=upstream.reasoning_effort if upstream is not None else None, + ) + + def codex_model_catalog( - models: Sequence[ListedModel], stock: Sequence[CodexStockModel], instructions: str + models: Sequence[ListedModel], + stock: Sequence[CodexStockModel], + instructions: str, + upstreams: Mapping[str, ModelUpstream], ) -> str | None: """The `model_catalog_json` body listing the proxy's chat models, or None if there are none. @@ -586,14 +708,23 @@ def codex_model_catalog( installed Codex knows keeps that Codex's own entry and the proxy only decides its place in the picker: the listing orders it, lists it even when Codex hides it or keeps it off the API, and keeps Codex's upgrade nudge only - when the model it points at is served too. A model Codex does not know gets the fallback + when the model it points at is served too. An alias of such a model, per + `upstreams`, takes that entry too, minus the defaults Codex picks for the + model: no service tier or upgrade nudge, the reasoning effort its + deployments pin, and a window no larger than its listed max_input_tokens. + A model Codex does not know gets the fallback entry, with the same base instructions Codex itself uses so the agent never - runs without a system prompt. The catalog is the one `/v1/models?client_version=...` - serves, built from the installed Codex's own entries instead of the proxy's vendored ones. + runs without a system prompt. Apart from aliases, the catalog is the one + `/v1/models?client_version=...` serves, built from the installed Codex's own + entries instead of the proxy's vendored ones. """ - rows: Final = tuple(CodexCatalogRow(id=m.id, mode=m.mode, max_input_tokens=m.max_input_tokens) for m in models) + rows: Final = tuple(_codex_row(m, upstreams.get(m.id)) for m in models) body: Final = codex_models_response_json( - rows, stock=MappingProxyType({m.slug: m for m in stock}), instructions=instructions, byte_limit=None + rows, + stock=MappingProxyType({m.slug: m for m in stock}), + instructions=instructions, + byte_limit=None, + alias_defaults="deployment", ) return body.json if body.listed else None @@ -678,12 +809,14 @@ def codex_model_sync_args( Codex has no env or inline equivalent of OPENCODE_CONFIG_CONTENT: the catalog must be a file, so it is written under $CODEX_HOME (default ~/.codex) and atomically replaced on every launch. The Codex at `binary` first lists its - own models, so the ones the proxy serves keep that Codex's entries, and then + own models, so the ones the proxy serves keep that Codex's entries, an alias + included when every unpaused deployment /v1/model/info lists for it routes + to the same OpenAI model and Codex knows that model, and then reads the file back once before it is handed over. The key never lands in the file. A failed fetch, read, listing, write or read-back is reported rather than raised: Codex still launches with its built-in catalog and takes a proxy model by name via -m, and a rejected file stays on disk to be looked - at. + at. An unavailable /v1/model/info only leaves every alias on the fallback entry. """ listing: Final = _fetch_model_listing(base_url, api_key, get=get) if isinstance(listing, ModelSyncSkipped): @@ -700,7 +833,8 @@ def codex_model_sync_args( stock: Final = _stock_codex_models(binary, base_env, run=run) if isinstance(stock, ModelSyncSkipped): return stock - catalog: Final = codex_model_catalog(listing, stock, instructions) + upstreams: Final = _fetch_model_upstreams(base_url, api_key, get=get) + catalog: Final = codex_model_catalog(listing, stock, instructions, upstreams) if catalog is None: return ModelSyncSkipped(f"{base_url.rstrip('/')}/v1/models lists no chat models") try: diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 9bb451495c8..fc29a6e3a08 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -724,7 +724,7 @@ def _router_defaults(llm_router: Router | None) -> Mapping[str, object]: return {name: value for name, value in _REQUEST_FIELDS.validate_python(configured).items() if value is not None} -def _counted_drop_policies( +def counted_drop_policies( data: Mapping[str, object], llm_router: Router | None, team_id: str | None ) -> tuple[DropPolicy, ...]: requested: Final = {**_router_defaults(llm_router), **data} @@ -740,7 +740,7 @@ def _counted_drop_policies( def resolve_counted_tool_fields( data: Mapping[str, object], llm_router: Router | None, team_id: str | None ) -> Mapping[str, object] | ProxyException: - resolved: Final = resolve_tool_fields(data, lambda: _counted_drop_policies(data, llm_router, team_id)) + resolved: Final = resolve_tool_fields(data, lambda: counted_drop_policies(data, llm_router, team_id)) if isinstance(resolved, MalformedField): return ProxyException( message=resolved.message, @@ -2581,7 +2581,6 @@ class ProxyBaseLLMRequestProcessing: # have mutated `self.data` in place, and the audit-trail snapshot taken in # add_litellm_data_to_request predates that mutation. refresh_proxy_server_request_body_snapshot(self.data, guardrails_applied=True) - verbose_proxy_logger.debug("receiving data: %s", self.data) if "messages" in self.data and self.data["messages"]: logging_obj.update_messages(self.data["messages"]) @@ -3207,7 +3206,7 @@ class ProxyBaseLLMRequestProcessing: request_data=self.data, request=request, ) - if route_type == "aresponses": + if route_type in ("aresponses", "aget_responses"): selected_data_generator = ( ProxyBaseLLMRequestProcessing._wrap_responses_stream_for_container_ownership( original_stream_response=response, @@ -4234,7 +4233,7 @@ class ProxyBaseLLMRequestProcessing: e = transformed_exception litellm_debug_info: Final = getattr(e, "litellm_debug_info", "") verbose_proxy_logger.debug( - "\033[1;31mAn error occurred: %s %s\n\n Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`", + "An error occurred: %s %s", e, litellm_debug_info, ) @@ -4597,7 +4596,7 @@ class ProxyBaseLLMRequestProcessing: if transformed_exception is not None: e = transformed_exception verbose_proxy_logger.debug( - "\x1b[1;31mAn error occurred: %s\n\n Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`", + "An error occurred: %s", e, ) diff --git a/litellm/proxy/common_utils/claude_code_marketplace_auth.py b/litellm/proxy/common_utils/claude_code_marketplace_auth.py new file mode 100644 index 00000000000..0cfb3061744 --- /dev/null +++ b/litellm/proxy/common_utils/claude_code_marketplace_auth.py @@ -0,0 +1,17 @@ +from typing import Final + +from pydantic import TypeAdapter, ValidationError + +_AUTH_REQUIRED_SETTING: Final[TypeAdapter[bool | None]] = TypeAdapter(bool | None) + + +def marketplace_auth_required() -> bool: + from litellm.proxy.proxy_server import general_settings_view + + try: + return ( + _AUTH_REQUIRED_SETTING.validate_python(general_settings_view().get("claude_code_marketplace_auth_required")) + is True + ) + except ValidationError: + return True diff --git a/litellm/proxy/common_utils/codex_model_catalog.py b/litellm/proxy/common_utils/codex_model_catalog.py index 9b42f99305e..47cd1db9f39 100644 --- a/litellm/proxy/common_utils/codex_model_catalog.py +++ b/litellm/proxy/common_utils/codex_model_catalog.py @@ -24,7 +24,7 @@ from functools import cache from itertools import accumulate, pairwise from pathlib import Path from types import MappingProxyType, NoneType -from typing import TYPE_CHECKING, Annotated, Final, Literal +from typing import TYPE_CHECKING, Annotated, Final, Literal, TypeAlias from pydantic import ConfigDict, StringConstraints, TypeAdapter, ValidationError @@ -102,6 +102,7 @@ class CodexFallbackModel(LiteLLMBaseModel): experimental_supported_tools: tuple[()] = () context_window: int | None base_instructions: str + supports_search_tool: bool = False class CodexStockUpgrade(LiteLLMBaseModel): @@ -110,6 +111,12 @@ class CodexStockUpgrade(LiteLLMBaseModel): model: str +class CodexReasoningLevel(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, extra="allow") + + effort: str + + class CodexStockModel(LiteLLMBaseModel): """One `ModelInfo` entry as Codex ships it or prints it from `codex debug models`. @@ -128,6 +135,9 @@ class CodexStockModel(LiteLLMBaseModel): upgrade: CodexStockUpgrade | None = None service_tiers: tuple[CodexServiceTier, ...] = () default_service_tier: str | None = None + supports_search_tool: bool = False + supported_reasoning_levels: tuple[CodexReasoningLevel, ...] = () + context_window: int | None = None class CodexStockCatalog(LiteLLMBaseModel): @@ -135,21 +145,24 @@ class CodexStockCatalog(LiteLLMBaseModel): CodexCatalogEntry = CodexFallbackModel | CodexStockModel +CodexAliasDefaults: TypeAlias = Literal["codex", "deployment"] @dataclass(frozen=True, slots=True) class CodexCatalogRow: """What the catalog needs to know about one listed model: its public id, the listing's mode and - input limit, the upstream model behind it, and the `model_info` values the operator set; - `service_tiers` holds one raw `model_info.service_tiers` value per deployment behind the id, None - where a deployment sets none.""" + input limit, the upstream model behind it, the reasoning effort its deployments pin, and the + `model_info` values the operator set; `service_tiers` holds one raw `model_info.service_tiers` + value per deployment behind the id, None where a deployment sets none.""" id: str mode: str | None = None max_input_tokens: int | None = None upstream_model: str | None = None + reasoning_effort: str | None = None display_name: str | None = None service_tiers: tuple[object, ...] = () + search_tool_support: tuple[bool | None, ...] = () @cache @@ -242,6 +255,12 @@ def configured_service_tiers( return offered +def offered_search_tool(per_deployment: Sequence[bool | None]) -> bool | None: + if False in per_deployment: + return False + return True if per_deployment and None not in per_deployment else None + + def _codex_can_drive(row: CodexCatalogRow) -> bool: return "*" not in row.id and (row.mode is None or row.mode in CODEX_CHAT_MODES) @@ -254,18 +273,49 @@ def _stock_for(row: CodexCatalogRow, stock: Mapping[str, CodexStockModel]) -> Co return upstream if upstream is not None else stock.get(row.upstream_model.partition("/")[2]) +def _alias_context_window(row: CodexCatalogRow, stock_model: CodexStockModel) -> Mapping[str, int]: + if row.max_input_tokens is None or stock_model.context_window is None: + return MappingProxyType({}) + return MappingProxyType({"context_window": min(stock_model.context_window, row.max_input_tokens)}) + + +def _alias_reasoning_level(row: CodexCatalogRow, stock_model: CodexStockModel) -> str | None: + efforts: Final = frozenset(level.effort for level in stock_model.supported_reasoning_levels) + return row.reasoning_effort if row.reasoning_effort in efforts else None + + +def _deployment_defaults(row: CodexCatalogRow, stock_model: CodexStockModel) -> Mapping[str, object]: + """What an alias's deployments decide in place of the defaults Codex picks for the model they route to: + no service tier, verbosity, upgrade nudge or startup promo of Codex's own, so what the deployment + pins (or OpenAI's API default) stays in force, the pinned reasoning effort when Codex offers it and + none otherwise, and a window no larger than the alias's own limit.""" + return MappingProxyType( + { + "service_tiers": (), + "default_service_tier": None, + "default_verbosity": None, + "upgrade": None, + "availability_nux": None, + "default_reasoning_level": _alias_reasoning_level(row, stock_model), + **_alias_context_window(row, stock_model), + } + ) + + def _entry( index: int, row: CodexCatalogRow, stock: Mapping[str, CodexStockModel], served: frozenset[str], instructions: str, + alias_defaults: CodexAliasDefaults, ) -> CodexCatalogEntry: stock_model: Final = _stock_for(row, stock) stock_tiers: Final = MappingProxyType( {tier.id: tier for tier in (stock_model.service_tiers if stock_model is not None else ())} ) configured: Final = configured_service_tiers(row.service_tiers, row.id, stock_tiers) + search_tool: Final = offered_search_tool(row.search_tool_support) if stock_model is None: return CodexFallbackModel( slug=row.id, @@ -274,11 +324,13 @@ def _entry( service_tiers=configured if configured is not None else (), context_window=row.max_input_tokens, base_instructions=instructions, + supports_search_tool=search_tool is True, ) upgrade: Final = ( stock_model.upgrade if stock_model.upgrade is not None and stock_model.upgrade.model in served else None ) - stock_name: Final = stock_model.display_name if row.id == stock_model.slug else row.id + is_alias: Final = row.id != stock_model.slug + stock_name: Final = row.id if is_alias else stock_model.display_name listing: Final = { "slug": row.id, "display_name": row.display_name if row.display_name is not None else stock_name, @@ -286,6 +338,8 @@ def _entry( "visibility": "list", "supported_in_api": True, "upgrade": upgrade, + **({} if search_tool is None else {"supports_search_tool": search_tool}), + **(_deployment_defaults(row, stock_model) if is_alias and alias_defaults == "deployment" else {}), } if configured is None: return stock_model.model_copy(update=listing) @@ -351,6 +405,7 @@ def codex_models_response_json( stock: Mapping[str, CodexStockModel] | None = None, instructions: str | None = None, byte_limit: int | None = CODEX_CATALOG_BYTE_LIMIT, + alias_defaults: CodexAliasDefaults = "codex", ) -> CodexCatalogBody: """Codex's `ModelsResponse` body for `rows`, in listing order, cut to Codex's byte limit. @@ -359,13 +414,18 @@ def codex_models_response_json( byte limit silently and keeps its bundled one, so the body holds only the entries that fit, the ones offering a service tier taken first and the rest in listing order, an entry too large for the bytes left passed over, every kept entry at its listing position; `byte_limit=None` - keeps every entry. + keeps every entry. A row listed under another name than the Codex model its `upstream_model` + names (an alias) takes that model's entry; `alias_defaults="deployment"` drops the defaults + Codex picks for that model in favor of what the alias's deployments pin, while `"codex"` keeps + them, as Codex would use them for the model's own name. """ known: Final = bundled_codex_models() if stock is None else stock prompt: Final = codex_base_instructions() if instructions is None else instructions drivable: Final = tuple(row for row in rows if _codex_can_drive(row)) served: Final = frozenset(row.id for row in drivable) - entries: Final = tuple(_entry(index, row, known, served, prompt) for index, row in enumerate(drivable)) + entries: Final = tuple( + _entry(index, row, known, served, prompt, alias_defaults) for index, row in enumerate(drivable) + ) serialized: Final = tuple(_serialized(entry) for entry in entries) kept: Final = _kept_positions(entries, serialized, byte_limit) return CodexCatalogBody( @@ -385,6 +445,7 @@ def _catalog_row( upstream_model=llm_router.get_routable_upstream_model(lookup_id, team_id) if llm_router is not None else None, display_name=llm_router.get_configured_display_name(label_id) if llm_router is not None else None, service_tiers=llm_router.get_configured_service_tiers(lookup_id, team_id) if llm_router is not None else (), + search_tool_support=llm_router.get_search_tool_support(lookup_id, team_id) if llm_router is not None else (), ) diff --git a/litellm/proxy/common_utils/html_forms/native_client_consent.py b/litellm/proxy/common_utils/html_forms/native_client_consent.py index dac92c4e787..aeadc93221c 100644 --- a/litellm/proxy/common_utils/html_forms/native_client_consent.py +++ b/litellm/proxy/common_utils/html_forms/native_client_consent.py @@ -62,7 +62,7 @@ button {{ flex: 1; padding: 10px; border-radius: 6px; font-size: 15px; cursor: p

Approving issues it a personal credential that expires within {CLI_JWT_EXPIRATION_HOURS} hours. lite logout stops it from being renewed. Only approve if you started this sign-in yourself.

-{_team_field(teams)} +{team_select_field(teams)}
@@ -74,7 +74,7 @@ button {{ flex: 1; padding: 10px; border-radius: 6px; font-size: 15px; cursor: p """ -def _team_field(teams: Sequence[tuple[str, str]]) -> str: +def team_select_field(teams: Sequence[tuple[str, str]], *, require_choice: bool = False) -> str: if not teams: return "" if len(teams) == 1: @@ -83,9 +83,12 @@ def _team_field(teams: Sequence[tuple[str, str]]) -> str: f'' f"

Requests are attributed to team {escape(team_label)}.

" ) + placeholder: Final = '' if require_choice else "" options: Final = "".join( f'' for team_id, team_label in teams ) + required: Final = " required" if require_choice else "" return ( - f'' + '' + f'' ) diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index c4c3f0ad634..56cd14d987d 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -845,5 +845,3 @@ def _add_vector_store_id_from_path(request_data: dict, request: Request) -> None "populate_request_with_path_params: Updated request_data with vector_store_ids=%s", request_data.get("vector_store_ids"), ) - else: - verbose_proxy_logger.debug("populate_request_with_path_params: No vector_store_id present in path=%s", path) diff --git a/litellm/proxy/common_utils/openai_error_payload.py b/litellm/proxy/common_utils/openai_error_payload.py index 2f01dbe11f7..3c91e87ee51 100644 --- a/litellm/proxy/common_utils/openai_error_payload.py +++ b/litellm/proxy/common_utils/openai_error_payload.py @@ -2,14 +2,17 @@ ``type`` is a required string and ``param`` is nullable, neither of which the literal string ``"None"`` satisfies.""" +import sys from collections.abc import Mapping from types import MappingProxyType from typing import Final -from fastapi import status +from fastapi import HTTPException, status from litellm.constants import STRINGIFIED_NONE -from litellm.proxy._types import ProxyException +from litellm.proxy._types import ProxyErrorTypes, ProxyException, proxy_exception_status_code +from litellm.proxy.db.db_lookup_gate import DBLookupDeadlineExceeded +from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler, exception_chain LITELLM_CALL_ID_HEADER: Final = "x-litellm-call-id" @@ -21,6 +24,19 @@ _OPENAI_ERROR_TYPE_BY_STATUS: Final[Mapping[int, str]] = MappingProxyType( } ) +_AUTH_FAILURE_STATUS_CODES: Final = frozenset({status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN}) + +_FAILOVER_SQLSTATES: Final = frozenset({"25006", "57P01", "57P02", "57P03"}) + +_FAILOVER_REFUSALS: Final = ( + "is not currently accepting connections", + "the database system is starting up", + "the database system is shutting down", + "the database system is in recovery mode", + "the database system is not yet accepting connections", + "terminating connection due to administrator command", +) + def attribute_of(value: object, name: str, default: object = None) -> object: return getattr(value, name, default) @@ -61,6 +77,82 @@ def openai_error_param(exc: object) -> str | None: return carried if isinstance(carried, str) and carried != STRINGIFIED_NONE else None +def proxy_exception_for(exc: Exception, default_status_code: int) -> ProxyException: + match exc: + case ProxyException(): + return exc + case _ if _is_database_outage(exc): + return ProxyException( + message=PrismaDBExceptionHandler.database_unavailable_message(exc), + type=ProxyErrorTypes.no_db_connection.value, + param=None, + code=status.HTTP_503_SERVICE_UNAVAILABLE, + ) + case HTTPException(status_code=status_code, detail=detail): + return _openai_shaped_proxy_exception(exc, str(detail), status_code) + case _: + return _openai_shaped_proxy_exception( + exc, str(exc) or type(exc).__name__, error_status_code(exc, default_status_code) + ) + + +def _is_database_outage(exc: Exception) -> bool: + return any(map(_is_database_outage_link, exception_chain(exc))) + + +def _is_database_outage_link(exc: BaseException) -> bool: + match exc: + case DBLookupDeadlineExceeded(): + return True + case Exception() if _prisma_is_loaded(): + return _is_prisma_outage(exc) + case _: + return False + + +def _prisma_is_loaded() -> bool: + return sys.modules.get("prisma") is not None + + +def _is_prisma_outage(exc: Exception) -> bool: + if PrismaDBExceptionHandler.is_prisma_engine_internal_error(exc): + return True + if type(exc).__module__.partition(".")[0] != "prisma": + return False + return PrismaDBExceptionHandler.is_database_service_unavailable_error(exc) or _is_failover_refusal(exc) + + +def _is_failover_refusal(exc: Exception) -> bool: + if PrismaDBExceptionHandler.is_read_only_transaction_error(exc): + return True + if PrismaDBExceptionHandler.postgres_sqlstate(exc) in _FAILOVER_SQLSTATES: + return True + message: Final = str(exc).lower() + return any(refusal in message for refusal in _FAILOVER_REFUSALS) + + +def _openai_shaped_proxy_exception(exc: Exception, message: str, status_code: int) -> ProxyException: + answered_status: Final = int(proxy_exception_status_code(message, str(status_code))) + return ProxyException( + message=message, + type=_catch_all_error_type(exc, answered_status), + param=openai_error_param(exc), + code=answered_status, + ) + + +def _catch_all_error_type(exc: Exception, status_code: int) -> str: + if _rejects_the_caller(exc, status_code): + return ProxyErrorTypes.auth_error.value + return openai_error_type(exc, status_code) + + +def _rejects_the_caller(exc: Exception, status_code: int) -> bool: + raised_as_http_answer: Final = isinstance(exc, HTTPException) + status_from_upstream: Final = error_status_code(exc, 0) == status_code + return status_code in _AUTH_FAILURE_STATUS_CODES and (raised_as_http_answer or not status_from_upstream) + + def litellm_call_id_headers(litellm_call_id: str | None) -> dict[str, str] | None: # mutable-ok: ProxyException.headers if litellm_call_id is None: return None diff --git a/litellm/proxy/credential_endpoints/endpoints.py b/litellm/proxy/credential_endpoints/endpoints.py index d20b5602186..57c7a753d71 100644 --- a/litellm/proxy/credential_endpoints/endpoints.py +++ b/litellm/proxy/credential_endpoints/endpoints.py @@ -5,6 +5,7 @@ CRUD endpoints for storing reusable credentials. import time import unicodedata from collections.abc import Mapping +from types import MappingProxyType from typing import ( Annotated, Final, @@ -23,7 +24,7 @@ from litellm.constants import ( GITHUB_COPILOT_PER_USER_AUTH_TYPE, ) from litellm.litellm_core_utils.credential_accessor import CredentialAccessor -from litellm.litellm_core_utils.litellm_logging import get_masked_values +from litellm.litellm_core_utils.litellm_logging import get_masked_values, has_sensitive_keyword from litellm.llms.anthropic.wif import ( ExportedJwks, NotAnInternalIssuerCredential, @@ -83,6 +84,11 @@ _CREDENTIAL_DICT_ADAPTER: Final = TypeAdapter(dict[str, object]) _GITHUB_COPILOT_PROVIDER: Final = "github_copilot" _DISPLAY_NAME_MAX_LENGTH: Final = 255 _DISPLAY_NAME_FORBIDDEN_CATEGORIES: Final = frozenset({"Cc", "Cf", "Cs", "Zl", "Zp"}) +_MASK_UNMASKED_LENGTH: Final = 4 +_MASK_ASTERISKS: Final = 4 +_SHORT_SECRET_MASK: Final = "*****" +_MASKED_READ_BACK_DEPTH: Final = 20 +_NO_FIELDS: Final[Mapping[str, object]] = MappingProxyType({}) def _normalized_github_copilot_values( @@ -395,7 +401,14 @@ async def get_credentials( """ try: masked_credentials: Final = [ - _credential_view(credential, get_masked_values(credential.credential_values)) + _credential_view( + credential, + get_masked_values( + credential.credential_values, + unmasked_length=_MASK_UNMASKED_LENGTH, + number_of_asterisks=_MASK_ASTERISKS, + ), + ) for credential in litellm.credential_list ] return {"success": True, "credentials": masked_credentials} @@ -423,7 +436,11 @@ async def get_credential_by_name( if credential.credential_name == credential_name: return _credential_view( credential, - get_masked_values(credential.credential_values, unmasked_length=4, number_of_asterisks=4), + get_masked_values( + credential.credential_values, + unmasked_length=_MASK_UNMASKED_LENGTH, + number_of_asterisks=_MASK_ASTERISKS, + ), ) raise HTTPException( status_code=404, @@ -521,8 +538,8 @@ async def get_credential_by_model( raise HTTPException(status_code=404, detail="Model not found") masked_credential_values: Final = get_masked_values( credential_values, - unmasked_length=4, - number_of_asterisks=4, + unmasked_length=_MASK_UNMASKED_LENGTH, + number_of_asterisks=_MASK_ASTERISKS, ) credential: Final = CredentialItem( credential_name=f"{model.model_name}-credential-{model_id}", @@ -908,6 +925,48 @@ def update_db_credential( return merged_credential +def _is_masked_read_back(value: object) -> bool: + if not isinstance(value, str): + return False + if value == _SHORT_SECRET_MASK: + return True + shown: Final = _MASK_UNMASKED_LENGTH // 2 + return len(value) == _MASK_UNMASKED_LENGTH + _MASK_ASTERISKS and value[shown:-shown] == "*" * _MASK_ASTERISKS + + +def _as_fields(value: object) -> Mapping[str, object] | None: + try: + return _CREDENTIAL_DICT_ADAPTER.validate_python(value, strict=True) + except ValidationError: + return None + + +def _kept_value(key: str, value: object, stored: Mapping[str, object], depth: int) -> tuple[str, object] | None: + if not has_sensitive_keyword(key): + return key, value + fields: Final = _as_fields(value) + if fields is not None and depth < _MASKED_READ_BACK_DEPTH: + return key, _without_masked_read_backs(fields, _as_fields(stored.get(key)) or _NO_FIELDS, depth + 1) + if not _is_masked_read_back(value): + return key, value + if key in stored: + return key, stored[key] + return None + + +def _without_masked_read_backs( + requested: Mapping[str, object], stored: Mapping[str, object], depth: int = 0 +) -> Mapping[str, object]: + kept: Final = (_kept_value(key, value, stored, depth) for key, value in requested.items()) + return _CREDENTIAL_DICT_ADAPTER.validate_python( + MappingProxyType(dict(entry for entry in kept if entry is not None)) + ) + + +def _stored_objects(stored_values: Mapping[str, object]) -> Mapping[str, object]: + return MappingProxyType({key: value for key, value in stored_values.items() if _as_fields(value) is not None}) + + @router.patch( "/credentials/{credential_name:path}", dependencies=[Depends(user_api_key_auth)], @@ -936,12 +995,12 @@ async def update_credential( ) requested_display_name: Final = _normalized_display_name(credential.display_name) _reject_overlapping_credential_values(credential) - incoming_values: Final = _CREDENTIAL_DICT_ADAPTER.validate_python( + requested_values: Final = _CREDENTIAL_DICT_ADAPTER.validate_python( _resolve_deployment_credentials(llm_router, credential.model_id) if credential.model_id else credential.credential_values or {} ) - _reject_non_admin_wif_fields(_incoming_wif_fields(incoming_values, credential), user_api_key_dict) + _reject_non_admin_wif_fields(_incoming_wif_fields(requested_values, credential), user_api_key_dict) if prisma_client is None: raise HTTPException( status_code=500, @@ -952,19 +1011,20 @@ async def update_credential( if db_credential is None: raise _not_found_unless_config_defined(credential_name, "Credential not found in DB.") _reject_non_admin_wif_fields(_stored_wif_fields(db_credential), user_api_key_dict) + decrypted_stored_credential: Final = decrypted_credential(db_credential) + decrypted_values: Final = _CREDENTIAL_DICT_ADAPTER.validate_python( + decrypted_stored_credential.credential_values + ) + incoming_values: Final = _without_masked_read_backs(requested_values, _stored_objects(decrypted_values)) patch: Final = CredentialItem( credential_name=credential_name, display_name=( requested_display_name if "display_name" in credential.model_fields_set else db_credential.display_name ), credential_info=_CREDENTIAL_DICT_ADAPTER.validate_python(credential.credential_info), - credential_values=incoming_values, + credential_values=_CREDENTIAL_DICT_ADAPTER.validate_python(incoming_values), credential_values_to_delete=credential.credential_values_to_delete, ) - decrypted_stored_credential: Final = decrypted_credential(db_credential) - decrypted_values: Final = _CREDENTIAL_DICT_ADAPTER.validate_python( - decrypted_stored_credential.credential_values - ) stored_info: Final = _CREDENTIAL_DICT_ADAPTER.validate_python( db_credential.credential_info # pyright: ignore[reportUnknownMemberType] # untyped credential field ) diff --git a/litellm/proxy/db/check_migration.py b/litellm/proxy/db/check_migration.py index 83cc4501466..d126f112f90 100644 --- a/litellm/proxy/db/check_migration.py +++ b/litellm/proxy/db/check_migration.py @@ -4,7 +4,7 @@ import os import subprocess from typing import Final -from litellm._logging import print_startup_message, verbose_logger +from litellm._logging import print_startup_message, print_startup_warning, verbose_logger def extract_sql_commands(diff_output: str) -> list[str]: @@ -63,7 +63,7 @@ def check_prisma_schema_diff_helper(db_url: str) -> tuple[bool, list[str]]: run_prisma, ) except ImportError as e: - print_startup_message( + print_startup_warning( f"Skipping the migration diff: litellm-proxy-extras has no Prisma runner. Error: {e}", ) return False, [] @@ -96,14 +96,14 @@ def check_prisma_schema_diff_helper(db_url: str) -> tuple[bool, list[str]]: else: return False, [] except subprocess.TimeoutExpired: - print_startup_message( + print_startup_warning( f"Timed out after {timeout}s generating the migration diff. " f"Raise {PRISMA_COMMAND_TIMEOUT_ENV_VAR} if this database needs longer.", ) return False, [] except subprocess.CalledProcessError as e: error_message: Final = f"Failed to generate migration diff. Error: {e.stderr}" - print_startup_message(error_message) + print_startup_warning(error_message) return False, [] diff --git a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py index 7f5efcdaec3..170eb450339 100644 --- a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py +++ b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py @@ -421,7 +421,10 @@ class SpendLogCleanup: general_settings=None, redis_cache: RedisCache | None = None, partition_manager: SpendLogsPartitionManager | None = None, + *, + clock: Callable[[], float] = time.monotonic, ): + self._clock = clock self.retention_seconds: int | None = None self.partition_manager = partition_manager or SpendLogsPartitionManager() from litellm.proxy.proxy_server import general_settings as default_settings @@ -586,7 +589,7 @@ class SpendLogCleanup: async def _probe_storage(self, prisma_client: PrismaClient, deadline: float) -> StorageProbe | None: """Read what the spend-log tables weigh from the catalog, or None when the probe fails or has no budget.""" - if time.monotonic() >= deadline: + if self._clock() >= deadline: return None timeout_ms: Final = self._timeout_ms(deadline) try: @@ -624,7 +627,7 @@ class SpendLogCleanup: stale.allocated_bytes, _ANALYZE_SQL, ) - if time.monotonic() >= deadline: + if self._clock() >= deadline: return None timeout_ms: Final = self._timeout_ms(deadline) try: @@ -653,7 +656,7 @@ class SpendLogCleanup: async def _load_size_state(self, prisma_client: PrismaClient, deadline: float) -> SizeCleanupState | None: """The reclaimable count saved by the last check, or None on the first check or when it cannot be read.""" - if time.monotonic() >= deadline: + if self._clock() >= deadline: return None timeout_ms: Final = self._timeout_ms(deadline) try: @@ -686,7 +689,7 @@ class SpendLogCleanup: the tool-index rows that expire with their spend logs. When fewer rows than that remain, the step is everything up to now. """ - if time.monotonic() >= deadline: + if self._clock() >= deadline: return None timeout_ms: Final = self._timeout_ms(deadline) async with db_span("size_step_cutoff", "LiteLLM_SpendLogs"), bounded_tx(prisma_client, timeout_ms) as tx: @@ -718,11 +721,11 @@ class SpendLogCleanup: Interpolating this into SQL is safe by construction: an int cannot carry SQL, and SET does not accept a bind parameter. """ - remaining_ms: Final = int((deadline - time.monotonic()) * 1000) + remaining_ms: Final = int((deadline - self._clock()) * 1000) return max(1, min(int(self.batch_timeout_seconds * 1000), remaining_ms)) @staticmethod - def _group_deadline(overall_deadline: float, groups_remaining: int) -> float: + def _group_deadline(overall_deadline: float, groups_remaining: int, now: float) -> float: """ Give each pending cleanup group an equal share of the time left. @@ -731,10 +734,9 @@ class SpendLogCleanup: """ if groups_remaining == 1: return overall_deadline - current_time: Final = time.monotonic() - if current_time >= overall_deadline: + if now >= overall_deadline: return overall_deadline - return current_time + (overall_deadline - current_time) / groups_remaining + return now + (overall_deadline - now) / groups_remaining def _remaining_timeout_ms(self, deadline: float) -> RemainingTimeoutMs: """ @@ -748,7 +750,7 @@ class SpendLogCleanup: """ def remaining() -> int | None: - return None if time.monotonic() >= deadline else self._timeout_ms(deadline) + return None if self._clock() >= deadline else self._timeout_ms(deadline) return remaining @@ -841,7 +843,7 @@ class SpendLogCleanup: run_count = 0 consecutive_failures = 0 while True: - if time.monotonic() >= deadline: + if self._clock() >= deadline: verbose_proxy_logger.info( "Run budget exhausted during %s cleanup after %d rows; the next run resumes from here", table_name, @@ -858,13 +860,13 @@ class SpendLogCleanup: prisma_client, cutoff_date, table_name, time_column, total_deleted, "batch_cap_reached", deadline ) # Find rows and delete them in one go without fetching to application - batch_started_at = time.monotonic() + batch_started_at = self._clock() try: batch_result = await self._execute_delete_batch( prisma_client, delete_sql, cutoff_date, table_name, deadline ) except Exception as batch_exc: - if time.monotonic() >= deadline: + if self._clock() >= deadline: # The statement timeout was clamped to the budget that was # left, so this batch was cancelled by the deadline itself. # That is the bound working, not a database fault, and @@ -920,7 +922,7 @@ class SpendLogCleanup: consecutive_failures = 0 deleted_count = batch_result - SpendLogCleanupMetrics.record_batch(table_name, deleted_count, time.monotonic() - batch_started_at) + SpendLogCleanupMetrics.record_batch(table_name, deleted_count, self._clock() - batch_started_at) verbose_proxy_logger.info("Deleted %s %s rows in this batch", deleted_count, table_name) if deleted_count == 0: @@ -957,7 +959,7 @@ class SpendLogCleanup: operator the backlog was not drained; the gauge simply keeps its value from the last run that finished inside its budget. """ - if time.monotonic() >= deadline: + if self._clock() >= deadline: return TableCleanupResult(table_name=table_name, rows_deleted=rows_deleted, stop_reason=stop_reason) remaining: Final = await self._count_remaining(prisma_client, cutoff_date, table_name, time_column, deadline) if remaining is not None: @@ -1070,7 +1072,7 @@ class SpendLogCleanup: # behind any long-running reader for as long as that reader lives, which # is the one way this job could still outlast its budget without bound. remaining_timeout_ms: Final = self._remaining_timeout_ms(deadline) - if time.monotonic() >= deadline: + if self._clock() >= deadline: verbose_proxy_logger.info("Run budget already spent, skipping partition maintenance this run") elif self.general_settings.get( "use_spend_logs_partitioning", False @@ -1082,7 +1084,7 @@ class SpendLogCleanup: verbose_proxy_logger.info("Dropped %d expired spend-log partitions: %s", len(dropped), dropped) logs_result: Final = await self._delete_old_logs( - prisma_client, cutoff_date, self._group_deadline(deadline, groups_remaining=2) + prisma_client, cutoff_date, self._group_deadline(deadline, groups_remaining=2, now=self._clock()) ) verbose_proxy_logger.info("Deleted %s logs", logs_result.rows_deleted) @@ -1210,7 +1212,7 @@ class SpendLogCleanup: self, prisma_client: PrismaClient, deadline: float, still_over_target: Callable[[], Awaitable[bool]] ) -> tuple[str, ...]: remaining_timeout_ms: Final = self._remaining_timeout_ms(deadline) - if time.monotonic() >= deadline or not self._setting("use_spend_logs_partitioning"): + if self._clock() >= deadline or not self._setting("use_spend_logs_partitioning"): return () if not await self.partition_manager.is_partitioned(prisma_client, remaining_timeout_ms): return () @@ -1257,7 +1259,7 @@ class SpendLogCleanup: needed: Final = rows_still_needed(state.logs_deleted + state.index_deleted) if needed <= 0: return state.stopped("target_reached") - if time.monotonic() >= deadline: + if self._clock() >= deadline: return state.stopped("budget_exhausted") if state.batches_left <= 0: return state.stopped("batch_cap_reached") @@ -1318,11 +1320,11 @@ class SpendLogCleanup: except Exception: # noqa: BLE001 # retained observations are retried by the next cleanup job verbose_proxy_logger.warning("Auto-router baseline retention remains pending") sessions_result: Final = await self._delete_old_autorouter_session_rows( - prisma_client, session_cutoff, self._group_deadline(deadline, 3) + prisma_client, session_cutoff, self._group_deadline(deadline, 3, self._clock()) ) verbose_proxy_logger.info("Deleted %s expired auto-router session rollup rows", sessions_result.rows_deleted) user_sessions_result: Final = await self._delete_old_autorouter_user_session_rows( - prisma_client, session_cutoff, self._group_deadline(deadline, 2) + prisma_client, session_cutoff, self._group_deadline(deadline, 2, self._clock()) ) verbose_proxy_logger.info( "Deleted %s expired auto-router user session rollup rows", user_sessions_result.rows_deleted @@ -1443,7 +1445,7 @@ class SpendLogCleanup: async def _run_passes(self, prisma_client: PrismaClient, age_rules: bool, size_rule: bool) -> None: lock_acquired = False - run_started_at: Final = time.monotonic() + run_started_at: Final = self._clock() progress: Final = _RunProgress() progress_token: Final = _run_progress.set(progress) try: @@ -1493,7 +1495,7 @@ class SpendLogCleanup: SpendLogCleanupMetrics.record_run("skipped_locked") return - deadline: Final = time.monotonic() + self.run_budget_seconds + deadline: Final = self._clock() + self.run_budget_seconds size_group: Final = int(active_size_rule is not None) configured_group_count: Final = ( int(delete_spend_logs and self.retention_seconds is not None) @@ -1506,7 +1508,7 @@ class SpendLogCleanup: spend_log_results: Final = ( await self._clean_spend_log_tables( prisma_client, - self._group_deadline(deadline, configured_group_count), + self._group_deadline(deadline, configured_group_count, self._clock()), ) if delete_spend_logs and self.retention_seconds is not None else () @@ -1526,7 +1528,7 @@ class SpendLogCleanup: await self._clean_session_rollup( prisma_client, autorouter_retention_seconds, - self._group_deadline(deadline, remaining_groups_after_spend_logs), + self._group_deadline(deadline, remaining_groups_after_spend_logs, self._clock()), ) if autorouter_retention_seconds is not None else () @@ -1535,7 +1537,7 @@ class SpendLogCleanup: await self._clean_health_checks( prisma_client, health_check_retention_seconds, - self._group_deadline(deadline, remaining_groups_after_sessions), + self._group_deadline(deadline, remaining_groups_after_sessions, self._clock()), ) if health_check_retention_seconds is not None else () @@ -1544,7 +1546,7 @@ class SpendLogCleanup: await self._clean_daily_tag_spend( prisma_client, daily_tag_spend_retention_seconds, - self._group_deadline(deadline, 1 + size_group), + self._group_deadline(deadline, 1 + size_group, self._clock()), ) if daily_tag_spend_retention_seconds is not None else () @@ -1561,12 +1563,12 @@ class SpendLogCleanup: ) outcome: Final = self._run_outcome(results) SpendLogCleanupMetrics.record_run(outcome) - self._log_run_summary(outcome, results, time.monotonic() - run_started_at, size_report) + self._log_run_summary(outcome, results, self._clock() - run_started_at, size_report) except asyncio.CancelledError: verbose_proxy_logger.error( "Spend log cleanup cancelled after %.2fs (rows_deleted=%d, batches=%d); the next run resumes from here", - time.monotonic() - run_started_at, + self._clock() - run_started_at, progress.rows_deleted, progress.batches, ) diff --git a/litellm/proxy/db/exception_handler.py b/litellm/proxy/db/exception_handler.py index 5dfcaa4b2f8..3f7ea334a99 100644 --- a/litellm/proxy/db/exception_handler.py +++ b/litellm/proxy/db/exception_handler.py @@ -36,7 +36,7 @@ _LOCK_TIMEOUT_SQLSTATE: Final = "55P03" _LOCK_TIMEOUT_PHRASE: Final = "canceling statement due to lock timeout" -def _exception_chain(e: BaseException) -> Iterator[BaseException]: +def exception_chain(e: BaseException) -> Iterator[BaseException]: current = e # rebind-ok: advances one link per iteration of the bounded walk for _ in range(_MAX_EXCEPTION_CHAIN_DEPTH): yield current @@ -49,7 +49,7 @@ def _exception_chain(e: BaseException) -> Iterator[BaseException]: def _database_service_unavailable_errors(e: BaseException) -> tuple[Exception, ...]: return tuple( link - for link in _exception_chain(e) + for link in exception_chain(e) if isinstance(link, Exception) and PrismaDBExceptionHandler.is_database_service_unavailable_error(link) ) diff --git a/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py b/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py index 792ef8a3f8c..36f424507d7 100644 --- a/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py +++ b/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py @@ -4,6 +4,7 @@ from typing import Final from fastapi import APIRouter +from litellm.proxy.common_utils.claude_code_marketplace_auth import marketplace_auth_required from litellm.proxy.common_utils.html_forms.default_credentials_hint import should_hide_default_credentials_hint from litellm.proxy.mcp_server.stdio_gate import is_mcp_stdio_enabled from litellm.types.proxy.discovery_endpoints.ui_discovery_endpoints import ( @@ -51,4 +52,5 @@ async def get_ui_config(): workers=proxy_config.worker_registry if is_control_plane else [], mcp_stdio_enabled=is_mcp_stdio_enabled(), use_si_terminology=use_si_terminology, + claude_code_marketplace_auth_required=marketplace_auth_required(), ) diff --git a/litellm/proxy/example_config_yaml/custom_callbacks.py b/litellm/proxy/example_config_yaml/custom_callbacks.py index 5ae55060930..d11e27e0718 100644 --- a/litellm/proxy/example_config_yaml/custom_callbacks.py +++ b/litellm/proxy/example_config_yaml/custom_callbacks.py @@ -28,7 +28,7 @@ class MyCustomHandler(CustomLogger): def __init__(self): blue_color_code: Final = "\033[94m" reset_color_code: Final = "\033[0m" - print_verbose(f"{blue_color_code}Initialized LiteLLM custom logger") + print_verbose(f"{blue_color_code}Initialized LiteLLM custom logger{reset_color_code}") try: print_verbose("Logger Initialized with following methods:") methods: Final = [ @@ -40,7 +40,6 @@ class MyCustomHandler(CustomLogger): # Pretty print_verbose the methods for method in methods: print_verbose(f" - {method}") - print_verbose(f"{reset_color_code}") except Exception: pass diff --git a/litellm/proxy/google_endpoints/endpoints.py b/litellm/proxy/google_endpoints/endpoints.py index 54976857584..c018f071b3c 100644 --- a/litellm/proxy/google_endpoints/endpoints.py +++ b/litellm/proxy/google_endpoints/endpoints.py @@ -1,11 +1,11 @@ from collections.abc import Mapping from itertools import chain from types import MappingProxyType -from typing import Final +from typing import Annotated, Final from fastapi import APIRouter, Depends, HTTPException, Request, Response from fastapi.responses import JSONResponse, ORJSONResponse -from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError +from pydantic import BaseModel, BeforeValidator, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError from litellm.litellm_core_utils.json_validation_rule import normalize_json_schema_types from litellm.proxy._types import * @@ -27,14 +27,67 @@ from litellm.proxy.common_utils.token_count_settings import count_source_fields from litellm.types.llms.vertex_ai import TokenCountDetailsResponse +def _google_repeated_field(value: JsonValue) -> JsonValue: + match value: + case None: + return [] + case dict(): + return [value] + case list(): + return [item for item in value if item is not None] + case _: + return value + + +def _google_content(value: JsonValue) -> JsonValue: + match value: + case {"parts": parts}: + return {**value, "parts": _google_repeated_field(parts)} + case _: + return value + + +def _google_contents(value: JsonValue) -> JsonValue: + match _google_repeated_field(value): + case list() as contents: + return [_google_content(content) for content in contents] + case other: + return other + + +_GOOGLE_PART_TEXT_AND_METADATA_FIELDS: Final = frozenset( + {"text", "thought", "thoughtSignature", "thought_signature", "partMetadata", "part_metadata"} +) + + +def _google_part_has_non_text_content(part: Mapping[str, JsonValue]) -> bool: + return any(field is not None for key, field in part.items() if key not in _GOOGLE_PART_TEXT_AND_METADATA_FIELDS) + + +def _google_system_part(value: JsonValue) -> JsonValue: + match value: + case dict() if value.get("text") is None and _google_part_has_non_text_content(value): + return {key: field for key, field in value.items() if key != "text"} + case dict(): + return {"text": None, **value} + case _: + return value + + +_GOOGLE_REPEATED_FIELD: Final = BeforeValidator(_google_repeated_field) + + class _GoogleCountTokensTextPart(BaseModel): model_config = ConfigDict(frozen=True) - text: str + text: str | None class _GoogleCountTokensSystemInstruction(BaseModel): model_config = ConfigDict(frozen=True) - parts: tuple[_GoogleCountTokensTextPart, ...] + parts: Annotated[ + tuple[Annotated[_GoogleCountTokensTextPart, BeforeValidator(_google_system_part)], ...], + _GOOGLE_REPEATED_FIELD, + ] = () class _GoogleCountTokensFunctionDeclaration(BaseModel): @@ -47,16 +100,16 @@ class _GoogleCountTokensFunctionDeclaration(BaseModel): class _GoogleCountTokensTool(BaseModel): model_config = ConfigDict(frozen=True, populate_by_name=True) - function_declarations: tuple[_GoogleCountTokensFunctionDeclaration, ...] = Field( + function_declarations: Annotated[tuple[_GoogleCountTokensFunctionDeclaration, ...], _GOOGLE_REPEATED_FIELD] = Field( default=(), alias="functionDeclarations" ) class _GoogleCountTokensRequest(BaseModel): model_config = ConfigDict(frozen=True, extra="ignore", populate_by_name=True) - contents: tuple[Mapping[str, JsonValue], ...] = () + contents: Annotated[tuple[Mapping[str, JsonValue], ...], BeforeValidator(_google_contents)] = () system_instruction: _GoogleCountTokensSystemInstruction | None = Field(default=None, alias="systemInstruction") - tools: tuple[_GoogleCountTokensTool, ...] = () + tools: Annotated[tuple[_GoogleCountTokensTool, ...], _GOOGLE_REPEATED_FIELD] = () router: Final = APIRouter( @@ -112,7 +165,11 @@ def _google_count_tokens_request(model_name: str, data: object) -> TokenCountReq GoogleGenAIAdapter().translate_generate_content_to_completion(model_name, contents).get("messages", []) ) system: Final = ( - [{"type": "text", "text": part.text} for part in validated_data.system_instruction.parts] + [ + {"type": "text", "text": part.text} + for part in validated_data.system_instruction.parts + if part.text is not None + ] if validated_data.system_instruction is not None else None ) diff --git a/litellm/proxy/guardrails/_content_utils.py b/litellm/proxy/guardrails/_content_utils.py index 9bea22acd71..51934c99db6 100644 --- a/litellm/proxy/guardrails/_content_utils.py +++ b/litellm/proxy/guardrails/_content_utils.py @@ -62,6 +62,20 @@ def is_non_conversational_call_type(call_type: str) -> bool: return call_type in NON_CONVERSATIONAL_CALL_TYPES +_PASSTHROUGH_EMBEDDINGS_ENDPOINT: Final = re.compile(r"/?(?:[\w-]+/)*embeddings/?") + + +def is_non_conversational_request(call_type: str, data: Mapping[str, object]) -> bool: + if is_non_conversational_call_type(call_type): + return True + endpoint: Final = data.get("endpoint") + return ( + call_type == "allm_passthrough_route" + and isinstance(endpoint, str) + and _PASSTHROUGH_EMBEDDINGS_ENDPOINT.fullmatch(endpoint) is not None + ) + + TEXT_PART_TYPES: Final[frozenset[str]] = frozenset( {"text", "input_text", "output_text", "summary_text", "reasoning_text"} ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index adda465ad95..9fa37e1827e 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -49,6 +49,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, bedrock_bearer_token, run_aws_signing from litellm.llms.bedrock.guardrail_attachments import find_request_attachments from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, get_async_httpx_client, httpxSpecialProvider, ) @@ -286,9 +287,12 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): streaming_sampling_rate: int | None = None, streaming_end_of_stream_only: bool | None = None, streaming_buffer_release_on_scan: bool | None = None, + async_handler: AsyncHTTPHandler | None = None, **kwargs, ): - self.async_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.GuardrailCallback) + self.async_handler = async_handler or get_async_httpx_client( + llm_provider=httpxSpecialProvider.GuardrailCallback + ) self._set_streaming_params( BedrockGuardrailStreamingParams.from_extras( MappingProxyType( diff --git a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/example_config.yaml b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/example_config.yaml index 9b5c4e557f3..665d8cde4cc 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/example_config.yaml +++ b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/example_config.yaml @@ -6,20 +6,19 @@ model_list: model: gpt-4 api_key: os.environ/OPENAI_API_KEY -litellm_settings: - guardrails: - - guardrail_name: "my-generic-guardrail" - litellm_params: - guardrail: generic_guardrail_api - mode: pre_call # Options: pre_call, post_call, during_call, [pre_call, post_call] - api_key: os.environ/GENERIC_GUARDRAIL_API_KEY # Optional if using Bearer auth - api_base: http://localhost:8080 # Required. Endpoint /beta/litellm_basic_guardrail_api is automatically appended - unreachable_fallback: fail_closed # Options: fail_closed (default, raise), fail_open (proceed if endpoint unreachable or upstream returns 502/503/504) - default_on: false # Set to true to apply to all requests by default - additional_provider_specific_params: - # Any additional parameters your guardrail API needs - api_version: "v1" - custom_param: "value" +guardrails: + - guardrail_name: "my-generic-guardrail" + litellm_params: + guardrail: generic_guardrail_api + mode: pre_call # Options: pre_call, post_call, during_call, [pre_call, post_call] + api_key: os.environ/GENERIC_GUARDRAIL_API_KEY # Optional if using Bearer auth + api_base: http://localhost:8080 # Required. Endpoint /beta/litellm_basic_guardrail_api is automatically appended + unreachable_fallback: fail_closed # Options: fail_closed (default, raise), fail_open (proceed if endpoint unreachable or upstream returns 502/503/504) + default_on: false # Set to true to apply to all requests by default + additional_provider_specific_params: + # Any additional parameters your guardrail API needs + api_version: "v1" + custom_param: "value" # Usage examples: diff --git a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py index d50001ba636..3940a320fb8 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py +++ b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py @@ -30,6 +30,7 @@ from litellm.integrations.custom_guardrail import ( from litellm.llms.base_llm.guardrail_translation.utils import ( effective_scan_only_tool_results_for_guardrail, effective_skip_system_message_for_guardrail, + effective_skip_tool_message_for_guardrail, role_out_of_guardrail_scope, ) from litellm.llms.custom_httpx.http_handler import ( @@ -37,12 +38,16 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, ) -from litellm.llms.openai.responses.guardrail_translation.handler import scannable_instructions +from litellm.llms.openai.responses.guardrail_translation.handler import ( + scannable_instructions, + tool_output_texts, +) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.callback_utils import ( add_guardrail_scan_id, add_guardrail_to_applied_guardrails_header, ) +from litellm.responses.litellm_completion_transformation.transformation import TOOL_CALL_OUTPUT_ITEM_TYPES from litellm.types.guardrails import GuardrailEventHooks from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import AllMessageValues @@ -112,12 +117,16 @@ class _ResponsesInputItem(LiteLLMBaseModel): type: str | None = None role: str | None = None content: str | tuple[_ResponsesContentPart, ...] | None = None + output: object = None + call_id: object = None - def text_count(self, *, skip_system: bool) -> int: - if role_out_of_guardrail_scope( - (self.role or "").lower(), skip_system_message=skip_system, skip_tool_message=False - ): + def text_count(self, *, skip_system: bool, skip_tool: bool) -> int: + is_tool_output: Final = self.type in TOOL_CALL_OUTPUT_ITEM_TYPES + role: Final = "tool" if is_tool_output else (self.role or "").lower() + if role_out_of_guardrail_scope(role, skip_system_message=skip_system, skip_tool_message=skip_tool): return 0 + if is_tool_output: + return len(tool_output_texts(self)) if isinstance(self.content, str): return 1 if self.content is None: @@ -1692,13 +1701,14 @@ class PanwPrismaAirsHandler(CustomGuardrail): request_data: Mapping[str, object], *, skip_system: bool, + skip_tool: bool, ) -> frozenset[int] | None: """Return the ``texts`` indices flattened from Responses ``reasoning`` input items. The Responses translation handler gives those model-authored items the default ``user`` role, so the latest-turn selection must not mistake one for a human turn. Empty for requests without a Responses ``input`` item list; None when the raw items - (after the leading ``instructions`` text, both minus whatever ``skip_system`` drops) + (after the leading ``instructions`` text, both minus whatever ``skip_system`` and ``skip_tool`` drop) do not account for every entry of ``texts``. """ try: @@ -1708,7 +1718,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): if not isinstance(raw_input, tuple): return frozenset() offset: Final = 0 if scannable_instructions(request_data, skip_system=skip_system) is None else 1 - counts: Final = tuple(item.text_count(skip_system=skip_system) for item in raw_input) + counts: Final = tuple(item.text_count(skip_system=skip_system, skip_tool=skip_tool) for item in raw_input) if offset + sum(counts) != len(texts): return None starts: Final = itertools.accumulate(counts, initial=offset) @@ -1738,7 +1748,10 @@ class PanwPrismaAirsHandler(CustomGuardrail): if sources is None: return None reasoning: Final = self._reasoning_item_text_indices( - texts, request_data, skip_system=effective_skip_system_message_for_guardrail(self) + texts, + request_data, + skip_system=effective_skip_system_message_for_guardrail(self), + skip_tool=effective_skip_tool_message_for_guardrail(self), ) if reasoning is None: return None diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index be32405154c..60d2abe0ab4 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -131,6 +131,52 @@ def _should_inject_health_check_max_tokens(model_info: Mapping[str, object], mod return (mode or "chat") in _MAX_TOKEN_SUPPORT_MODES +_AUDIO_OUTPUT_PROBE_PROVIDERS: Final[frozenset[str]] = frozenset({"openai", "azure"}) + + +def _health_check_requires_audio_output( + model_info: Mapping[str, object], probe_params: Mapping[str, object], mode: str | None, deployment_model: str | None +) -> bool: + if (mode or "chat") != "chat": + return False + operator_modalities: Final = probe_params.get("modalities") + if isinstance(operator_modalities, list) and "audio" not in operator_modalities: + return False + model: Final = probe_params.get("model") + if not isinstance(model, str): + return False + provider_param: Final = probe_params.get("custom_llm_provider") + try: + _, provider, _, _ = litellm.get_llm_provider( + model=model, custom_llm_provider=provider_param if isinstance(provider_param, str) else None + ) + except litellm.exceptions.BadRequestError: + return False + if provider not in _AUDIO_OUTPUT_PROBE_PROVIDERS: + return False + base_model: Final = model_info.get("base_model") or probe_params.get("base_model") + checked_model: Final = base_model if isinstance(base_model, str) and model == deployment_model else model + checked_model_prefix: Final = checked_model.partition("/")[0] + return litellm.supports_audio_output( + model=checked_model, + custom_llm_provider=checked_model_prefix if checked_model_prefix in _AUDIO_OUTPUT_PROBE_PROVIDERS else provider, + ) + + +def _health_check_audio_output_params( + model_info: Mapping[str, object], probe_params: Mapping[str, object], mode: str | None, deployment_model: str | None +) -> Mapping[str, object]: + if not _health_check_requires_audio_output(model_info, probe_params, mode, deployment_model): + return MappingProxyType({}) + voice: Final = model_info.get("health_check_voice") + return MappingProxyType( + { + "modalities": ["text", "audio"], + "audio": {"voice": voice if isinstance(voice, str) else "alloy", "format": "wav"}, + } + ) + + # Health-check modes that forward `reasoning_effort` to the provider (chat-style calls). _HEALTH_CHECK_MODES_SUPPORTING_REASONING_EFFORT: Final = frozenset((None, "chat", "completion")) @@ -782,9 +828,12 @@ def _resolve_health_check_max_tokens(model_info: dict, litellm_params: dict) -> _PROBE_PARAMS_ADAPTER: Final = TypeAdapter(dict[str, object]) +_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object]) -def update_litellm_params_for_health_check(model_info: dict, litellm_params: dict) -> dict: +def update_litellm_params_for_health_check( + model_info: dict, litellm_params: dict, deployment_model: str | None = None +) -> dict: """ Update the litellm params for health check. @@ -800,7 +849,15 @@ def update_litellm_params_for_health_check(model_info: dict, litellm_params: dic - updates the `model` param with the `health_check_model` if it exists Doc: https://docs.litellm.ai/docs/proxy/health#wildcard-routes - updates the `voice` param with the `health_check_voice` for `audio_speech` mode if it exists Doc: https://docs.litellm.ai/docs/proxy/health#text-to-speech-models - for Bedrock models with region routing (bedrock/region/model), strips the litellm routing prefix but preserves the model ID, and pins `custom_llm_provider` to `bedrock` (only when the deployment hasn't already set one, so an explicit `bedrock_converse` survives) so the bare model id still resolves to the provider (e.g. cross-region ids like `us.cohere.embed-v4:0`) + - asks OpenAI and Azure chat models that support audio output for `modalities` text + audio and a wav + `audio` in the `health_check_voice` (default "alloy"), since those models reject text-only requests. + `base_model` (from `model_info`, then `litellm_params`) decides it only while the probe still targets + `deployment_model` (default: the incoming `litellm_params` model), so a `health_check_model` or a `model` + in `health_check_params` is judged on its own. Keys already set through `health_check_params` are kept, + and `modalities` set there without "audio" turns the audio fields off """ + incoming_model: Final = _PROBE_PARAMS_ADAPTER.validate_python(litellm_params).get("model") + base_model_owner: Final = deployment_model or (incoming_model if isinstance(incoming_model, str) else None) mode: Final = resolve_health_check_mode( model_info, litellm_params, # any-ok: untyped router config dict @@ -875,7 +932,14 @@ def update_litellm_params_for_health_check(model_info: dict, litellm_params: dic ) probe_params: Final = _PROBE_PARAMS_ADAPTER.validate_python(litellm_params) - return {key: value for key, value in probe_params.items() if not is_model_capability_field(key)} + audio_output_params: Final = _health_check_audio_output_params( + _MODEL_INFO_ADAPTER.validate_python(model_info), probe_params, mode, base_model_owner + ) + return { + key: value + for key, value in {**audio_output_params, **probe_params}.items() + if not is_model_capability_field(key) + } _update_litellm_params_for_health_check: Final = update_litellm_params_for_health_check diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 53c54fe15bf..396a00ab1a1 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -32,7 +32,6 @@ from litellm.proxy._types import ( EnterpriseLicenseData, Litellm_EntityType, LitellmUserRoles, - ProxyErrorTypes, ProxyException, SpecialModelNames, UserAPIKeyAuth, @@ -47,6 +46,7 @@ from litellm.proxy.auth.auth_utils import ( from litellm.proxy.auth.model_checks import get_key_models from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils import http_parsing_utils +from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for from litellm.proxy.db.db_lookup_gate import db_lookup_stall_tracker from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler from litellm.proxy.db.health_check_latest import ( @@ -683,21 +683,9 @@ async def health_services_endpoint( except Exception as e: verbose_proxy_logger.error("litellm.proxy.proxy_server.health_services_endpoint(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) - if isinstance(e, HTTPException): - raise ProxyException( - message=getattr(e, "detail", f"Authentication Error({e})"), - type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), - code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), - ) - elif isinstance(e, ProxyException): - raise e - raise ProxyException( - message="Authentication Error, " + str(e), - type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), - code=status.HTTP_500_INTERNAL_SERVER_ERROR, - ) + if isinstance(e, ProxyException): + raise + raise proxy_exception_for(e, default_status_code=status.HTTP_500_INTERNAL_SERVER_ERROR) from e def convert_health_check_to_dict(check) -> dict: @@ -2330,9 +2318,11 @@ async def test_model_connection( stored_params=_OBJECT_MAPPING.validate_python(config_litellm_params), request_params=_OBJECT_MAPPING.validate_python(request_litellm_params), ) + stored_model: Final = _OBJECT_MAPPING.validate_python(config_litellm_params).get("model") litellm_params = update_litellm_params_for_health_check( model_info=dict(probe_model_info), litellm_params=litellm_params, + deployment_model=stored_model if isinstance(stored_model, str) else None, ) ## Auth check, on the final probe params so health_check_params cannot retarget it afterwards diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index e44d9304ac1..a3a6f582dce 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -76,6 +76,7 @@ from litellm.proxy.hooks.batch_enqueued_tokens import ( canonical_provider_batch_id, ) from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit +from litellm.proxy.spend_tracking.input_tokens import tokenizer_models from litellm.responses.litellm_completion_transformation.compaction_items import billable_input_since_newest_compaction from litellm.router_utils.add_retry_fallback_headers import ( ensure_response_additional_headers, @@ -159,6 +160,12 @@ def _resolve_model_group_alias_via_proxy_router(model: str) -> str | None: return resolve_model_group_alias(llm_router.model_group_alias, model) +def _tokenizer_models_via_proxy_router(model: str) -> tuple[str, ...]: + from litellm.proxy.proxy_server import llm_router + + return tokenizer_models(model, llm_router) + + def _resolve_ptu_team_ceiling_via_proxy_router(team_id: str, model_group: str) -> PTUTeamCeiling | None: from litellm.proxy.proxy_server import llm_router @@ -884,6 +891,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ] = _resolve_ptu_team_ceiling_via_proxy_router, fail_closed_resolver: Callable[[], bool] = _fail_closed_rate_limit_enforcement_from_general_settings, force_hash_tag_grouping_resolver: Callable[[], bool] = _force_redis_hash_tag_grouping_from_litellm_settings, + tokenizer_models_resolver: Callable[[str], tuple[str, ...]] = _tokenizer_models_via_proxy_router, ): self.internal_usage_cache = internal_usage_cache self._time_provider = time_provider or datetime.now @@ -893,6 +901,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): self._ptu_team_ceiling_resolver = ptu_team_ceiling_resolver self._fail_closed_resolver = fail_closed_resolver self._force_hash_tag_grouping_resolver = force_hash_tag_grouping_resolver + self._tokenizer_models_resolver = tokenizer_models_resolver if self.internal_usage_cache.dual_cache.redis_cache is not None: self.batch_rate_limiter_script = self.internal_usage_cache.dual_cache.redis_cache.async_register_script( BATCH_RATE_LIMITER_SCRIPT @@ -4009,15 +4018,18 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): try: estimate: Final = max( 0, - int( - token_counter( - model=model or "", - messages=countable_messages, - text=selected_text, - tools=countable_tools, - tool_choice=countable_tool_choice, - use_default_image_token_count=True, + *( + int( + token_counter( + model=tokenizer_model, + messages=countable_messages, + text=selected_text, + tools=countable_tools, + tool_choice=countable_tool_choice, + use_default_image_token_count=True, + ) ) + for tokenizer_model in self._tokenizer_models_resolver(model or "") ), ) return estimate + audio_token_estimate diff --git a/litellm/proxy/hooks/responses_id_security.py b/litellm/proxy/hooks/responses_id_security.py index 27b5924c540..771d68a0ea0 100644 --- a/litellm/proxy/hooks/responses_id_security.py +++ b/litellm/proxy/hooks/responses_id_security.py @@ -31,7 +31,7 @@ if TYPE_CHECKING: _RESPONSES_API_PROVIDER_PREFIX: Final = "/openai" -_RESPONSES_API_CREATE_ROUTES: Final = frozenset({"/v1/responses", "/responses"}) +_RESPONSES_API_ROUTES: Final = frozenset({"/v1/responses", "/responses"}) _ADDRESSED_RESPONSE_ID_FIELD_BY_CALL_TYPE: Final[Mapping[str, str]] = MappingProxyType( { "aresponses": "previous_response_id", @@ -104,7 +104,7 @@ def _rewrite_advertised_id( return event -def _is_responses_api_create_route(request_route: str | None) -> bool: +def _streams_response_ids(request_route: str | None) -> bool: if request_route is None: return False canonical: Final = ( @@ -112,7 +112,8 @@ def _is_responses_api_create_route(request_route: str | None) -> bool: if request_route.startswith(_RESPONSES_API_PROVIDER_PREFIX + "/") else request_route ) - return canonical in _RESPONSES_API_CREATE_ROUTES + parent, _, response_id = canonical.rpartition("/") + return canonical in _RESPONSES_API_ROUTES or (parent in _RESPONSES_API_ROUTES and response_id != "") class ResponsesIDSecurity(CustomLogger): @@ -339,7 +340,7 @@ class ResponsesIDSecurity(CustomLogger): async for chunk in response: if ( isinstance(chunk, BaseLiteLLMOpenAIResponseObject) - and _is_responses_api_create_route(user_api_key_dict.request_route) + and _streams_response_ids(user_api_key_dict.request_route) and not general_settings.get("disable_responses_id_security", False) ): chunk = self._encrypt_response_id(chunk, user_api_key_dict, request_encryption_cache) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 2c05f724374..14d941ff62c 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -3024,13 +3024,7 @@ async def add_litellm_data_to_request( getattr(request.state, "litellm_roi_estimator", False) is True ) - verbose_proxy_logger.debug( - "[PROXY] returned data from litellm_pre_call_utils: %s", - data, - ) - # Team/Project credential overrides from model_config - # Placed after the debug log to avoid leaking credential secrets in logs _apply_credential_overrides_from_model_config( data=data, user_api_key_dict=user_api_key_dict, diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index f2f0bc5be0b..bb901f22649 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -46,6 +46,7 @@ from litellm.proxy.auth.password_policy import ( ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast +from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for from litellm.proxy.common_utils.user_api_key_cache import ( AUTH_OBJECTS_TARGET, object_permission_cache_key, @@ -1729,21 +1730,9 @@ async def user_update( except Exception as e: verbose_proxy_logger.exception("litellm.proxy.proxy_server.user_update(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) - if isinstance(e, HTTPException): - raise ProxyException( - message=getattr(e, "detail", f"Authentication Error({e})"), - type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), - code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), - ) - elif isinstance(e, ProxyException): - raise e - raise ProxyException( - message="Authentication Error, " + str(e), - type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), - code=status.HTTP_400_BAD_REQUEST, - ) + if isinstance(e, ProxyException): + raise + raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e async def bulk_update_processed_users( diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index a19ecb8a9e9..55e59ca5776 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -75,6 +75,7 @@ from litellm.proxy.common_utils.config_sync_pubsub import ( coordination_redis_cache, publish_config_change, ) +from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for from litellm.proxy.common_utils.rbac_utils import check_org_admin_can_generate_keys from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET, UserApiKeyCache @@ -4087,21 +4088,9 @@ async def update_key_fn( # update based on remaining passed in values except Exception as e: verbose_proxy_logger.exception("litellm.proxy.proxy_server.update_key_fn(): Exception occured - %s", e) - if isinstance(e, HTTPException): - raise ProxyException( - message=getattr(e, "detail", f"Authentication Error({e})"), - type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), - code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), - ) - elif isinstance(e, ProxyException): - raise e - raise ProxyException( - message="Authentication Error, " + str(e), - type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), - code=status.HTTP_400_BAD_REQUEST, - ) + if isinstance(e, ProxyException): + raise + raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e @router.post( @@ -7244,21 +7233,9 @@ async def list_keys( except Exception as e: verbose_proxy_logger.exception("Error in list_keys: %s", e) - if isinstance(e, HTTPException): - raise ProxyException( - message=getattr(e, "detail", f"error({e})"), - type=ProxyErrorTypes.internal_server_error, - param=getattr(e, "param", "None"), - code=getattr(e, "status_code", fastapi.status.HTTP_500_INTERNAL_SERVER_ERROR), - ) - elif isinstance(e, ProxyException): - raise e - raise ProxyException( - message="Authentication Error, " + str(e), - type=ProxyErrorTypes.internal_server_error, - param=getattr(e, "param", "None"), - code=fastapi.status.HTTP_500_INTERNAL_SERVER_ERROR, - ) + if isinstance(e, ProxyException): + raise + raise proxy_exception_for(e, default_status_code=fastapi.status.HTTP_500_INTERNAL_SERVER_ERROR) from e async def _apply_non_admin_alias_scope( @@ -7402,21 +7379,9 @@ async def key_aliases( except Exception as e: verbose_proxy_logger.exception("Error in key_aliases: %s", e) - if isinstance(e, HTTPException): - raise ProxyException( - message=getattr(e, "detail", f"error({e})"), - type=ProxyErrorTypes.internal_server_error, - param=getattr(e, "param", "None"), - code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), - ) - elif isinstance(e, ProxyException): - raise e - raise ProxyException( - message="Authentication Error, " + str(e), - type=ProxyErrorTypes.internal_server_error, - param=getattr(e, "param", "None"), - code=status.HTTP_500_INTERNAL_SERVER_ERROR, - ) + if isinstance(e, ProxyException): + raise + raise proxy_exception_for(e, default_status_code=status.HTTP_500_INTERNAL_SERVER_ERROR) from e def _validate_sort_params( diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 8cd3559ba86..e7ad7d04070 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -77,6 +77,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, ) +from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient from litellm.proxy.management.teams.authz import TEAM_ADMIN_ONLY, is_team_admin @@ -2845,21 +2846,9 @@ async def delete_model( except Exception as e: verbose_proxy_logger.exception("Failed to delete model. Due to error - %s", e) - if isinstance(e, HTTPException): - raise ProxyException( - message=getattr(e, "detail", f"Authentication Error({e})"), - type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), - code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), - ) - elif isinstance(e, ProxyException): - raise e - raise ProxyException( - message="Authentication Error, " + str(e), - type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), - code=status.HTTP_400_BAD_REQUEST, - ) + if isinstance(e, ProxyException): + raise + raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e async def delete_team_model_alias( @@ -2899,6 +2888,13 @@ async def delete_team_model_alias( return removed_model_aliases +def _model_not_added_to_db() -> HTTPException: + return HTTPException( + status_code=500, + detail={"error": "Failed to add model to db. Check your server logs for more details."}, + ) + + @router.post( "/model/new", description="Allows adding new models to the model list in the config.yaml", @@ -3080,6 +3076,8 @@ async def add_new_model( if isinstance(e, HTTPException): raise verbose_proxy_logger.exception("Exception in add_new_model: %s", e) + if model_response is None: + raise _model_not_added_to_db() from e else: raise HTTPException( @@ -3088,10 +3086,7 @@ async def add_new_model( ) if model_response is None: - raise HTTPException( - status_code=500, - detail={"error": "Failed to add model to db. Check your server logs for more details."}, - ) + raise _model_not_added_to_db() ## CREATE AUDIT LOG ## asyncio.create_task( @@ -3121,21 +3116,9 @@ async def add_new_model( except Exception as e: verbose_proxy_logger.exception("litellm.proxy.proxy_server.add_new_model(): Exception occured - %s", e) - if isinstance(e, HTTPException): - raise ProxyException( - message=getattr(e, "detail", f"Authentication Error({e})"), - type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), - code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), - ) - elif isinstance(e, ProxyException): - raise e - raise ProxyException( - message="Authentication Error, " + str(e), - type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), - code=status.HTTP_400_BAD_REQUEST, - ) + if isinstance(e, ProxyException): + raise + raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e #### MODEL MANAGEMENT #### @@ -3364,21 +3347,9 @@ async def update_model( return None if model_response is None else _model_write_response(model_response, member_write) except Exception as e: verbose_proxy_logger.exception("litellm.proxy.proxy_server.update_model(): Exception occured - %s", e) - if isinstance(e, HTTPException): - raise ProxyException( - message=getattr(e, "detail", f"Authentication Error({e})"), - type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), - code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), - ) - elif isinstance(e, ProxyException): - raise e - raise ProxyException( - message="Authentication Error, " + str(e), - type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), - code=status.HTTP_400_BAD_REQUEST, - ) + if isinstance(e, ProxyException): + raise + raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e @router.post( diff --git a/litellm/proxy/management_endpoints/organization_endpoints.py b/litellm/proxy/management_endpoints/organization_endpoints.py index e1d19d8efae..8a1b9c112dc 100644 --- a/litellm/proxy/management_endpoints/organization_endpoints.py +++ b/litellm/proxy/management_endpoints/organization_endpoints.py @@ -41,6 +41,7 @@ from litellm.proxy.auth.auth_checks import ( ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast +from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.management_endpoints.budget_management_endpoints import ( @@ -1396,21 +1397,9 @@ async def organization_member_add( ) except Exception as e: verbose_proxy_logger.exception("Error adding member to organization: %s", e) - if isinstance(e, HTTPException): - raise ProxyException( - message=getattr(e, "detail", f"Authentication Error({e})"), - type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), - code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), - ) - elif isinstance(e, ProxyException): - raise e - raise ProxyException( - message="Authentication Error, " + str(e), - type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), - code=status.HTTP_500_INTERNAL_SERVER_ERROR, - ) + if isinstance(e, ProxyException): + raise + raise proxy_exception_for(e, default_status_code=status.HTTP_500_INTERNAL_SERVER_ERROR) from e async def find_member_if_email(user_email: str, prisma_client: PrismaClient) -> LiteLLM_UserTable: diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index a256f417dfa..50b700e712d 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -132,6 +132,7 @@ from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import ( ) from litellm.proxy.common_utils.callback_utils import encrypt_callback_vars from litellm.proxy.common_utils.json_merge_patch import apply_json_merge_patch +from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for from litellm.proxy.common_utils.user_api_key_cache import ( UserApiKeyCache, team_membership_auth_cache_key, @@ -5526,21 +5527,9 @@ async def team_info( e, traceback.format_exc(), ) - if isinstance(e, HTTPException): - raise ProxyException( - message=getattr(e, "detail", f"Authentication Error({e})"), - type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), - code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), - ) - elif isinstance(e, ProxyException): - raise e - raise ProxyException( - message="Authentication Error, " + str(e), - type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), - code=status.HTTP_400_BAD_REQUEST, - ) + if isinstance(e, ProxyException): + raise + raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e @router.get( diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 22c4f27d94a..a980b9c33ff 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -16,6 +16,7 @@ import json import os import re import secrets +from collections import Counter from collections.abc import Awaitable, Callable, Mapping, Sequence from copy import deepcopy from html import escape @@ -42,7 +43,7 @@ if TYPE_CHECKING: import jwt from fastapi import APIRouter, Depends, Header, HTTPException, Request, Response, status from fastapi.responses import RedirectResponse -from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError, field_validator +from pydantic import AliasPath, BaseModel, ConfigDict, Field, TypeAdapter, ValidationError, field_validator import litellm from litellm._internal_context import with_service_target @@ -106,6 +107,7 @@ from litellm.proxy.common_utils.html_forms.default_credentials_hint import shoul from litellm.proxy.common_utils.html_forms.jwt_display_template import ( jwt_display_template, ) +from litellm.proxy.common_utils.html_forms.native_client_consent import team_select_field from litellm.proxy.common_utils.html_forms.ui_login import build_ui_login_form from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET, UserApiKeyCache from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler @@ -184,6 +186,8 @@ _CLI_SSO_START_RATE_LIMIT_WINDOW_SECONDS: Final = 60 _CLI_SSO_START_RATE_LIMIT_MAX_ATTEMPTS: Final = 30 _CLI_SSO_USER_CODE_ALPHABET: Final = "ABCDEFGHJKLMNPQRSTUVWXYZ23456789" _CLI_SSO_LOGIN_ID_RE: Final = re.compile(r"^cli-[A-Za-z0-9_-]{12,124}$") +CLI_SSO_BROWSER_TEAM_CHOICE: Final = "browser_team_choice" +CLI_SSO_CHOSEN_TEAM_ID: Final = "chosen_team_id" _CLI_SSO_USER_CODE_RE = re.compile(rf"^[{_CLI_SSO_USER_CODE_ALPHABET}]{{4}}-[{_CLI_SSO_USER_CODE_ALPHABET}]{{4}}$") _CLI_SSO_SCALAR_TYPES: Final = (str, int, float, bool) _CLI_SSO_DEST_KEY_RE: Final = re.compile(r"^[A-Za-z0-9_.-]+$") @@ -670,6 +674,7 @@ def _render_cli_sso_verification_page( verify_url: str, browser_complete_token: str, prefill_user_code: str | None = None, + team_choices: Sequence[tuple[str, str]] = (), ) -> str: escaped_verify_url: Final = escape(verify_url, quote=True) escaped_browser_complete_token: Final = escape(browser_complete_token, quote=True) @@ -716,6 +721,15 @@ def _render_cli_sso_verification_page( letter-spacing: 0.08em; text-transform: uppercase; }} + select {{ + box-sizing: border-box; + width: 100%; + padding: 12px; + margin-bottom: 18px; + border: 1px solid #cbd5e1; + border-radius: 6px; + font-size: 16px; + }} button {{ margin-top: 16px; width: 100%; @@ -735,6 +749,7 @@ def _render_cli_sso_verification_page(

{instructions}

+ {team_select_field(team_choices, require_choice=True)} @@ -823,13 +838,46 @@ async def cli_sso_complete(request: Request, login_id: str): ): raise HTTPException(status_code=400, detail="Invalid verification code") - flow["user_code_verified"] = True - _set_cli_sso_flow(login_id=login_id, cache=cli_sso_session_cache, flow=flow) + team_choice_flow: Final = _CliSsoTeamChoiceFlow.model_validate(flow) + chosen_team_id: Final = _browser_chosen_team_id(team_choice_flow, form_values) + if chosen_team_id is not None and chosen_team_id not in team_choice_flow.teams: + raise HTTPException(status_code=400, detail="Choose one of the teams you belong to") + + _set_cli_sso_flow( + login_id=login_id, + cache=cli_sso_session_cache, + flow={ + **flow, + "user_code_verified": True, + CLI_SSO_CHOSEN_TEAM_ID: chosen_team_id, + "session_data": ( + flow["session_data"] + if chosen_team_id is None + else { + **flow["session_data"], + "teams": [chosen_team_id, *(team for team in team_choice_flow.teams if team != chosen_team_id)], + } + ), + }, + ) html_content: Final = render_cli_sso_success_page() return HTMLResponse(content=html_content, status_code=200) +class _CliSsoTeamChoiceFlow(BaseModel): + model_config = ConfigDict(frozen=True) + + browser_team_choice: bool = Field(default=False, validation_alias=CLI_SSO_BROWSER_TEAM_CHOICE) + teams: tuple[str, ...] = Field(default=(), validation_alias=AliasPath("session_data", "teams")) + + +def _browser_chosen_team_id(flow: _CliSsoTeamChoiceFlow, form_values: Mapping[str, Sequence[str]]) -> str | None: + if not flow.browser_team_choice: + return None + return (form_values.get("team_id") or [""])[0] or None + + def normalize_email(email: str | None) -> str | None: """ Normalize email address to lowercase for consistent storage and comparison. @@ -2416,6 +2464,18 @@ def _cli_sso_session_teams(team_details: Sequence[CliSsoTeamDetail]) -> list[str return [detail.team_id for detail in team_details if detail.team_id is not None] +def _browser_team_choices( + flow: _CliSsoTeamChoiceFlow, team_details: Sequence[CliSsoTeamDetail] +) -> tuple[tuple[str, str], ...]: + if not flow.browser_team_choice: + return () + labeled: Final = tuple( + (detail.team_id, detail.team_alias or detail.team_id) for detail in team_details if detail.team_id is not None + ) + label_counts: Final = Counter(label for _, label in labeled) + return tuple((team_id, label if label_counts[label] == 1 else f"{label} ({team_id})") for team_id, label in labeled) + + def selected_cli_sso_team_detail(team_details: object, team_id: str | None) -> CliSsoTeamDetail | None: """``None`` means the team's grants are unknown. An empty grant is a real value meaning unrestricted, so an unknown one must not be minted as empty.""" @@ -2484,6 +2544,7 @@ async def _complete_cli_sso_callback_session( request.scope.get("litellm_microsoft_interactive_subject"), resolved_user_id, prisma_client ) resolved_teams: Final = _cli_sso_session_teams(team_details) + team_choices: Final = _browser_team_choices(_CliSsoTeamChoiceFlow.model_validate(flow), team_details) attribution_metadata: Final = build_cli_sso_attribution_metadata(result=result) if attribution_metadata: await _persist_cli_sso_user_metadata( @@ -2521,6 +2582,7 @@ async def _complete_cli_sso_callback_session( verify_url=verify_url, browser_complete_token=browser_complete_token, prefill_user_code=prefill_user_code, + team_choices=team_choices, ), status_code=200, ) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index e478096f7ad..f96fab75bd1 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -50,6 +50,7 @@ from litellm.constants import ( from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException from litellm.integrations.custom_guardrail import guardrail_request_data_with_streaming, is_guardrail_intervention from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix +from litellm.litellm_core_utils.fallback_generalizations import match_routing_generalization from litellm.llms.anthropic.common_utils import AnthropicModelInfo, merge_anthropic_beta_headers from litellm.llms.azure.passthrough.transformation import ( foreign_azure_deployment, @@ -67,6 +68,7 @@ from litellm.llms.deepgram.common_utils import ( from litellm.llms.fal_ai.cost_calculator import fal_ai_passthrough_cost, fal_ai_queue_base from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_model_group_in_path from litellm.llms.openai.common_utils import OpenAIError as LiteLLMOpenAIError +from litellm.llms.openai.common_utils import is_openai_agents_api_path, openai_api_resource from litellm.llms.openai.workload_identity import get_workload_identity_bearer_token_for_api_base from litellm.llms.oss_decision import OssDecisionProvider, oss_connection, validate_oss_request from litellm.llms.vertex_ai.vertex_llm_base import VertexBase @@ -698,6 +700,20 @@ async def typesafe_proxy_route( return await endpoint_func(request, fastapi_response, user_api_key_dict) +_COST_MAP_ENTRIES: Final = TypeAdapter(Mapping[str, object]) + + +def typesafe_allowlist_model_names() -> tuple[str, ...]: + if not passthrough_endpoint_router.has_credentials(custom_llm_provider="typesafe"): + return () + cost_map: Final = _COST_MAP_ENTRIES.validate_python(litellm.model_cost) # pyright: ignore[reportUnknownMemberType] # legacy registry is validated at this boundary + priced_names: Final = (name.removeprefix("typesafe/") for name in cost_map if name.startswith("typesafe/")) + jev_models_typesafe_serves: Final = sorted( + name for name in priced_names if match_routing_generalization(name) == "typesafe" + ) + return ("typesafe/*", *jev_models_typesafe_serves) + + @router.post( "/laya/v1/systemone", tags=["Laya Pass-through", "pass-through"], @@ -3515,6 +3531,10 @@ async def deepgram_listen_websocket_route( ) +_OPENAI_BETA_HEADER: Final = "openai-beta" +_CLIENT_OPENAI_BETA_RESOURCES: Final = frozenset({"agents", "vaults", "chatkit"}) + + class BaseOpenAIPassThroughHandler: @staticmethod async def base_openai_pass_through_handler( @@ -3525,7 +3545,7 @@ class BaseOpenAIPassThroughHandler: base_target_url: str, api_key: str | None, custom_llm_provider: litellm.LlmProviders, - extra_headers: dict | None = None, + extra_headers: Mapping[str, str] | None = None, ): encoded_endpoint = httpx.URL(endpoint).path # Ensure endpoint starts with '/' for proper URL construction @@ -3550,7 +3570,11 @@ class BaseOpenAIPassThroughHandler: endpoint=endpoint, target=str(updated_url), custom_headers=BaseOpenAIPassThroughHandler._assemble_headers( - api_key=api_key, request=request, extra_headers=extra_headers + api_key=api_key, + endpoint=endpoint, + request=request, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, ), is_streaming_request=is_streaming_request, custom_llm_provider=( @@ -3572,27 +3596,41 @@ class BaseOpenAIPassThroughHandler: _base_openai_pass_through_handler = base_openai_pass_through_handler @staticmethod - def _append_openai_beta_header(headers: dict, request: Request) -> dict: - """ - Appends the OpenAI-Beta header to the headers if the request is an OpenAI Assistants API request - """ - if RouteChecks.is_assistants_api_request(request) is True and "OpenAI-Beta" not in headers: - headers["OpenAI-Beta"] = "assistants=v2" - return headers + def _openai_beta(endpoint: str, request: Request, custom_llm_provider: litellm.LlmProviders) -> str | None: + if ( + custom_llm_provider == litellm.LlmProviders.OPENAI + and openai_api_resource(endpoint) in _CLIENT_OPENAI_BETA_RESOURCES + ): + return request.headers.get(_OPENAI_BETA_HEADER) or ( + "agents=v1" if is_openai_agents_api_path(endpoint) else None + ) + return "assistants=v2" if RouteChecks.is_assistants_api_request(request) else None @staticmethod - def _assemble_headers(api_key: str | None, request: Request, extra_headers: dict | None = None) -> dict: - base_headers = {} - if api_key is not None: - base_headers = { - "authorization": f"Bearer {api_key}", - "api-key": f"{api_key}", - } - if extra_headers is not None: - base_headers.update(extra_headers) + def _append_openai_beta_header( + headers: Mapping[str, str], endpoint: str, request: Request, custom_llm_provider: litellm.LlmProviders + ) -> dict[str, str]: + if any(name.lower() == _OPENAI_BETA_HEADER for name in headers): + return {**headers} + openai_beta: Final = BaseOpenAIPassThroughHandler._openai_beta(endpoint, request, custom_llm_provider) + return {**headers} if openai_beta is None else {**headers, _OPENAI_BETA_HEADER: openai_beta} + + @staticmethod + def _assemble_headers( + api_key: str | None, + endpoint: str, + request: Request, + custom_llm_provider: litellm.LlmProviders, + extra_headers: Mapping[str, str] | None = None, + ) -> dict[str, str]: + api_key_headers: Final[Mapping[str, str]] = ( + {} if api_key is None else {"authorization": f"Bearer {api_key}", "api-key": api_key} + ) return BaseOpenAIPassThroughHandler._append_openai_beta_header( - headers=base_headers, + headers={**api_key_headers, **(extra_headers or {})}, + endpoint=endpoint, request=request, + custom_llm_provider=custom_llm_provider, ) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py index a11b0be3983..4eeb9b26174 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py @@ -1,10 +1,12 @@ from datetime import datetime from typing import Final +from urllib.parse import urlparse import httpx import litellm from litellm import stream_chunk_builder +from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.hidden_params import set_hidden_param from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.litellm_logging import ( @@ -16,7 +18,7 @@ from litellm.llms.cohere.chat.v2_transformation import CohereV2ChatConfig from litellm.llms.cohere.common_utils import ( ModelResponseIterator as CohereModelResponseIterator, ) -from litellm.llms.cohere.embed.v1_transformation import CohereEmbeddingConfig +from litellm.llms.cohere.embed.transformation import CohereEmbeddingConfig, CohereEmbeddingRowShape from litellm.proxy._types import PassThroughEndpointLoggingTypedDict from litellm.types.passthrough_endpoints.pass_through_endpoints import ( PassthroughStandardLoggingPayload, @@ -29,6 +31,12 @@ from litellm.types.utils import ( from .base_passthrough_logging_handler import BasePassthroughLoggingHandler +COHERE_EMBED_PATHS: Final = ("/v1/embed", "/v2/embed") + + +def is_cohere_embed_route(url_route: str) -> bool: + return urlparse(url_route).path.rstrip("/").endswith(COHERE_EMBED_PATHS) + class CoherePassthroughLoggingHandler(BasePassthroughLoggingHandler): @property @@ -82,28 +90,24 @@ class CoherePassthroughLoggingHandler(BasePassthroughLoggingHandler): """ Handle Cohere passthrough logging with route detection and cost tracking. """ - # Check if this is an embed endpoint - if "/v1/embed" in url_route and "/v1/embeddings" not in url_route: + if is_cohere_embed_route(url_route): model: Final = request_body.get("model", response_body.get("model", "")) try: - cohere_embed_config: Final = CohereEmbeddingConfig() - litellm_model_response = litellm.EmbeddingResponse() handler_instance: Final = CoherePassthroughLoggingHandler() input_texts = request_body.get("texts", []) if not input_texts: input_texts = request_body.get("input", []) - # Transform the response - litellm_model_response = cohere_embed_config.transform_response( + litellm_model_response: Final = CohereEmbeddingConfig().transform_native_embed_response( response=httpx_response, api_key="", logging_obj=logging_obj, data=request_body, - model_response=litellm_model_response, model=model, encoding=litellm.encoding, input=input_texts, + row_shape=CohereEmbeddingRowShape.TYPED, ) # Calculate cost using LiteLLM's cost calculator @@ -153,20 +157,11 @@ class CoherePassthroughLoggingHandler(BasePassthroughLoggingHandler): "result": litellm_model_response, "kwargs": kwargs, } - except Exception: - # For other routes (e.g., /v2/chat), fall back to chat handler - return super().passthrough_chat_handler( - httpx_response=httpx_response, - response_body=response_body, - logging_obj=logging_obj, - url_route=url_route, - result=result, - start_time=start_time, - end_time=end_time, - cache_hit=cache_hit, - request_body=request_body, - **kwargs, + except Exception as e: + verbose_proxy_logger.warning( + "Could not price the Cohere passthrough embed response, logging it without a cost: %s", e ) + return {"result": None, "kwargs": kwargs} # For non-embed routes (e.g., /v2/chat), fall back to chat handler return super().passthrough_chat_handler( diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py index c410150df70..b98dd42e3de 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py @@ -10,7 +10,7 @@ from typing import Final, Literal, TypeAlias from urllib.parse import urlparse import httpx -from pydantic import TypeAdapter +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError import litellm from litellm._logging import verbose_proxy_logger @@ -21,6 +21,8 @@ from litellm.litellm_core_utils.litellm_logging import ( get_standard_logging_object_payload, ) from litellm.litellm_core_utils.token_counter import high_detail_image_token_upper_bound +from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator +from litellm.llms.openai.common_utils import is_openai_agents_api_path from litellm.llms.openai.openai import OpenAIConfig from litellm.llms.openai.openai import OpenAIConfig as OpenAIConfigType from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig @@ -38,7 +40,13 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( EndpointType, PassthroughStandardLoggingPayload, ) -from litellm.types.utils import EmbeddingResponse, ImageResponse, LlmProviders, PassthroughCallTypes +from litellm.types.utils import ( + EmbeddingResponse, + ImageResponse, + LlmProviders, + PassthroughCallTypes, + StandardPassThroughResponseObject, +) from litellm.utils import ModelResponse, TextCompletionResponse, convert_to_model_response_object # Hostnames that route to OpenAI-compatible APIs. @@ -121,6 +129,52 @@ def _responses_sub_route(path: str) -> tuple[str, ...] | None: return None if anchor is None else tuple(segments[anchor + 1 :]) +_STR_MAPPING: Final = TypeAdapter(Mapping[str, object]) + + +class _AgentConfig(BaseModel): + model_config = ConfigDict(frozen=True) + + model: str + + +class _AgentsApiBody(BaseModel): + model_config = ConfigDict(frozen=True) + + agent: _AgentConfig + + +class _AgentsApiSessionEvent(BaseModel): + model_config = ConfigDict(frozen=True) + + session: _AgentsApiBody + + +_AGENTS_API_BODY: Final = TypeAdapter(_AgentsApiBody | _AgentConfig) + + +def _agent_model(body: Mapping[str, object]) -> str | None: + try: + parsed: Final = _AGENTS_API_BODY.validate_python(body) + except ValidationError: + return None + return parsed.agent.model if isinstance(parsed, _AgentsApiBody) else parsed.model + + +def _session_event_model(chunk: str) -> str | None: + event: Final = BaseModelResponseIterator.string_to_dict_parser(str_line=chunk) + if event is None: + return None + try: + return _AgentsApiSessionEvent.model_validate(event).session.agent.model + except ValidationError: + return None + + +def _streamed_agent_model(all_chunks: Sequence[str]) -> str | None: + return next((model for chunk in all_chunks if (model := _session_event_model(chunk)) is not None), None) + + def _is_remote_high_detail_image(part: object) -> bool: if not isinstance(part, Mapping) or part.get("type") != "image_url": return False @@ -220,14 +274,44 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): return "billed" @staticmethod - def _label_unbilled_responses_call( + def is_openai_agents_route(url_route: str) -> bool: + return is_openai_compatible_url(url_route) and is_openai_agents_api_path(urlparse(url_route).path) + + @staticmethod + def _unbilled_call_model( + url_route: str, request_body: Mapping[str, object], response_body: Mapping[str, object] + ) -> str | None: + if OpenAIPassthroughLoggingHandler.is_openai_agents_route(url_route): + return _agent_model(response_body) or _agent_model(request_body) + response_model: Final = response_body.get("model") + if OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route) and isinstance(response_model, str): + return response_model + return None + + @staticmethod + def _label_model(logging_obj: LiteLLMLoggingObj, model: str | None) -> None: + if model is not None: + logging_obj.model_call_details["model"] = model # rebind-ok: spend rows read the model from here + + @staticmethod + def _label_unbilled_call( logging_obj: LiteLLMLoggingObj, url_route: str, + request_body: Mapping[str, object], response_body: Mapping[str, object], ) -> None: - model: Final = response_body.get("model") - if OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route) and isinstance(model, str): - logging_obj.model_call_details["model"] = model # rebind-ok: spend rows read the model from here + OpenAIPassthroughLoggingHandler._label_model( + logging_obj, OpenAIPassthroughLoggingHandler._unbilled_call_model(url_route, request_body, response_body) + ) + + @staticmethod + def _log_agents_stream( + litellm_logging_obj: LiteLLMLoggingObj, request_body: Mapping[str, object], all_chunks: Sequence[str] + ) -> PassThroughEndpointLoggingTypedDict: + OpenAIPassthroughLoggingHandler._label_model( + litellm_logging_obj, _streamed_agent_model(all_chunks) or _agent_model(request_body) + ) + return {"result": StandardPassThroughResponseObject(response="\n".join(all_chunks)), "kwargs": {}} @staticmethod def is_openai_embeddings_route(url_route: str) -> bool: @@ -397,7 +481,12 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): is_responses: Final = responses_billing != "unbilled" if not (is_chat_completions or is_embeddings or is_image_generation or is_image_editing or is_responses): - OpenAIPassthroughLoggingHandler._label_unbilled_responses_call(logging_obj, url_route, response_body) + OpenAIPassthroughLoggingHandler._label_unbilled_call( + logging_obj, + url_route, + request_body=_STR_MAPPING.validate_python(request_body), + response_body=_STR_MAPPING.validate_python(response_body), + ) return { "result": None, "kwargs": kwargs, @@ -676,6 +765,10 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): """ Handle logging for collected OpenAI streaming chunks with cost tracking. """ + if OpenAIPassthroughLoggingHandler.is_openai_agents_route(url_route): + return OpenAIPassthroughLoggingHandler._log_agents_stream( + litellm_logging_obj, _STR_MAPPING.validate_python(request_body), all_chunks + ) try: is_responses: Final = OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route) terminal_response: Final = ( diff --git a/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py b/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py index 270a1cc2f3b..1d0b3d00434 100644 --- a/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py +++ b/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py @@ -1,4 +1,5 @@ import json +import os from collections.abc import Callable from typing import TYPE_CHECKING, Final @@ -88,6 +89,14 @@ class PassthroughEndpointRouter: ) return _resolve_oidc_reference(get_secret_str(_env_variable_name)) + def has_credentials(self, custom_llm_provider: str) -> bool: + if self._get_deployment_api_key(custom_llm_provider=custom_llm_provider, region_name=None) is not None: + return True + env_variable_name: Final = self._get_default_env_variable_name_passthrough_endpoint( + custom_llm_provider=custom_llm_provider, + ) + return bool(os.environ.get(env_variable_name)) + def _get_deployment_api_key( self, custom_llm_provider: str, diff --git a/litellm/proxy/pass_through_endpoints/success_handler.py b/litellm/proxy/pass_through_endpoints/success_handler.py index 9c6c3111209..27e31c8f74d 100644 --- a/litellm/proxy/pass_through_endpoints/success_handler.py +++ b/litellm/proxy/pass_through_endpoints/success_handler.py @@ -22,6 +22,7 @@ from .llm_provider_handlers.assembly_passthrough_logging_handler import ( ) from .llm_provider_handlers.cohere_passthrough_logging_handler import ( CoherePassthroughLoggingHandler, + is_cohere_embed_route, ) from .llm_provider_handlers.cursor_passthrough_logging_handler import ( CursorPassthroughLoggingHandler, @@ -91,7 +92,7 @@ class PassThroughEndpointLogging: self.TRACKED_ANTHROPIC_ROUTES = ["/messages", "/v1/messages/batches"] # Cohere - self.TRACKED_COHERE_ROUTES = ["/v2/chat", "/v1/embed"] + self.TRACKED_COHERE_ROUTES = ["/v2/chat"] self.assemblyai_passthrough_logging_handler = AssemblyAIPassthroughLoggingHandler() # Langfuse @@ -536,13 +537,7 @@ class PassThroughEndpointLogging: return False def is_cohere_route(self, url_route: str) -> bool: - for route in self.TRACKED_COHERE_ROUTES: - if route not in url_route: - continue - if route == "/v1/embed" and "/v1/embeddings" in url_route: - continue - return True - return False + return is_cohere_embed_route(url_route) or any(route in url_route for route in self.TRACKED_COHERE_ROUTES) def is_assemblyai_route(self, url_route: str): parsed_url: Final = urlparse(url_route) @@ -645,6 +640,7 @@ class PassThroughEndpointLogging: or OpenAIPassthroughLoggingHandler.is_openai_image_generation_route(url_route) or OpenAIPassthroughLoggingHandler.is_openai_image_editing_route(url_route) or OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route) + or OpenAIPassthroughLoggingHandler.is_openai_agents_route(url_route) ) def _set_cost_per_request( diff --git a/litellm/proxy/policy_engine/init_policies.py b/litellm/proxy/policy_engine/init_policies.py index 59d15b0ba0a..2d0e9d25368 100644 --- a/litellm/proxy/policy_engine/init_policies.py +++ b/litellm/proxy/policy_engine/init_policies.py @@ -11,7 +11,7 @@ from collections.abc import Mapping, Sequence from itertools import chain from typing import TYPE_CHECKING, Any, Final, Optional -from litellm._logging import print_startup_message, verbose_proxy_logger +from litellm._logging import print_startup_message, print_startup_warning, verbose_proxy_logger from litellm.proxy.policy_engine.attachment_registry import get_attachment_registry from litellm.proxy.policy_engine.policy_registry import get_policy_registry from litellm.proxy.policy_engine.policy_validator import PolicyValidator @@ -93,7 +93,7 @@ def _print_policies_on_startup( ) else: print_startup_message("\n".join((header, *policy_lines))) - print_startup_message( + print_startup_warning( f"\n{_yellow_color_code}Warning: No policy_attachments configured. Policies will not be applied to any requests.{_reset_color_code}\n", ) sys.stdout.flush() diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 909f6de40cd..0d8ab98a0b1 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -10,16 +10,17 @@ import sys import urllib.parse as urlparse from collections.abc import Callable, Iterable, Mapping, Sequence from pathlib import Path -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Annotated, Any, Final import click import httpx from click.core import ParameterSource from dotenv import load_dotenv -from pydantic import ConfigDict +from pydantic import BeforeValidator, ConfigDict, TypeAdapter +from typing_extensions import ReadOnly, TypedDict import litellm -from litellm._logging import print_startup_message +from litellm._logging import StartupOutputSettings, print_startup_message, print_startup_warning, terminal_text from litellm.constants import DEFAULT_NUM_WORKERS_LITELLM_PROXY from litellm.proxy.db.pgbouncer import ( PgBouncerError, @@ -205,7 +206,7 @@ def deprecated_v2_flag_passed_on_cli() -> bool: def _print_enforce_prisma_migration_check_notice() -> None: - print_startup_message( + print_startup_warning( "\033[1;33mLiteLLM Proxy: --enforce_prisma_migration_check is " "deprecated and has no effect, because the proxy always exits " "when database setup fails at startup. You can safely remove it.\033[0m", @@ -227,6 +228,36 @@ def _print_or_hold_enforce_prisma_migration_check_notice( return print_once +def _empty_as_none(value: object) -> object: + return value or None + + +class _ConfigStartupOutputSettings(TypedDict, total=False): + litellm_settings: ReadOnly[Annotated[StartupOutputSettings | None, BeforeValidator(_empty_as_none)]] + + +_CONFIG_STARTUP_OUTPUT_SETTINGS: Final = TypeAdapter(_ConfigStartupOutputSettings) + + +def _apply_config_suppress_startup_output(litellm_settings: StartupOutputSettings | None) -> None: + if litellm_settings is None or litellm_settings.get("suppress_startup_output") is not True: + return + litellm.suppress_startup_output = True + os.environ["LITELLM_SUPPRESS_STARTUP_OUTPUT"] = "true" + + +def _apply_config_logging_settings(litellm_settings: StartupOutputSettings | None) -> None: + """Turn on the config's json_logs and suppress_startup_output before any worker starts. Workers that + uvicorn or granian spawn import litellm afresh and read both from the env at import, so they log JSON + and stay quiet from their first line instead of from when load_config reaches the key.""" + _apply_config_suppress_startup_output(litellm_settings) + if litellm_settings is None or litellm_settings.get("json_logs") is not True: + return + litellm.json_logs = True + os.environ["JSON_LOGS"] = "true" + litellm.turn_on_json() + + class ProxyInitializationHelpers: @staticmethod def echo_litellm_version(): @@ -373,7 +404,7 @@ class ProxyInitializationHelpers: if "timeout_worker_healthcheck" in inspect.signature(uvicorn.Config.__init__).parameters: uvicorn_args["timeout_worker_healthcheck"] = timeout_worker_healthcheck else: - print_startup_message( + print_startup_warning( f"\033[1;33mLiteLLM Proxy: --timeout_worker_healthcheck " f"requires uvicorn>=0.37.0, but installed uvicorn=={uvicorn.__version__}. " f"Ignoring the flag.\033[0m", @@ -396,7 +427,7 @@ class ProxyInitializationHelpers: import uvicorn if max_requests_before_restart is None: - print_startup_message( + print_startup_warning( "\033[1;33mLiteLLM Proxy: --max_requests_before_restart_jitter " "has no effect without --max_requests_before_restart\033[0m\n", ) @@ -404,7 +435,7 @@ class ProxyInitializationHelpers: if "limit_max_requests_jitter" in inspect.signature(uvicorn.Config.__init__).parameters: uvicorn_args["limit_max_requests_jitter"] = jitter else: - print_startup_message( + print_startup_warning( f"\033[1;33mLiteLLM Proxy: --max_requests_before_restart_jitter " f"requires uvicorn>=0.41.0, but installed uvicorn=={uvicorn.__version__}. " f"Ignoring the flag.\033[0m", @@ -571,12 +602,12 @@ class ProxyInitializationHelpers: print_startup_message(f"\033[1;32mLiteLLM Proxy: Starting server on {host}:{port} using Granian\033[0m\n") if max_requests_before_restart is not None: - print_startup_message( + print_startup_warning( "\033[1;33mLiteLLM: --max_requests_before_restart is not supported by Granian " "(Granian uses workers_lifetime in seconds, not a per-request limit).\033[0m\n", ) if ciphers is not None: - print_startup_message("\033[1;33mLiteLLM: --ciphers is not applied when using --run_granian.\033[0m\n") + print_startup_warning("\033[1;33mLiteLLM: --ciphers is not applied when using --run_granian.\033[0m\n") kwargs: Final[dict[str, object]] = { "target": "litellm.proxy.proxy_server:app", @@ -698,7 +729,7 @@ class ProxyInitializationHelpers: gunicorn_options["max_requests"] = max_requests_before_restart if max_requests_before_restart_jitter is not None: if max_requests_before_restart is None: - print_startup_message( + print_startup_warning( "\033[1;33mLiteLLM Proxy: --max_requests_before_restart_jitter " "has no effect without --max_requests_before_restart\033[0m\n", ) @@ -739,7 +770,7 @@ class ProxyInitializationHelpers: with open(os.devnull, "w") as devnull: subprocess.Popen(command, stdout=devnull, stderr=devnull) except Exception as e: - print_startup_message(f""" + print_startup_warning(f""" LiteLLM Warning: proxy started with `ollama` model\n`ollama serve` failed with Exception{e}. \nEnsure you run `ollama serve` """) @@ -1325,6 +1356,19 @@ def run_server( os.environ[k] = v litellm_settings = None + env_config_file: Final = os.environ.get("CONFIG_FILE_PATH") + if ( + env_config_file + and os.environ.get("LITELLM_CONFIG_BUCKET_NAME") is None + and ProxyConfig().is_yaml(config_file_path=env_config_file) + ): + import asyncio + + _apply_config_suppress_startup_output( + _CONFIG_STARTUP_OUTPUT_SETTINGS.validate_python( + asyncio.run(ProxyConfig().read_config_file(config_file_path=env_config_file)) + ).get("litellm_settings") + ) if config is not None: """ Allow user to pass in db url via config @@ -1342,17 +1386,9 @@ def run_server( ### LITELLM SETTINGS ### litellm_settings = _config.get("litellm_settings", None) - if ( - litellm_settings is not None - and "json_logs" in litellm_settings - and litellm_settings["json_logs"] is True - ): - import litellm - - litellm.json_logs = True - os.environ["JSON_LOGS"] = "true" - - litellm.turn_on_json() + _apply_config_logging_settings( + _CONFIG_STARTUP_OUTPUT_SETTINGS.validate_python(_config).get("litellm_settings") + ) if held_prisma_notice is not None: held_prisma_notice() ### GENERAL SETTINGS ### @@ -1450,7 +1486,9 @@ def run_server( _bad_scheme = unsupported_db_scheme(_candidate_url) if _bad_scheme is not None: print( - f"\033[1;31mLiteLLM Proxy: {unsupported_db_scheme_message(_db_env, _bad_scheme)}\033[0m", + terminal_text( + f"\033[1;31mLiteLLM Proxy: {unsupported_db_scheme_message(_db_env, _bad_scheme)}\033[0m" + ), file=sys.stderr, flush=True, ) @@ -1553,7 +1591,7 @@ def run_server( env_value=os.getenv("USE_V2_MIGRATION_RESOLVER"), ) if deprecated_v2_flag_passed_on_cli() and use_v2_resolver: - print_startup_message( + print_startup_warning( "\033[1;33mLiteLLM Proxy: --use_v2_migration_resolver is " "deprecated and has no effect, because the v2 migration " "resolver is now the default. You can safely remove it. To " @@ -1561,7 +1599,7 @@ def run_server( "--use_legacy_migration_resolver.\033[0m", ) if not use_v2_resolver: - print_startup_message( + print_startup_warning( "\033[1;33mLiteLLM Proxy: Using the legacy (v1) migration " "resolver. It performs the diff-and-force recovery that can " "cause schema thrashing during rolling deploys where two " @@ -1583,22 +1621,26 @@ def run_server( # issues, and any `prisma db push` against a # partitioned LiteLLM_SpendLogs. print( - f"\033[1;31mLiteLLM Proxy: Database migration cannot proceed. {e}\033[0m", + terminal_text(f"\033[1;31mLiteLLM Proxy: Database migration cannot proceed. {e}\033[0m"), file=sys.stderr, flush=True, ) sys.exit(2) if not setup_ok: print( - "\033[1;31mLiteLLM Proxy: Database setup failed after multiple retries. " - "The proxy cannot start safely. Please check your database connection and migration status.\033[0m" + terminal_text( + "\033[1;31mLiteLLM Proxy: Database setup failed after multiple retries. " + "The proxy cannot start safely. Please check your database connection and migration status.\033[0m" + ) ) sys.exit(1) else: print( - "\033[1;31mLiteLLM Proxy: a database URL is set but the prisma CLI is neither on PATH nor importable " - "as a package, so the database cannot be set up. Install it with `pip install 'litellm[extra_proxy]'` " - "or run a shipped LiteLLM image.\033[0m" + terminal_text( + "\033[1;31mLiteLLM Proxy: a database URL is set but the prisma CLI is neither on PATH nor importable " + "as a package, so the database cannot be set up. Install it with `pip install 'litellm[extra_proxy]'` " + "or run a shipped LiteLLM image.\033[0m" + ) ) sys.exit(1) pgbouncer_settings: Final = PgBouncerSettings() @@ -1609,8 +1651,10 @@ def run_server( ) if isinstance(pooled_database_url, PgBouncerError): print( - f"\033[1;31mLiteLLM Proxy: LITELLM_PGBOUNCER_ENABLED is set but the in-container pgbouncer " - f"could not start: {pooled_database_url.reason}\033[0m", + terminal_text( + f"\033[1;31mLiteLLM Proxy: LITELLM_PGBOUNCER_ENABLED is set but the in-container pgbouncer " + f"could not start: {pooled_database_url.reason}\033[0m" + ), file=sys.stderr, flush=True, ) @@ -1675,7 +1719,7 @@ def run_server( run_gunicorn=run_gunicorn is True, ) if single_worker_warning is not None: - print_startup_message(f"\033[1;33m{single_worker_warning}\033[0m\n") + print_startup_warning(f"\033[1;33m{single_worker_warning}\033[0m\n") if run_gunicorn is False and run_hypercorn is False and run_granian is False: if limit_concurrency is not None: uvicorn_args["limit_concurrency"] = limit_concurrency diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b263412ce0f..091e05610ba 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -100,7 +100,11 @@ from litellm.litellm_core_utils.litellm_logging import ( ) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads -from litellm.litellm_core_utils.token_counter import UnsupportedContentTypeError, offload_token_count +from litellm.litellm_core_utils.token_counter import ( + UnsupportedContentTypeError, + deployment_openai_model, + offload_token_count, +) from litellm.llms.bedrock.count_tokens.kimi_k3 import ( HUGGING_FACE_SOURCE, KimiK3Encodings, @@ -306,6 +310,7 @@ import litellm._redis from litellm import Router from litellm._internal_context import service_target, with_service_target from litellm._logging import ( + STARTUP_OUTPUT_SETTINGS, print_startup_message, redact_string, verbose_proxy_logger, @@ -524,6 +529,7 @@ from litellm.proxy.common_utils.openai_error_payload import ( LITELLM_CALL_ID_HEADER, headers_with_litellm_call_id, litellm_call_id_headers, + proxy_exception_for, with_litellm_call_id, ) from litellm.proxy.common_utils.periodic_reload_schedule import ( @@ -5800,6 +5806,12 @@ class ProxyConfig: # verbose_proxy_logger.debug(f"loaded config={json.dumps(config, indent=4)}") return config + async def read_config_file(self, config_file_path: str) -> Mapping[str, object]: + """The config file with its includes merged, without the bucket, database and secret steps of get_config.""" + return await self._process_includes( + config=self._load_yaml_file(config_file_path), config_file_path=os.path.abspath(config_file_path) + ) + async def _process_includes(self, config: dict, config_file_path: str) -> dict: """ Process includes by appending their contents to the main config @@ -6424,13 +6436,8 @@ class ProxyConfig: search_tools_parsed: Final[list[SearchToolTypedDict]] = [] - print_startup_message("\033[32mLiteLLM: Proxy initialized with Search Tools:\033[0m") - for search_tool in search_tools_raw: - # Display loaded search tool search_tool_name = search_tool.get("search_tool_name", "") - search_provider = search_tool.get("litellm_params", {}).get("search_provider", "") - print_startup_message(f"\033[32m {search_tool_name} ({search_provider})\033[0m") # Handle os.environ/ variables in litellm_params litellm_params = search_tool.get("litellm_params", {}) @@ -6600,13 +6607,15 @@ class ProxyConfig: from litellm.types.integrations.prometheus import validate_caller_identity_settings validate_caller_identity_settings(litellm_settings) + if STARTUP_OUTPUT_SETTINGS.validate_python(litellm_settings).get("suppress_startup_output") is True: + litellm.suppress_startup_output = True # ANSI escape code for blue text blue_color_code: Final = "\033[94m" reset_color_code: Final = "\033[0m" for key, value in litellm_settings.items(): if key == "cache" and value is True: - print_startup_message(f"{blue_color_code}\nSetting Cache on Proxy") + print_startup_message(f"\n{blue_color_code}Setting Cache on Proxy{reset_color_code}") from litellm.caching.caching import Cache cache_params = {} @@ -6921,6 +6930,8 @@ class ProxyConfig: setattr(litellm, key, value) elif key == "drop_params": litellm.drop_params = drop_params_flag(value, "litellm_settings.drop_params", verbose_proxy_logger) + elif key == "suppress_startup_output": + continue else: verbose_proxy_logger.debug( "%s setting litellm.%s=%s%s", @@ -6967,6 +6978,7 @@ class ProxyConfig: for name in ( "vector_store_deny_by_default", "search_tool_deny_by_default", + "claude_code_marketplace_auth_required", "vertex_ai_pass_through_drop_anthropic_betas", "request_tag_mappings", ) @@ -7276,6 +7288,13 @@ class ProxyConfig: ## SEARCH TOOLS SETTINGS search_tools: Final[list[SearchToolTypedDict] | None] = self.parse_search_tools(config) + if search_tools: + print_startup_message("\033[32mLiteLLM: Proxy initialized with Search Tools:\033[0m") + for tool in search_tools: + print_startup_message( + f"\033[32m {tool.get('search_tool_name', '')} " + f"({tool.get('litellm_params', {}).get('search_provider', '')})\033[0m" + ) ## SANDBOX TOOLS SETTINGS from litellm.sandbox.sandbox_tools import register_sandbox_tools @@ -9678,12 +9697,9 @@ async def initialize( master_key, \ user_custom_auth, \ prisma_client - from litellm.proxy.common_utils.banner import LITELLM_BANNER, show_banner + from litellm.proxy.common_utils.banner import LITELLM_BANNER - if litellm.json_logs: - print_startup_message(LITELLM_BANNER) - else: - show_banner() + print_startup_message(f"\n{LITELLM_BANNER}\n") if os.getenv("LITELLM_DONT_SHOW_FEEDBACK_BOX", "").lower() != "true": generate_feedback_box() user_model = model @@ -9825,7 +9841,7 @@ async def async_assistants_data_generator(response, user_api_key_dict: UserAPIKe request_data=request_data, ) verbose_proxy_logger.debug( - "\x1b[1;31mAn error occurred: %s\n\n Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`", + "An error occurred: %s", e, ) if isinstance(e, HTTPException): @@ -10655,7 +10671,7 @@ async def async_data_generator( ) client_error: Final[Exception] = e if replacement is None else replacement verbose_proxy_logger.debug( - "\x1b[1;31mAn error occurred: %s\n\n Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`", + "An error occurred: %s", e, ) @@ -12390,6 +12406,12 @@ async def model_list( hidden_names: Final = blocked_names | unhealthy_names + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import typesafe_allowlist_model_names + + asks_for_allowlist_names: Final = ( + include_model_access_groups is True and return_wildcard_routes is True and only_model_access_groups is not True + ) + # If scope=expand and user has admin privileges, return all proxy models if should_expand_scope: # Get all proxy models as if user is a proxy admin @@ -12410,7 +12432,10 @@ async def model_list( all_models = get_complete_model_list( key_models=[], team_models=[], - proxy_model_list=proxy_model_list, + proxy_model_list=[ + *proxy_model_list, + *(typesafe_allowlist_model_names() if asks_for_allowlist_names else ()), + ], user_model=None, infer_model_from_keys=False, return_wildcard_routes=include_wildcard_routes, @@ -12471,7 +12496,7 @@ async def model_list( # Otherwise, use the normal behavior (current implementation) # Get available models for the user - all_models = await get_available_models_for_user( + available_models: Final = await get_available_models_for_user( user_api_key_dict=user_api_key_dict, llm_router=llm_router, general_settings=general_settings, @@ -12484,6 +12509,11 @@ async def model_list( return_wildcard_routes=include_wildcard_routes, user_api_key_cache=user_api_key_cache, ) + offers_typesafe_allowlist_names: Final = ( + asks_for_allowlist_names and team_id is None and user_api_key_has_admin_view(user_api_key_dict) + ) + typesafe_names_to_offer: Final = typesafe_allowlist_model_names() if offers_typesafe_allowlist_names else () + all_models = [*available_models, *(name for name in typesafe_names_to_offer if name not in available_models)] undiscoverable_names: Final = undiscoverable_model_names( all_models, llm_router, user_api_key_dict, team_id or user_api_key_dict.team_id @@ -15527,7 +15557,7 @@ async def _count_request_tokens( count_source="local_estimate", ) estimate: Final = await _local_token_estimate( - model=model_to_use, + model=deployment_openai_model(deployment_view) or model_to_use, prompt=prompt if counted_messages is None else None, messages=counted_messages, tools=counted_tools, @@ -18437,7 +18467,6 @@ async def async_queue_request( "body": copy.copy(data), # use copy instead of deepcopy } - verbose_proxy_logger.debug("receiving data: %s", data) data["model"] = ( general_settings.get("completion_model", None) # server default or user_model # model name passed via cli args @@ -18523,25 +18552,13 @@ async def async_queue_request( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - if isinstance(e, HTTPException): - raise ProxyException( - message=getattr(e, "detail", f"Authentication Error({e})"), - type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), - code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), - ) - elif isinstance(e, ProxyException): - raise e - elif isinstance(e, litellm.BadRequestError): + if isinstance(e, litellm.BadRequestError): raise ProxyException( message=e.message, type="invalid_request_error", param=getattr(e, "param", None), code=e.status_code - ) - raise ProxyException( - message="Authentication Error, " + str(e), - type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), - code=status.HTTP_400_BAD_REQUEST, - ) + ) from e + if isinstance(e, ProxyException): + raise + raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e @app.get("/fallback/login", tags=["experimental"], include_in_schema=False) @@ -19820,21 +19837,9 @@ async def update_config( except Exception as e: verbose_proxy_logger.error("litellm.proxy.proxy_server.update_config(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) - if isinstance(e, HTTPException): - raise ProxyException( - message=getattr(e, "detail", f"Authentication Error({e})"), - type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), - code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), - ) - elif isinstance(e, ProxyException): - raise e - raise ProxyException( - message="Authentication Error, " + str(e), - type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), - code=status.HTTP_400_BAD_REQUEST, - ) + if isinstance(e, ProxyException): + raise + raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e ### CONFIG GENERAL SETTINGS @@ -21017,21 +21022,9 @@ async def get_config( } except Exception as e: verbose_proxy_logger.exception("litellm.proxy.proxy_server.get_config(): Exception occured - %s", e) - if isinstance(e, HTTPException): - raise ProxyException( - message=getattr(e, "detail", f"Authentication Error({e})"), - type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), - code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), - ) - elif isinstance(e, ProxyException): - raise e - raise ProxyException( - message="Authentication Error, " + str(e), - type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), - code=status.HTTP_400_BAD_REQUEST, - ) + if isinstance(e, ProxyException): + raise + raise proxy_exception_for(e, default_status_code=status.HTTP_500_INTERNAL_SERVER_ERROR) from e @router.get( diff --git a/litellm/proxy/public_endpoints/agent_create_fields.json b/litellm/proxy/public_endpoints/agent_create_fields.json index f289a6c1805..53b32d249c7 100644 --- a/litellm/proxy/public_endpoints/agent_create_fields.json +++ b/litellm/proxy/public_endpoints/agent_create_fields.json @@ -127,8 +127,8 @@ { "key": "agent_id", "label": "Agent ID", - "placeholder": "asst_abc123", - "tooltip": "The agent/assistant ID from your Azure AI Foundry project (e.g., asst_abc123)", + "placeholder": "my-agent", + "tooltip": "The agent name from your Azure AI Foundry project, optionally with a version (e.g., my-agent or my-agent:2), or a classic assistant ID (e.g., asst_abc123)", "required": true, "field_type": "text", "default_value": null, diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index f2d88702470..4237517508b 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -2023,6 +2023,34 @@ ], "default_model_placeholder": "jina_ai/" }, + { + "provider": "KYMA", + "provider_display_name": "Kyma API", + "litellm_provider": "kyma", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://kymaapi.com/v1", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "kyma/qwen3.7-flash" + }, { "provider": "Laya", "provider_display_name": "Laya", @@ -3519,6 +3547,34 @@ ], "default_model_placeholder": "inferenceapis/deepseek-ai/DeepSeek-V4-Flash" }, + { + "provider": "StreamLake", + "provider_display_name": "StreamLake", + "litellm_provider": "streamlake", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://vanchin.streamlake.ai/api/gateway/v1/endpoints", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "streamlake/" + }, { "provider": "Snowflake", "provider_display_name": "Snowflake", @@ -4200,5 +4256,33 @@ } ], "default_model_placeholder": "cursor/claude-4-sonnet" + }, + { + "provider": "BIGMODEL", + "provider_display_name": "BigModel (Zhipu AI)", + "litellm_provider": "bigmodel", + "credential_fields": [ + { + "key": "api_key", + "label": "BigModel (Zhipu AI) API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + }, + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://open.bigmodel.cn", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": "https://open.bigmodel.cn" + } + ], + "default_model_placeholder": "bigmodel/glm-5.3" } ] diff --git a/litellm/proxy/public_endpoints/public_endpoints.py b/litellm/proxy/public_endpoints/public_endpoints.py index 9f33e0d2adc..b8cdea0dea4 100644 --- a/litellm/proxy/public_endpoints/public_endpoints.py +++ b/litellm/proxy/public_endpoints/public_endpoints.py @@ -367,19 +367,23 @@ async def get_mcp_servers(): tags=["public", "Claude Code Marketplace"], ) async def public_skill_hub(): - """Return enabled (public) Claude Code skills — no auth required.""" + """Return enabled (public) Claude Code skills, or none when the marketplace requires a key.""" from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_marketplace import ( MANIFEST_FIELDS, get_installation_preference, - get_prisma_client, ) + from litellm.proxy.common_utils.claude_code_marketplace_auth import marketplace_auth_required + from litellm.proxy.proxy_server import prisma_client from litellm.types.proxy.claude_code_endpoints import ( ListPluginsResponse, PluginListItem, ) + if marketplace_auth_required(): + return ListPluginsResponse(plugins=[], count=0) + if prisma_client is None: + raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) try: - prisma_client: Final = await get_prisma_client() plugins: Final = await _plugin_table(prisma_client).find_many(where={"enabled": True}) items: Final = [] for plugin in plugins: @@ -406,7 +410,8 @@ async def public_skill_hub(): ) return ListPluginsResponse(plugins=items, count=len(items)) except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + verbose_logger.exception("Error listing public skills: %s", e) + raise HTTPException(status_code=500, detail="Failed to list skills") @router.get( diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 4f8b1697f7e..9e642a02d49 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -39,6 +39,7 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping, get_metadata from litellm.litellm_core_utils.internal_call_metadata import is_background_response, requests_background from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup +from litellm.litellm_core_utils.token_counter import usage_tokenizer_model from litellm.litellm_core_utils.tool_fields import TOOL_FIELDS, first_validation_error from litellm.llms.openai.responses.guardrail_translation.handler import ( OpenAIResponsesHandler, @@ -85,6 +86,7 @@ from litellm.proxy.route_llm_request import raise_if_required_body_param_missing from litellm.proxy.utils import count_request_input_tokens, handle_exception_on_proxy from litellm.responses.litellm_completion_transformation.compaction_items import messages_since_newest_compaction from litellm.responses.litellm_completion_transformation.count_request import MODEL_FILE_ID_MAPPING_FIELD +from litellm.responses.retrieve_stream_query import RETRIEVE_INCLUDE_QUERY_KEYS, RETRIEVE_STREAM_QUERY_KEYS from litellm.responses.streaming_iterator import GuardedResponsesWsFrame, ResponsesWsFrameGuard, spawn_logging_task from litellm.router import Router from litellm.types.llms.base import LiteLLMBaseModel @@ -96,6 +98,7 @@ from litellm.types.llms.openai import ( from litellm.types.responses.main import DeleteResponseResult from litellm.types.token_count import TokenCountSourceFields from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, TokenCountResponse +from litellm.utils import get_base_model_from_metadata if TYPE_CHECKING: from litellm.proxy.utils import ProxyLogging @@ -104,11 +107,43 @@ router: Final = APIRouter() _RESPONSES_WS_CONFIG_VALUE_ADAPTER: Final[TypeAdapter[object | None]] = TypeAdapter(object | None) _DECODED_JSON: Final = TypeAdapter(object) _PROXY_SERVER_REQUEST_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) +_RETRIEVE_QUERY_PARAMS: Final = tuple(sorted(RETRIEVE_INCLUDE_QUERY_KEYS | RETRIEVE_STREAM_QUERY_KEYS)) +_RETRIEVE_QUERY_OPENAPI: Final = { + "parameters": [ + { + "name": "stream", + "in": "query", + "required": False, + "schema": {"type": "boolean"}, + "description": "Replay the response's events as server-sent events (OpenAI, Azure, and Azure AI Foundry)", + }, + { + "name": "starting_after", + "in": "query", + "required": False, + "schema": {"type": "integer", "minimum": 0}, + "description": "With stream=true, replay only the events after this sequence number", + }, + { + "name": "include", + "in": "query", + "required": False, + "schema": {"type": "array", "items": {"type": "string"}}, + "description": "Additional output data to include, forwarded to the provider", + }, + { + "name": "include_obfuscation", + "in": "query", + "required": False, + "schema": {"type": "boolean"}, + "description": "With stream=true, whether delta events carry obfuscation padding", + }, + ] +} _ResponseDocSchemas: TypeAlias = dict[int | str, dict[str, object]] # fastapi's responses kwarg -RESPONSES_API_RESPONSE_SCHEMAS: Final[_ResponseDocSchemas] = {200: {"model": ResponsesAPIResponse}} -RESPONSES_API_CREATE_RESPONSE_SCHEMAS: Final[_ResponseDocSchemas] = { +RESPONSES_API_RESPONSE_SCHEMAS: Final[_ResponseDocSchemas] = { 200: { "model": ResponsesAPIResponse, "content": { @@ -252,19 +287,19 @@ async def _resolve_cursor_model_variant_before_auth(request: Request) -> None: "/v1/responses", dependencies=[Depends(user_api_key_auth)], tags=["responses"], - responses=RESPONSES_API_CREATE_RESPONSE_SCHEMAS, + responses=RESPONSES_API_RESPONSE_SCHEMAS, ) @router.post( "/responses", dependencies=[Depends(user_api_key_auth)], tags=["responses"], - responses=RESPONSES_API_CREATE_RESPONSE_SCHEMAS, + responses=RESPONSES_API_RESPONSE_SCHEMAS, ) @router.post( "/openai/v1/responses", dependencies=[Depends(user_api_key_auth)], tags=["responses"], - responses=RESPONSES_API_CREATE_RESPONSE_SCHEMAS, + responses=RESPONSES_API_RESPONSE_SCHEMAS, ) async def responses_api( request: Request, @@ -537,7 +572,10 @@ async def cursor_model_list( def _dispatched_prompt_token_counter(logging_obj: LiteLLMLoggingObj) -> Callable[[], int]: return lambda: count_request_input_tokens( - model=str(logging_obj.model_call_details.get("model") or ""), + model=usage_tokenizer_model( + str(logging_obj.model_call_details.get("model") or ""), + get_base_model_from_metadata(logging_obj.model_call_details), + ), request_input=messages_since_newest_compaction(logging_obj.model_call_details.get("messages")), system_input=None, ) @@ -744,18 +782,21 @@ async def cursor_chat_completions( dependencies=[Depends(user_api_key_auth)], tags=["responses"], responses=RESPONSES_API_RESPONSE_SCHEMAS, + openapi_extra=_RETRIEVE_QUERY_OPENAPI, ) @router.get( "/responses/{response_id}", dependencies=[Depends(user_api_key_auth)], tags=["responses"], responses=RESPONSES_API_RESPONSE_SCHEMAS, + openapi_extra=_RETRIEVE_QUERY_OPENAPI, ) @router.get( "/openai/v1/responses/{response_id}", dependencies=[Depends(user_api_key_auth)], tags=["responses"], responses=RESPONSES_API_RESPONSE_SCHEMAS, + openapi_extra=_RETRIEVE_QUERY_OPENAPI, ) async def get_response( response_id: str, @@ -780,6 +821,10 @@ async def get_response( # Get provider response curl -X GET http://localhost:4000/v1/responses/resp_abc123 \ -H "Authorization: Bearer $LITELLM_MASTER_KEY" + + # Resume a background response's event stream after sequence number 7 + curl -N "http://localhost:4000/v1/responses/resp_abc123?stream=true&starting_after=7" \ + -H "Authorization: Bearer $LITELLM_MASTER_KEY" ``` """ from litellm.proxy.proxy_server import ( @@ -827,6 +872,14 @@ async def get_response( data: Final = await read_request_body(request=request) read_settles_billing: Final = not carries_client_side_overrides(data) data["response_id"] = response_id + retrieve_query: Final = { + key: values[0] if len(values) == 1 else values + for key in _RETRIEVE_QUERY_PARAMS + if (values := request.query_params.getlist(key)) + } + if retrieve_query: + body_extra_query: Final[object] = data.get("extra_query") + data["extra_query"] = {**(body_extra_query if isinstance(body_extra_query, dict) else {}), **retrieve_query} processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: response: Final[object] = await processor.base_process_llm_request( @@ -838,7 +891,12 @@ async def get_response( llm_router=llm_router, general_settings=general_settings, proxy_config=proxy_config, - select_data_generator=select_data_generator, + select_data_generator=partial( + _streamed_retrieval_data_generator, + partial(select_data_generator, responses_stream_errors=True), + response_id, + read_settles_billing, + ), model=None, user_model=user_model, user_temperature=user_temperature, @@ -912,12 +970,24 @@ async def _bill_finished_background_response( fastapi_response: Response, read_settles_billing: bool, ) -> None: + _report_charge( + await _settle_finished_background_read(response, response_id, user_api_key_dict, read_settles_billing), + fastapi_response, + ) + + +async def _settle_finished_background_read( + response: object, + response_id: str, + user_api_key_dict: UserAPIKeyAuth, + read_settles_billing: bool, +) -> "_BilledCharge | None": from litellm.proxy.proxy_server import proxy_logging_obj if not isinstance(response, ResponsesAPIResponse) or not is_background_response(response): - return + return None if response.status not in TERMINAL_RESPONSE_STATUSES: - return + return None managed_files_hook: Final = _managed_files_hook(proxy_logging_obj) track_untracked: Final = ( partial( @@ -928,12 +998,11 @@ async def _bill_finished_background_response( if managed_files_hook is not None else None ) - charge: Final = await _settle_background_billing( + return await _settle_background_billing( (lambda cost_check: cost_check.bill_if_finished(response_id, response, track_untracked)) if read_settles_billing else (lambda cost_check: cost_check.bill_from_deployment(response_id, track_untracked)) ) - _report_charge(charge, fastapi_response) class _BilledCharge(Protocol): @@ -1083,27 +1152,53 @@ def _streamed_background_data_generator( user_api_key_dict=user_api_key_dict, request_tags=_background_response_spend_tags(request_data), ) - return _bill_streamed_background_response(response, chunks, track_untracked) + return _bill_streamed_background_response( + response, chunks, partial(_bill_finished_stream, track_untracked=track_untracked) + ) + + +def _streamed_retrieval_data_generator( + select_data_generator: Callable[..., AsyncIterator[str | bytes]], + response_id: str, + read_settles_billing: bool, + response: object, + user_api_key_dict: UserAPIKeyAuth, + request_data: Mapping[str, object], + request: Request | None = None, +) -> AsyncIterator[str | bytes]: + chunks: Final = select_data_generator( + response=response, user_api_key_dict=user_api_key_dict, request_data=request_data, request=request + ) + return _bill_streamed_background_response( + response, + chunks, + partial( + _settle_finished_background_read, + response_id=response_id, + user_api_key_dict=user_api_key_dict, + read_settles_billing=read_settles_billing, + ), + ) async def _bill_streamed_background_response( stream: object, chunks: AsyncIterator[str | bytes], - track_untracked: Callable[[str, ResponsesAPIResponse], Awaitable[bool]], + bill: Callable[[ResponsesAPIResponse], Awaitable[object]], ) -> AsyncIterator[str | bytes]: async for chunk in chunks: completed: object = ProxyBaseLLMRequestProcessing.extract_completed_responses_response(stream) if not isinstance(completed, ResponsesAPIResponse): yield chunk continue - await _bill_finished_stream(completed, track_untracked) + await bill(completed) yield chunk async for remaining_chunk in chunks: yield remaining_chunk return late_completed: Final = ProxyBaseLLMRequestProcessing.extract_completed_responses_response(stream) if isinstance(late_completed, ResponsesAPIResponse): - await _bill_finished_stream(late_completed, track_untracked) + await bill(late_completed) async def _bill_finished_stream( diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index 0b91e57f01e..08840b44532 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -7,6 +7,7 @@ import time from collections.abc import AsyncIterator, Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timedelta, timezone +from itertools import chain from types import MappingProxyType from typing import Final, NoReturn, SupportsFloat, SupportsIndex, SupportsInt, cast @@ -39,7 +40,7 @@ from litellm.proxy.common_utils.user_api_key_cache import ( tag_cache_key, team_membership_reservation_cache_key, ) -from litellm.proxy.spend_tracking.input_tokens import count_input_tokens, count_input_tokens_for_model +from litellm.proxy.spend_tracking.input_tokens import count_input_tokens, count_input_tokens_for_model, tokenizer_models from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncrement, spend_counter_batch_scope from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.router import Router @@ -1599,7 +1600,19 @@ async def count_request_input_tokens( models: Final = _get_request_models(request_body=request_body, route=route, llm_router=llm_router) if not models: return MappingProxyType({}) - return await count_input_tokens(request_body=request_body, raw_body=raw_body, models=models) + counted_as: Final = MappingProxyType({model: tokenizer_models(model, llm_router) for model in models}) + counts: Final = await count_input_tokens( + request_body=request_body, + raw_body=raw_body, + models=tuple(dict.fromkeys(chain.from_iterable(counted_as.values()))), + ) + return MappingProxyType( + { + model: count + for model, names in counted_as.items() + if (count := max((counts[name] for name in names if name in counts), default=None)) is not None + } + ) def _estimate_input_tokens( diff --git a/litellm/proxy/spend_tracking/input_tokens.py b/litellm/proxy/spend_tracking/input_tokens.py index 927152296e9..0139869c6c6 100644 --- a/litellm/proxy/spend_tracking/input_tokens.py +++ b/litellm/proxy/spend_tracking/input_tokens.py @@ -15,14 +15,16 @@ import asyncio import json from collections.abc import Mapping, Sequence from types import MappingProxyType -from typing import Final +from typing import TYPE_CHECKING, Final import litellm from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.token_counter import served_openai_models, unresolved_tokenizer_model from litellm.responses.litellm_completion_transformation.compaction_items import ( billable_input_since_newest_compaction, holds_compaction_summary, ) +from litellm.router_utils.common_utils import resolve_model_group_alias from litellm.rust_bridge import runtime from litellm.rust_bridge.catalog import Route, RouteContext from litellm.rust_bridge.token_counter import ( @@ -33,6 +35,9 @@ from litellm.rust_bridge.token_counter import ( rust_tokenizer, ) +if TYPE_CHECKING: + from litellm.router import Router + TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS: Final = 30_000 _INPUT_SIZE_FIELDS: Final = ("messages", "prompt", "input", "query", "documents", "tools", "tool_choice") @@ -47,6 +52,25 @@ def _approximate_input_size(request_body: Mapping[str, object]) -> int: return sum(len(str(request_body.get(field, ""))) for field in _INPUT_SIZE_FIELDS) +def tokenizer_models(model: str, llm_router: Router | None) -> tuple[str, ...]: + """The names to count a request for `model` with, keeping the largest count. + + An alias resolves to its target group. A group served only by OpenAI models counts as those models, so it gets + their tokenizer. Any other group counts as its OpenAI models, if any, and as the requested name with the encoding + tiktoken gives that name as written, so its other deployments keep that count without loading their vendor's + tokenizer. A name the router does not index counts as written.""" + if llm_router is None: + return (model,) + group: Final = resolve_model_group_alias(llm_router.model_group_alias, model) or model + served: Final = served_openai_models(llm_router.deployments_named(group) or ()) + if not served: + return (model,) + openai_served: Final = tuple(served_model for served_model in served if served_model is not None) + if len(openai_served) == len(served): + return openai_served + return (unresolved_tokenizer_model(model), *openai_served) + + async def count_input_tokens( request_body: dict, raw_body: bytes | None, diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 94b079163e9..833ee71ae9c 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -80,7 +80,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( SETTLES_BACKGROUND_RESPONSE_KWARG, BackgroundResponseSettlement, ) -from litellm.types.router import DeploymentTypedDict, LiteLLM_Params +from litellm.types.router import DeploymentTypedDict, LiteLLM_Params, SilentExperimentFlag from litellm.types.utils import ( BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, PROMPT_CARRYING_GUARDRAIL_FIELDS, @@ -194,6 +194,7 @@ _STAMPED_METADATA_KEYS: Final = frozenset( "used_client_oauth_token", KUBERNETES_POD_ROUTING_KEY, "litellm_roi_estimator", + "is_silent_experiment", BILLED_RETRY_ATTEMPT_METADATA_KEY, "user_api_key_user_email", ) @@ -227,6 +228,7 @@ def _get_spend_logs_metadata( azure_spillover: AzureSpillover | None = None, used_client_oauth_token: bool | None = None, kubernetes_pod_routing: StandardLoggingKubernetesPodRouting | None = None, + is_silent_experiment: bool = False, ) -> SpendLogsMetadata: if metadata is None: return SpendLogsMetadata( @@ -294,6 +296,7 @@ def _get_spend_logs_metadata( used_client_oauth_token=used_client_oauth_token, kubernetes_pod_routing=kubernetes_pod_routing, litellm_roi_estimator=metadata.get("litellm_roi_estimator") is True, + **(SilentExperimentFlag(is_silent_experiment=True) if is_silent_experiment else SilentExperimentFlag()), billed_retry_attempt=billed_retry_attempt, user_api_key_user_email=_stored_user_email(metadata), ) @@ -738,6 +741,11 @@ def get_logging_payload( # clean up litellm metadata clean_metadata = _get_spend_logs_metadata( persisted_metadata, + is_silent_experiment=( + standard_logging_payload is not None + and "is_silent_experiment" in standard_logging_payload["metadata"] + and standard_logging_payload["metadata"]["is_silent_experiment"] + ), applied_guardrails=( standard_logging_payload["metadata"].get("applied_guardrails", None) if standard_logging_payload is not None diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 0ddfc5ce62f..9f58c267231 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -168,7 +168,7 @@ from litellm.litellm_core_utils.served_output_texts import ( record_served_output_texts, served_stream_output_texts, ) -from litellm.litellm_core_utils.token_counter import offload_token_count +from litellm.litellm_core_utils.token_counter import offload_token_count, usage_tokenizer_model from litellm.llms import load_guardrail_translation_mappings from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.proxy._types import ( @@ -1139,7 +1139,9 @@ def _estimated_prompt_usage( None, ) return _estimate_dispatched_failure_usage( - model=str(model_call_details.get("model") or ""), + model=usage_tokenizer_model( + str(model_call_details.get("model") or ""), get_base_model_from_metadata(model_call_details) + ), request_input=( messages_since_newest_compaction(dispatched_input) if dispatched_call_type in _RESPONSES_CALL_TYPES diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 7b9068ff95e..e9b534bdd89 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -31,7 +31,12 @@ from litellm.types.realtime import ( ) from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import CallTypes, LlmProviders -from litellm.utils import ProviderConfigManager, resolve_api_key_secret_reference +from litellm.utils import ( + ModelInfoLookupMiss, + ProviderConfigManager, + lookup_model_info_for_registration, + resolve_api_key_secret_reference, +) from ..litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params from ..litellm_core_utils.credential_accessor import CredentialAccessor @@ -42,8 +47,13 @@ from ..llms.azure.common_utils import get_azure_ad_token from ..llms.azure.realtime.handler import AzureOpenAIRealtime, azure_realtime_protocol_for_client from ..llms.bedrock.realtime.handler import BedrockRealtime, raise_if_session_committed from ..llms.custom_httpx.http_handler import realtime_ssl_for_url +from ..llms.openai.common_utils import is_openai_backed_api_base from ..llms.openai.live.handler import OpenAILiveSessions -from ..llms.openai.realtime.handler import OpenAIRealtime +from ..llms.openai.realtime.handler import ( + OpenAIRealtime, + confirm_gateway_session_started, + confirm_session_started, +) from ..llms.openai.workload_identity import openai_static_api_key, resolve_openai_bearer_token_async from ..llms.vertex_ai.audio_transcription.realtime_transformation import is_vertex_speech_to_text_model from ..llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig, vertex_realtime_config @@ -740,6 +750,18 @@ def _is_transcription_only_realtime_model(model: str, custom_llm_provider: str) return "/v1/realtime/transcription_sessions" in (model_info.get("supported_endpoints") or ()) +_REALTIME_SERVED_ENDPOINTS: Final = ("/v1/realtime", "/v1/audio/transcriptions", "/v1/live") +_GATEWAY_FIRST_EVENT_WAIT_SECONDS: Final = 3.0 + + +def _cost_map_lists_outside_realtime(model: str) -> bool: + model_info: Final = lookup_model_info_for_registration(model, "openai") + endpoints: Final = ( + () if isinstance(model_info, ModelInfoLookupMiss) else model_info.get("supported_endpoints") or () + ) + return bool(endpoints) and not any(endpoint.startswith(_REALTIME_SERVED_ENDPOINTS) for endpoint in endpoints) + + _TRANSCRIPTION_QUERY_PARAMS: Final[RealtimeQueryParams] = {"intent": "transcription"} @@ -785,6 +807,7 @@ async def realtime_health_check( api_version: str | None = None, realtime_protocol: str | None = None, model_params: Mapping[str, object] | None = None, + gateway_first_event_wait_seconds: float = _GATEWAY_FIRST_EVENT_WAIT_SECONDS, ) -> bool: """ Health check for realtime API - tries connection to the realtime API websocket @@ -800,9 +823,16 @@ async def realtime_health_check( without the OpenAI-Beta header is bridged to, with transcription-only models adding intent=transcription Returns: - bool - True if connection is successful, False otherwise + bool - True once the connection is open, and on OpenAI's own hosts once the first server event is not an + error, since OpenAI accepts the websocket handshake with missing or invalid credentials and only reports + the failure in its first server event. Behind any other OpenAI api_base the first event is read for + `gateway_first_event_wait_seconds` only, and a gateway that stays silent that long counts as healthy. + An `invalid_model` error also counts as healthy unless litellm's + cost map lists only endpoints realtime never serves for the model: OpenAI checks the key first, and Live + and transcription models are served through other endpoints. The listed `mode` is not read, since a + deployment's own `mode: realtime` replaces it when the router registers the deployment Raises: - Exception - if the connection is not successful + Exception - if the connection is not successful or if OpenAI's first server event is any other error """ import websockets @@ -822,10 +852,11 @@ async def realtime_health_check( str | None, api_version or resolved_params.get("api_version") ) url: str | None = None + openai_api_base: Final = resolved_api_base or litellm.api_base or "https://api.openai.com/" health_check_api_key: Final = ( await resolve_openai_bearer_token_async( static_api_key=openai_static_api_key(resolved_api_key), - api_base=resolved_api_base, + api_base=openai_api_base, litellm_params=resolved_params, ) if custom_llm_provider == LlmProviders.OPENAI.value @@ -850,10 +881,7 @@ async def realtime_health_check( query_params=azure_query_params, ) elif custom_llm_provider == "openai": - url = openai_realtime.construct_url( - api_base=resolved_api_base or "https://api.openai.com/", - query_params={"model": model}, - ) + url = openai_realtime.construct_url(api_base=openai_api_base, query_params={"model": model}) elif custom_llm_provider == "xai": url = xai_realtime.construct_url( api_base=resolved_api_base or "https://api.x.ai/v1", query_params={"model": model} @@ -899,8 +927,23 @@ async def realtime_health_check( additional_headers=auth_headers, max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, ssl=realtime_ssl_for_url(url), - ): - return True + ) as connection: + if custom_llm_provider != "openai": + return True + if not is_openai_backed_api_base(url): + return await confirm_gateway_session_started( + connection, + model, + sent_credentials=bool(auth_headers), + model_listed_outside_realtime=_cost_map_lists_outside_realtime(model), + first_event_wait_seconds=gateway_first_event_wait_seconds, + ) + return await confirm_session_started( + connection, + model, + sent_credentials=bool(auth_headers), + model_listed_outside_realtime=_cost_map_lists_outside_realtime(model), + ) _realtime_health_check = realtime_health_check diff --git a/litellm/responses/additional_tools.py b/litellm/responses/additional_tools.py index 4b897ff58c0..ce530447c0d 100644 --- a/litellm/responses/additional_tools.py +++ b/litellm/responses/additional_tools.py @@ -1,5 +1,8 @@ -from collections.abc import Sequence +import json +from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass +from itertools import groupby +from types import MappingProxyType from typing import Final, cast # noqa: TID251 # validating the openai tool union strips vendor keys from raw tools from pydantic import ValidationError @@ -15,8 +18,8 @@ class _InputItemType(LiteLLMBaseModel): type: str = "" -class _AdditionalToolsItem(LiteLLMBaseModel): - tools: tuple[dict[str, object], ...] = () +class _ToolList(LiteLLMBaseModel): + tools: tuple[Mapping[str, object], ...] = () @dataclass(frozen=True, slots=True) @@ -33,12 +36,54 @@ def _is_additional_tools_item(item: object) -> bool: return False -def _tools_of_item(item: object) -> tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]: +def _tool_list_of(value: object) -> tuple[Mapping[str, object], ...]: try: - parsed: Final = _AdditionalToolsItem.model_validate(item) + return _ToolList.model_validate(value).tools except ValidationError: return () - return tuple(cast("ALL_RESPONSES_API_TOOL_PARAMS", tool) for tool in parsed.tools) + + +def _tools_of_item(item: object) -> tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]: + return tuple(cast("ALL_RESPONSES_API_TOOL_PARAMS", tool) for tool in _tool_list_of(item)) + + +def _tool_identity(tool: Mapping[str, object]) -> str: + name: Final = tool.get("name") + if isinstance(name, str) and name: + return f"{tool.get('type')}:{name}" + return json.dumps(tool, sort_keys=True, default=str) + + +def _members_of_copies(copies: Sequence[Mapping[str, object]]) -> Iterator[Mapping[str, object]]: + for namespace in copies: + yield from _tool_list_of(namespace) + + +def _merged(copies: Sequence[Mapping[str, object]]) -> Mapping[str, object]: + newest: Final = copies[-1] + if newest.get("type") != "namespace" or len(copies) == 1: + return newest + members: Final = tuple(_members_of_copies(copies)) + return {**newest, "tools": list(_unique_tools(members))} if members else newest + + +def _identity_of(keyed_tool: tuple[str, Mapping[str, object]]) -> str: + return keyed_tool[0] + + +def _unique_tools(tools: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, object], ...]: + """Codex re-sends its tool list in the input on every turn, so the same tool arrives many times. + Keep each tool once, at its first position, with the newest copy's definition, and merge the + members of a namespace sent more than once.""" + keyed: Final = tuple((_tool_identity(tool), tool) for tool in tools) + copies_by_identity: Final = MappingProxyType( + { + identity: tuple(tool for _, tool in group) + for identity, group in groupby(sorted(keyed, key=_identity_of), key=_identity_of) + } + ) + first_seen_order: Final = tuple(dict.fromkeys(identity for identity, _ in keyed)) + return tuple(_merged(copies_by_identity[identity]) for identity in first_seen_order) def hoist_additional_tools( @@ -58,4 +103,8 @@ def hoist_additional_tools( len(items), ) remaining_input: Final = [item for item in input if not _is_additional_tools_item(item)] - return HoistedAdditionalTools(input=remaining_input, tools=(*existing, *hoisted), hoisted=hoisted) + unique: Final = tuple( + cast("ALL_RESPONSES_API_TOOL_PARAMS", tool) # cast-ok: a merged namespace is rebuilt from the same tool dicts + for tool in _unique_tools((*existing, *hoisted)) + ) + return HoistedAdditionalTools(input=remaining_input, tools=unique, hoisted=hoisted) diff --git a/litellm/responses/bridge_check.py b/litellm/responses/bridge_check.py index 504150b330c..826de8cfdc9 100644 --- a/litellm/responses/bridge_check.py +++ b/litellm/responses/bridge_check.py @@ -9,6 +9,7 @@ from litellm.llms.azure_ai.chat.transformation import AzureAIGPT5Config from litellm.llms.azure_ai.common_utils import ( azure_ai_supports_native_responses, foundry_chat_rejects_function_tools_while_reasoning, + is_foundry_project_agent, ) from litellm.llms.openai.chat.gpt_reasoning_transformation import OpenAIGPT5Config from litellm.secret_managers.main import get_secret_str @@ -181,6 +182,9 @@ def responses_api_bridge_check( elif pinned_mode == "chat" and backend_serves_chat_completions(model_info): model_info["mode"] = "chat" + if custom_llm_provider == LlmProviders.AZURE_AI.value and is_foundry_project_agent(model): + model_info["mode"] = "responses" + if web_search_options is not None and custom_llm_provider == "xai": model_info["mode"] = "responses" model = model.replace("responses/", "") diff --git a/litellm/responses/litellm_completion_transformation/client_tool_search.py b/litellm/responses/litellm_completion_transformation/client_tool_search.py new file mode 100644 index 00000000000..030bb014b7a --- /dev/null +++ b/litellm/responses/litellm_completion_transformation/client_tool_search.py @@ -0,0 +1,268 @@ +import json +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from itertools import chain +from types import MappingProxyType +from typing import Final, cast # noqa: TID251 # loaded tool specs are forwarded as the client sent them + +from openai.types.responses.response_input_param import ( + FunctionCallOutput, + ResponseFunctionToolCallParam, + ResponseInputItemParam, +) +from pydantic import ConfigDict, TypeAdapter, ValidationError + +from litellm.types.llms.base import LiteLLMBaseModel +from litellm.types.llms.openai import ( + ALL_RESPONSES_API_TOOL_PARAMS, + ChatCompletionToolParam, + ChatCompletionToolParamFunctionChunk, + ResponseInputParam, +) + +from .custom_tools import function_chat_tool +from .tool_search import CLIENT_TOOL_SEARCH_FUNCTION_NAME, has_client_tool_search + +__all__ = [ + "ClientToolSearchBridge", + "bridge_client_tool_search", + "client_tool_search_chat_tool", + "client_tool_search_loaded_tools", +] + +_DEFAULT_SEARCH_DESCRIPTION: Final = ( + "Search the deferred tools by keyword. The matching tools are loaded and become callable on the next turn." +) +_DEFAULT_SEARCH_PARAMETERS: Final = MappingProxyType( + { + "type": "object", + "properties": { + "query": {"type": "string", "description": "Search query for deferred tools."}, + "limit": {"type": "integer", "description": "Maximum number of tools to return. Defaults to 8."}, + }, + "required": ["query"], + "additionalProperties": False, + } +) + + +class _ClientToolSearchItem(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + type: str = "" + execution: str | None = None + call_id: str | None = None + arguments: object = None + tools: tuple[Mapping[str, object], ...] = () + + +class _ToolSpec(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + type: str = "" + name: str = "" + description: str = "" + defer_loading: object = None + tools: tuple[Mapping[str, object], ...] = () + + +class _ClientToolSearchTool(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + description: str = "" + parameters: Mapping[str, object] | None = None + + +@dataclass(frozen=True, slots=True) +class _LoadedTool: + namespace: str | None + namespace_description: str + spec: Mapping[str, object] + + @property + def callable_name(self) -> str: + name: Final = str(self.spec.get("name") or "") + return name if self.namespace is None else f"{self.namespace}__{name}" + + +@dataclass(frozen=True, slots=True) +class ClientToolSearchBridge: + input: str | ResponseInputParam + tools: tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...] + bridged: bool + + +def client_tool_search_chat_tool(tool: Mapping[str, object]) -> ChatCompletionToolParam: + spec: Final = _ClientToolSearchTool.model_validate(tool) + function: Final = ChatCompletionToolParamFunctionChunk( + name=CLIENT_TOOL_SEARCH_FUNCTION_NAME, + description=spec.description or _DEFAULT_SEARCH_DESCRIPTION, + parameters=dict(spec.parameters or _DEFAULT_SEARCH_PARAMETERS), + ) + return function_chat_tool(function, None, False) + + +def _client_tool_search_item(item: object) -> _ClientToolSearchItem | None: + try: + parsed: Final = _ClientToolSearchItem.model_validate(item) + except ValidationError: + return None + if parsed.type not in ("tool_search_call", "tool_search_output") or parsed.execution != "client": + return None + return parsed + + +_INPUT_ITEMS: Final = TypeAdapter(tuple[object, ...]) + + +def _searched_tools(item: object) -> tuple[Mapping[str, object], ...]: + parsed: Final = _client_tool_search_item(item) + if parsed is None or parsed.type != "tool_search_output" or not parsed.call_id: + return () + return parsed.tools + + +def client_tool_search_loaded_tools(input: object) -> tuple[Mapping[str, object], ...]: + if not isinstance(input, (list, tuple)): + return () + return tuple(chain.from_iterable(map(_searched_tools, _INPUT_ITEMS.validate_python(input)))) + + +def _tool_spec(tool: object) -> _ToolSpec | None: + try: + return _ToolSpec.model_validate(tool) + except ValidationError: + return None + + +def _as_tool_param(spec: Mapping[str, object]) -> ALL_RESPONSES_API_TOOL_PARAMS: + return cast("ALL_RESPONSES_API_TOOL_PARAMS", spec) # cast-ok: client tool specs are forwarded as sent + + +def _without_deferral(spec: Mapping[str, object]) -> Mapping[str, object]: + return {key: value for key, value in spec.items() if key != "defer_loading"} + + +def _loaded_tools(tool: Mapping[str, object]) -> tuple[_LoadedTool, ...]: + spec: Final = _tool_spec(tool) + if spec is None: + return () + if spec.type != "namespace": + return (_LoadedTool(namespace=None, namespace_description="", spec=_without_deferral(tool)),) + return tuple( + _LoadedTool(namespace=spec.name, namespace_description=spec.description, spec=_without_deferral(member)) + for member in spec.tools + ) + + +def _all_loaded_tools(tools: Iterable[Mapping[str, object]]) -> tuple[_LoadedTool, ...]: + return tuple(chain.from_iterable(map(_loaded_tools, tools))) + + +def _callable_names(tool: object) -> tuple[str, ...]: + spec: Final = _tool_spec(tool) + if spec is None: + return () + if spec.type != "namespace": + return (spec.name,) + return tuple(f"{spec.name}__{member.get('name') or ''}" for member in spec.tools) + + +def _first_occurrences(loaded: Sequence[_LoadedTool], taken: frozenset[str]) -> tuple[_LoadedTool, ...]: + first_index: Final = {tool.callable_name: index for index, tool in reversed(tuple(enumerate(loaded)))} + return tuple( + tool + for index, tool in enumerate(loaded) + if first_index[tool.callable_name] == index and tool.callable_name not in taken + ) + + +def _regrouped(loaded: Sequence[_LoadedTool]) -> tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]: + namespaces: Final = {tool.namespace: tool.namespace_description for tool in reversed(loaded) if tool.namespace} + ordered_namespaces: Final = tuple(dict.fromkeys(tool.namespace for tool in loaded if tool.namespace)) + top_level: Final = tuple(tool.spec for tool in loaded if tool.namespace is None) + grouped: Final = tuple( + { + "type": "namespace", + "name": namespace, + "description": namespaces[namespace], + "tools": tuple(tool.spec for tool in loaded if tool.namespace == namespace), + } + for namespace in ordered_namespaces + ) + return tuple(_as_tool_param(tool) for tool in (*top_level, *grouped)) + + +def _is_deferred(tool: object) -> bool: + spec: Final = _tool_spec(tool) + return spec is not None and spec.defer_loading is True + + +def _without_deferred(tool: ALL_RESPONSES_API_TOOL_PARAMS) -> ALL_RESPONSES_API_TOOL_PARAMS | None: + spec: Final = _tool_spec(tool) + if spec is None: + return tool + if spec.type != "namespace": + return None if spec.defer_loading is True else tool + eager_members: Final = tuple(member for member in spec.tools if not _is_deferred(member)) + if len(eager_members) == len(spec.tools): + return tool + if not eager_members: + return None + return _as_tool_param({**tool, "tools": eager_members}) + + +def _search_output_text(loaded: Sequence[_LoadedTool]) -> str: + if not loaded: + return "No tools matched the search." + return "Loaded these tools, which are now callable: " + ", ".join(tool.callable_name for tool in loaded) + + +def _search_arguments_json(arguments: object) -> str: + return arguments if isinstance(arguments, str) else json.dumps(arguments or {}) + + +def _bridged_item(item: ResponseInputItemParam) -> ResponseInputItemParam | None: + parsed: Final = _client_tool_search_item(item) + if parsed is None: + return item + if not parsed.call_id: + return None + if parsed.type == "tool_search_call": + return ResponseFunctionToolCallParam( + type="function_call", + call_id=parsed.call_id, + name=CLIENT_TOOL_SEARCH_FUNCTION_NAME, + arguments=_search_arguments_json(parsed.arguments), + ) + return FunctionCallOutput( + type="function_call_output", call_id=parsed.call_id, output=_search_output_text(_all_loaded_tools(parsed.tools)) + ) + + +def bridge_client_tool_search( + input: str | ResponseInputParam, + tools: Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None, + loaded_earlier: Sequence[Mapping[str, object]] = (), +) -> ClientToolSearchBridge: + existing: Final = tuple(tools or ()) + searches_on_client: Final = has_client_tool_search(existing) + items: Final = () if isinstance(input, str) else tuple(input) + search_items: Final = tuple(parsed for item in items if (parsed := _client_tool_search_item(item)) is not None) + if not searches_on_client and not search_items: + return ClientToolSearchBridge(input=input, tools=existing, bridged=False) + eager: Final = ( + tuple(kept for tool in existing if (kept := _without_deferred(tool)) is not None) + if searches_on_client + else existing + ) + loaded: Final = _all_loaded_tools((*loaded_earlier, *client_tool_search_loaded_tools(input))) + taken: Final = frozenset(chain.from_iterable(map(_callable_names, eager))) + bridged_input: Final = ( + input if isinstance(input, str) else [bridged for item in input if (bridged := _bridged_item(item)) is not None] + ) + return ClientToolSearchBridge( + input=bridged_input, + tools=(*eager, *_regrouped(_first_occurrences(loaded, taken))), + bridged=True, + ) diff --git a/litellm/responses/litellm_completion_transformation/count_request.py b/litellm/responses/litellm_completion_transformation/count_request.py index 7679b4ede71..8e8e4066bb8 100644 --- a/litellm/responses/litellm_completion_transformation/count_request.py +++ b/litellm/responses/litellm_completion_transformation/count_request.py @@ -20,7 +20,7 @@ from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) from litellm.responses.main import previous_response_client_ids -from litellm.types.llms.openai import ResponseInputParam +from litellm.types.llms.openai import ALL_RESPONSES_API_TOOL_PARAMS, ResponseInputParam from litellm.utils import ProviderConfigManager _COUNTED_CHAT_PARAMS: Final = ( @@ -119,14 +119,13 @@ def _with_hoisted_additional_tools(responses_request: Mapping[str, object]) -> M if not isinstance(responses_input, list): return responses_request input_items: Final = cast(ResponseInputParam, responses_input) # cast-ok: forwarded as sent, as inference does - hoisted: Final = hoist_additional_tools(input_items, None) + top_level_tools: Final = cast( # cast-ok: tools are forwarded as sent, as inference does + "tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]", _ITEMS.validate_python(responses_request.get("tools") or ()) + ) + hoisted: Final = hoist_additional_tools(input_items, top_level_tools) if not hoisted.hoisted: return responses_request - return { - **responses_request, - "input": hoisted.input, - "tools": [*_ITEMS.validate_python(responses_request.get("tools") or ()), *hoisted.hoisted], - } + return {**responses_request, "input": hoisted.input, "tools": list(hoisted.tools)} def _with_emulated_file_search(responses_request: Mapping[str, object]) -> Mapping[str, object]: diff --git a/litellm/responses/litellm_completion_transformation/handler.py b/litellm/responses/litellm_completion_transformation/handler.py index ea3bda1b8b5..141e97d902d 100644 --- a/litellm/responses/litellm_completion_transformation/handler.py +++ b/litellm/responses/litellm_completion_transformation/handler.py @@ -2,17 +2,21 @@ Handler for transforming responses api requests to litellm.completion requests """ -from collections.abc import Coroutine, Mapping +from collections.abc import Coroutine, Mapping, Sequence +from dataclasses import dataclass from typing import Final from pydantic import TypeAdapter import litellm -from litellm.responses.additional_tools import hoist_additional_tools +from litellm.responses.additional_tools import HoistedAdditionalTools, hoist_additional_tools +from litellm.responses.litellm_completion_transformation.client_tool_search import bridge_client_tool_search from litellm.responses.litellm_completion_transformation.compaction_items import holds_compaction_summary +from litellm.responses.litellm_completion_transformation.session_handler import ResponsesSessionHandler, StoredTurns from litellm.responses.litellm_completion_transformation.streaming_iterator import ( LiteLLMCompletionStreamingIterator, ) +from litellm.responses.litellm_completion_transformation.tool_search import has_client_tool_search from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, with_compaction_items_as_params, @@ -28,6 +32,28 @@ from litellm.types.utils import ModelResponse _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(dict[str, object]) +@dataclass(frozen=True, slots=True) +class _BridgedRequest: + input: str | ResponseInputParam + responses_api_request: ResponsesAPIOptionalRequestParams + + +def _bridged_request( + hoisted: HoistedAdditionalTools, + responses_api_request: ResponsesAPIOptionalRequestParams, + loaded_earlier: Sequence[Mapping[str, object]], +) -> _BridgedRequest: + searched: Final = bridge_client_tool_search(hoisted.input, hoisted.tools, loaded_earlier) + return _BridgedRequest( + input=searched.input, + responses_api_request=( + {**responses_api_request, "tools": list(searched.tools)} + if hoisted.hoisted or searched.bridged + else responses_api_request + ), + ) + + class LiteLLMCompletionTransformationHandler: def response_api_handler( self, @@ -48,10 +74,22 @@ class LiteLLMCompletionTransformationHandler: | Coroutine[object, object, ResponsesAPIResponse | BaseResponsesAPIStreamingIterator] ): hoisted: Final = hoist_additional_tools(input, responses_api_request.get("tools")) - bridged_input: Final = hoisted.input - bridged_request: Final[ResponsesAPIOptionalRequestParams] = ( - {**responses_api_request, "tools": list(hoisted.tools)} if hoisted.hoisted else responses_api_request - ) + previous_response_id: Final = responses_api_request.get("previous_response_id") + if _is_async and previous_response_id and has_client_tool_search(hoisted.tools): + return self._async_response_api_handler_after_earlier_searches( + previous_response_client_ids, + previous_response_id, + hoisted, + model=model, + responses_api_request=responses_api_request, + custom_llm_provider=custom_llm_provider, + stream=stream, + extra_headers=extra_headers, + request_kwargs=_REQUEST_KWARGS_ADAPTER.validate_python(kwargs), + ) + bridged: Final = _bridged_request(hoisted, responses_api_request, ()) + bridged_input: Final = bridged.input + bridged_request: Final = bridged.responses_api_request litellm_completion_request: Final[dict] = ( LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( model=model, @@ -73,13 +111,8 @@ class LiteLLMCompletionTransformationHandler: **kwargs, ) - completion_args: Final = {} - completion_args.update(kwargs) - completion_args.update(litellm_completion_request) - completion_args["_skip_responses_api_bridge"] = True - litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = litellm.completion( - **completion_args, + **{**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True}, ) if isinstance(litellm_completion_response, ModelResponse): @@ -104,9 +137,53 @@ class LiteLLMCompletionTransformationHandler: ) raise ValueError(f"Unexpected response type: {type(litellm_completion_response)}") + async def _async_response_api_handler_after_earlier_searches( + self, + previous_response_client_ids: tuple[str, ...], + previous_response_id: str, + hoisted: HoistedAdditionalTools, + /, + *, + model: str, + responses_api_request: ResponsesAPIOptionalRequestParams, + custom_llm_provider: str | None, + stream: bool | None, + extra_headers: Mapping[str, object] | None, + request_kwargs: Mapping[str, object], + ) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator: + stored_turns: Final = await ResponsesSessionHandler.get_stored_turns_for_previous_response_id( + previous_response_id, previous_response_client_ids + ) + bridged: Final = _bridged_request( + hoisted, + responses_api_request, + ResponsesSessionHandler.client_tool_search_tools_loaded_before( + stored_turns, previous_response_id, previous_response_client_ids + ), + ) + return await self.async_response_api_handler( + previous_response_client_ids, + stored_turns, + litellm_completion_request=( + LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model=model, + input=bridged.input, + responses_api_request=bridged.responses_api_request, + custom_llm_provider=custom_llm_provider, + stream=stream, + extra_headers=extra_headers, + **request_kwargs, + ) + ), + request_input=bridged.input, + responses_api_request=bridged.responses_api_request, + **request_kwargs, + ) + async def async_response_api_handler( self, previous_response_client_ids: tuple[str, ...] = (), + stored_turns: StoredTurns | None = None, /, *, litellm_completion_request: dict, @@ -122,16 +199,12 @@ class LiteLLMCompletionTransformationHandler: instructions=responses_api_request.get("instructions"), request_kwargs=_REQUEST_KWARGS_ADAPTER.validate_python(kwargs), previous_response_client_ids=previous_response_client_ids, + stored_turns=stored_turns, input_replaces_history=holds_compaction_summary(with_compaction_items_as_params(request_input)), ) - acompletion_args: Final = {} - acompletion_args.update(kwargs) - acompletion_args.update(litellm_completion_request) - acompletion_args["_skip_responses_api_bridge"] = True - litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = await litellm.acompletion( - **acompletion_args, + **{**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True}, ) if isinstance(litellm_completion_response, ModelResponse): responses_api_response: Final[ResponsesAPIResponse] = ( diff --git a/litellm/responses/litellm_completion_transformation/session_handler.py b/litellm/responses/litellm_completion_transformation/session_handler.py index 1ab6293b432..ce2396355a2 100644 --- a/litellm/responses/litellm_completion_transformation/session_handler.py +++ b/litellm/responses/litellm_completion_transformation/session_handler.py @@ -9,6 +9,10 @@ import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import REDACTED_BY_LITELLM, REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER from litellm.litellm_core_utils.cold_storage_handler import ColdStorageHandler +from litellm.responses.litellm_completion_transformation.client_tool_search import ( + bridge_client_tool_search, + client_tool_search_loaded_tools, +) from litellm.responses.litellm_completion_transformation.compaction_items import ( holds_compaction_summary, input_items, @@ -88,6 +92,12 @@ class _ReplayStart: at_output: bool +@dataclass(frozen=True, slots=True) +class StoredTurns: + spend_logs: tuple["SpendLogsPayload", ...] + proxy_server_requests: tuple[Mapping[str, object] | None, ...] + + def _compaction_point( index: int, spend_log: "SpendLogsPayload", proxy_server_request: Mapping[str, object] | None ) -> _ReplayStart | None: @@ -190,6 +200,7 @@ class ResponsesSessionHandler: previous_response_id: str, item_cache_control: bool = False, client_response_ids: tuple[str, ...] = (), + stored_turns: StoredTurns | None = None, ) -> ChatCompletionSession: """ Return the chat completion message history for a previous response id @@ -200,23 +211,21 @@ class ResponsesSessionHandler: ) verbose_proxy_logger.debug("inside get_chat_completion_message_history_for_previous_response_id") - all_spend_logs: Final = [ - spend_log - for spend_log in await ResponsesSessionHandler.get_all_spend_logs_for_previous_response_id( + turns: Final = ( + stored_turns + if stored_turns is not None + else await ResponsesSessionHandler.get_stored_turns_for_previous_response_id( previous_response_id, client_response_ids ) - if _is_a_conversation_turn(spend_log) - ] + ) + all_spend_logs: Final = turns.spend_logs verbose_proxy_logger.debug("found %s spend logs for this response id", len(all_spend_logs)) litellm_session_id: str | None = None if len(all_spend_logs) > 0: litellm_session_id = all_spend_logs[0].get("session_id") - proxy_server_requests: Final = [ - await ResponsesSessionHandler.get_proxy_server_request_from_spend_log(spend_log=spend_log) - for spend_log in all_spend_logs - ] + proxy_server_requests: Final = turns.proxy_server_requests replayable_turns: Final = _turns_through_referenced_response( all_spend_logs, _referenced_request_ids(previous_response_id, client_response_ids) ) @@ -252,6 +261,45 @@ class ResponsesSessionHandler: instructions=_latest_replayable_instructions(proxy_server_requests), ) + @staticmethod + async def get_stored_turns_for_previous_response_id( + previous_response_id: str, + client_response_ids: tuple[str, ...] = (), + ) -> StoredTurns: + spend_logs: Final = tuple( + spend_log + for spend_log in await ResponsesSessionHandler.get_all_spend_logs_for_previous_response_id( + previous_response_id, client_response_ids + ) + if _is_a_conversation_turn(spend_log) + ) + return StoredTurns( + spend_logs=spend_logs, + proxy_server_requests=tuple( + [ + await ResponsesSessionHandler.get_proxy_server_request_from_spend_log(spend_log=spend_log) + for spend_log in spend_logs + ] + ), + ) + + @staticmethod + def client_tool_search_tools_loaded_before( + stored_turns: StoredTurns, + previous_response_id: str, + client_response_ids: tuple[str, ...] = (), + ) -> tuple[Mapping[str, object], ...]: + turns: Final = _turns_through_referenced_response( + stored_turns.spend_logs, _referenced_request_ids(previous_response_id, client_response_ids) + ) + return tuple( + chain.from_iterable( + client_tool_search_loaded_tools(request.get("input")) + for request in stored_turns.proxy_server_requests[:turns] + if request is not None + ) + ) + @staticmethod def _history_of_turns( spend_logs: Sequence["SpendLogsPayload"], @@ -310,7 +358,7 @@ class ResponsesSessionHandler: if response_input_param: chat_completion_message_history.extend( LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( - input=response_input_param, + input=bridge_client_tool_search(response_input_param, None).input, responses_api_request={}, replay_reasoning=True, item_cache_control=item_cache_control, @@ -325,7 +373,7 @@ class ResponsesSessionHandler: if (responses_api_output := _stored_responses_api_output(_response_output)) is not None: chat_completion_message_history.extend( LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( - input=responses_api_output, + input=bridge_client_tool_search(responses_api_output, None).input, responses_api_request={}, replay_reasoning=True, item_cache_control=item_cache_control, diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 982158040c4..892da541935 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -1,7 +1,8 @@ import time import uuid -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from itertools import filterfalse +from types import MappingProxyType from typing import Any, Final, cast import litellm @@ -18,9 +19,9 @@ from litellm.responses.litellm_completion_transformation.custom_tools import ( ) from litellm.responses.litellm_completion_transformation.reasoning_items import mint_reasoning_item_id from litellm.responses.litellm_completion_transformation.tool_search import ( + ToolSearchExecution, build_tool_search_call, - has_server_tool_search, - is_anthropic_tool_search_use, + tool_search_routing, ) from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, @@ -30,6 +31,7 @@ from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ( PART_UNION_TYPES, BaseLiteLLMOpenAIResponseObject, + ChatCompletionAnnotation, ContentPartAddedEvent, ContentPartDoneEvent, ContentPartDonePartOutputText, @@ -56,6 +58,7 @@ from litellm.types.llms.openai import ( WebSearchCallInProgressEvent, WebSearchCallSearchingEvent, ) +from litellm.types.responses.main import GenericResponseOutputItemContentAnnotation from litellm.types.utils import Delta as ChatCompletionDelta from litellm.types.utils import ( ModelResponse, @@ -105,7 +108,6 @@ def _delta_carries_output(delta: ChatCompletionDelta) -> bool: getattr(delta, "reasoning_content", None), delta.tool_calls, delta.function_call, - getattr(delta, "annotations", None), getattr(delta, "images", None), getattr(delta, "audio", None), ) @@ -150,7 +152,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self.sent_output_text_done_event: bool = False self.sent_output_content_part_done_event: bool = False self.sent_output_item_done_event: bool = False - self.sent_annotation_events: bool = False + self._annotation_index: int = 0 + self._annotations_awaiting_message: tuple[ChatCompletionAnnotation, ...] = () self.litellm_model_response: ModelResponse | TextCompletionResponse | None = None self.completed_response = None self.final_text: str = "" @@ -190,16 +193,21 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._namespace_tool_names = LiteLLMCompletionResponsesConfig.namespace_tool_name_map(request_tools) self._web_search_calls: dict[str, object] = {} # mutable-ok: latest call by provider id self._queued_web_search_call_ids: set[str] = set() # mutable-ok: emitted call ids - self._tool_search_requested: bool = has_server_tool_search(request_tools) - self._tool_search_call_ids: frozenset[str] = frozenset() + self._tool_search = tool_search_routing(request_tools) + self._tool_search_executions: Mapping[str, ToolSearchExecution] = MappingProxyType({}) - def _is_tool_search_call(self, call_id: str, fn_name: str) -> bool: - if call_id in self._tool_search_call_ids: - return True - if not self._tool_search_requested or not is_anthropic_tool_search_use(call_id, fn_name): - return False - self._tool_search_call_ids = self._tool_search_call_ids | {call_id} - return True + async def aclose(self) -> None: + await self.litellm_custom_stream_wrapper.aclose() + + def _tool_search_execution(self, call_id: str, fn_name: str) -> ToolSearchExecution | None: + known: Final = self._tool_search_executions.get(call_id) + if known is not None: + return known + execution: Final = self._tool_search.execution_of(call_id, fn_name) + if execution is None: + return None + self._tool_search_executions = MappingProxyType({**self._tool_search_executions, call_id: execution}) + return execution def _next_output_item_added(self, output_index: int, item: BaseLiteLLMOpenAIResponseObject) -> OutputItemAddedEvent: self._sequence_number += 1 @@ -214,18 +222,22 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): def _queue_output_item_added(self, output_index: int, item: BaseLiteLLMOpenAIResponseObject) -> None: self._pending_tool_events.append(self._next_output_item_added(output_index, item)) - def _queue_tool_search_call_added(self, call_id: str, output_index: int, args_delta: str) -> None: + def _queue_tool_search_call_added( + self, call_id: str, output_index: int, args_delta: str, execution: ToolSearchExecution + ) -> None: if call_id not in self._tool_args_by_call_id: self._tool_args_by_call_id[call_id] = "" - item: Final = build_tool_search_call(call_id, "", "in_progress") + item: Final = build_tool_search_call(call_id, "", "in_progress", execution) self._tool_item_id_by_call_id[call_id] = item.id self._queue_output_item_added(output_index, BaseLiteLLMOpenAIResponseObject(**item.model_dump())) self._tool_args_by_call_id[call_id] += args_delta - def _queue_tool_search_call_done(self, call_id: str, output_index: int, final_args: str) -> None: - self._queue_tool_search_call_added(call_id, output_index, "") + def _queue_tool_search_call_done( + self, call_id: str, output_index: int, final_args: str, execution: ToolSearchExecution + ) -> None: + self._queue_tool_search_call_added(call_id, output_index, "", execution) item: Final = build_tool_search_call( - call_id, final_args or self._tool_args_by_call_id.get(call_id, ""), "completed" + call_id, final_args or self._tool_args_by_call_id.get(call_id, ""), "completed", execution ) self._sequence_number += 1 self._pending_tool_events.append( @@ -339,7 +351,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): namespace_kwargs: Final = {"namespace": tool_namespace} if tool_namespace else {} return {**item_kwargs, "name": tool_name, **namespace_kwargs} - def _is_reasoning_end(self, chunk): + def _is_reasoning_end(self, chunk: ModelResponseStream) -> bool: if not chunk.choices: return False delta: Final = chunk.choices[0].delta @@ -350,7 +362,9 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if hasattr(delta, "thinking_blocks") and delta.thinking_blocks: return False - return delta.content or delta.function_call or delta.tool_calls or chunk.choices[0].finish_reason is not None + return bool( + delta.content or delta.function_call or delta.tool_calls or chunk.choices[0].finish_reason is not None + ) def _reserve_web_search_indexes(self, provider_fields: object) -> None: if not isinstance(provider_fields, dict): @@ -428,10 +442,12 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): else: fn_name = str(getattr(fn, "name", "") or "") fn_args_delta = serialize_tool_call_arguments(getattr(fn, "arguments", "")) + if call_id not in self._tool_output_index_by_call_id: + self._queue_held_annotations() output_index = self._get_or_assign_tool_output_index(call_id) - if self._is_tool_search_call(call_id, fn_name): - self._queue_tool_search_call_added(call_id, output_index, fn_args_delta) + if (search_execution := self._tool_search_execution(call_id, fn_name)) is not None: + self._queue_tool_search_call_added(call_id, output_index, fn_args_delta, search_execution) continue self._started_tool_call = True @@ -499,8 +515,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._queue_web_search_events(call_id, web_search_call) self._queued_web_search_call_ids.add(call_id) continue - if self._is_tool_search_call(call_id, fn_name): - self._queue_tool_search_call_done(call_id, output_index, fn_args) + if (search_execution := self._tool_search_execution(call_id, fn_name)) is not None: + self._queue_tool_search_call_done(call_id, output_index, fn_args, search_execution) continue # Track if this is a new tool call that wasn't streamed @@ -782,8 +798,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): return event def _queue_message_item_added_events(self) -> None: - if self._cached_item_id is None: - self._cached_item_id = f"msg_{uuid.uuid4()}" + item_id: Final = self._message_item_id() self.sent_message_item_added_event = True self.sent_content_part_added_event = True self._sequence_number += 1 @@ -792,7 +807,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): output_index=self._message_index(), item=BaseLiteLLMOpenAIResponseObject( **{ - "id": self._cached_item_id, + "id": item_id, "type": "message", "role": "assistant", "status": "in_progress", @@ -1081,6 +1096,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if self.litellm_model_response: self._queue_compaction_item_added() self._queue_compaction_item_done() + self._queue_held_annotations() if self._pending_response_events: return self._pending_response_events.pop(0) # If tool calls exist, emit tool events before finishing/response.completed. @@ -1163,7 +1179,11 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): delta: Final = chunk.choices[0].delta if chunk.choices else None if delta and getattr(delta, "reasoning_content", None): self._accumulated_reasoning_content_parts.append(delta.reasoning_content) - if not self._is_reasoning_end(chunk): + if self._is_reasoning_end(chunk): + self._queue_reasoning_done_events() + + def _queue_reasoning_done_events(self) -> None: + if not self._reasoning_active or self._reasoning_done_emitted: return reasoning_content: Final = "".join(self._accumulated_reasoning_content_parts) self._cached_reasoning_item_id = ( @@ -1237,9 +1257,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._snapshot_chunk_for_stream_chunk_builder(chunk) ) self._queue_reasoning_done_events_on_reasoning_end(chunk) - response_api_chunk = self._transform_chat_completion_chunk_to_response_api_chunk(chunk) - if response_api_chunk: - self._pending_response_events.append(response_api_chunk) + self._queue_chunk_events(chunk) if self._pending_response_events: return self._pending_response_events.pop(0) @@ -1300,9 +1318,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._snapshot_chunk_for_stream_chunk_builder(cast(ModelResponseStream, chunk)) ) self._queue_reasoning_done_events_on_reasoning_end(chunk) - response_api_chunk = self._transform_chat_completion_chunk_to_response_api_chunk(chunk) - if response_api_chunk: - self._pending_response_events.append(response_api_chunk) + self._queue_chunk_events(chunk) if self._pending_response_events: return self._pending_response_events.pop(0) # Otherwise, loop to next chunk @@ -1313,6 +1329,58 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self.finished = True raise e + def _queue_chunk_events(self, chunk: ModelResponseStream) -> None: + self._hold_chunk_annotations(chunk) + response_api_chunk: Final = self._transform_chat_completion_chunk_to_response_api_chunk(chunk) + if response_api_chunk: + self._pending_response_events.append(response_api_chunk) + if self.sent_message_item_added_event: + self._queue_held_annotations() + + def _hold_chunk_annotations(self, chunk: ModelResponseStream) -> None: + chunk_annotations: Final = (getattr(chunk.choices[0].delta, "annotations", None) or ()) if chunk.choices else () + self._annotations_awaiting_message = (*self._annotations_awaiting_message, *chunk_annotations) + + def _queue_held_annotations(self) -> None: + annotations: Final = self._annotations_awaiting_message + if not annotations: + return + self._annotations_awaiting_message = () + self._queue_reasoning_done_events() + self._queue_annotation_added_events(annotations) + + def _queue_annotation_added_events(self, annotations: Sequence[ChatCompletionAnnotation]) -> None: + if not self.sent_message_item_added_event: + self._queue_message_item_added_events() + item_id: Final = self._message_item_id() + response_annotations: Final = ( + LiteLLMCompletionResponsesConfig.transform_chat_completion_annotations_to_response_output_annotations( + annotations=annotations + ) + ) + self._pending_response_events.extend( + self._annotation_added_event(item_id, annotation) for annotation in response_annotations + ) + + def _message_item_id(self) -> str: + if self._cached_item_id is None: + self._cached_item_id = f"msg_{uuid.uuid4()}" + return self._cached_item_id + + def _annotation_added_event( + self, item_id: str, annotation: GenericResponseOutputItemContentAnnotation + ) -> OutputTextAnnotationAddedEvent: + event: Final = OutputTextAnnotationAddedEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED, + item_id=item_id, + output_index=self._message_index(), + content_index=0, + annotation_index=self._annotation_index, + annotation=annotation.model_dump(), + ) + self._annotation_index += 1 + return event + def _transform_chat_completion_chunk_to_response_api_chunk( self, chunk: ModelResponseStream ) -> ResponsesAPIStreamingResponse | None: @@ -1321,37 +1389,9 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): This currently handles emitting the OutputTextDeltaEvent, which is used by other tools using the responses API and the ReasoningSummaryTextDeltaEvent, which is used by the responses API to emit reasoning content. - It also handles emitting annotation.added events when annotations are detected in the chunk. """ - if self._cached_item_id is None: - self._cached_item_id = f"msg_{uuid.uuid4()}" - item_id: Final = self._cached_item_id + item_id: Final = self._message_item_id() - # Check if this chunk has annotations first (before processing text/reasoning) - # This ensures we detect and queue annotation events from the annotation chunk - if chunk.choices and hasattr(chunk.choices[0].delta, "annotations"): - annotations: Final = chunk.choices[0].delta.annotations - if annotations and self.sent_annotation_events is False: - self.sent_annotation_events = True - # Store annotation events to emit them one by one - if not hasattr(self, "_pending_annotation_events"): - response_annotations = LiteLLMCompletionResponsesConfig.transform_chat_completion_annotations_to_response_output_annotations( - annotations=annotations - ) - self._pending_annotation_events = [] - for idx, annotation in enumerate(response_annotations): - annotation_dict = ( - annotation.model_dump() if hasattr(annotation, "model_dump") else dict(annotation) - ) - event = OutputTextAnnotationAddedEvent( - type=ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED, - item_id=item_id, - output_index=self._message_index(), - content_index=0, - annotation_index=idx, - annotation=annotation_dict, - ) - self._pending_annotation_events.append(event) # Priority 1: Handle reasoning content (highest priority) if ( chunk.choices @@ -1395,13 +1435,6 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if self._pending_tool_events: return self._pending_tool_events.pop(0) - # Priority 4: If we have pending annotation events, emit the next one - # This happens when the current chunk has no text/reasoning content - if hasattr(self, "_pending_annotation_events") and self._pending_annotation_events: - event = self._pending_annotation_events.pop(0) - return event - - # Priority 5: If we have pending tool events (from earlier chunk), emit the next one if self._pending_tool_events: return self._pending_tool_events.pop(0) @@ -1444,7 +1477,10 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): def _is_unstreamed_empty_message(self, item: object) -> bool: if getattr(item, "type", None) != "message" or self._message_output_index is not None: return False - return not any(getattr(part, "text", None) for part in attribute_items(item, "content")) + return not any( + getattr(part, "text", None) or getattr(part, "annotations", None) + for part in attribute_items(item, "content") + ) def finished_round_event(self) -> ResponseCompletedEvent | ResponseIncompleteEvent | None: if not isinstance(self.litellm_model_response, ModelResponse): diff --git a/litellm/responses/litellm_completion_transformation/tool_search.py b/litellm/responses/litellm_completion_transformation/tool_search.py index 780a23e02b2..5a1b27f9092 100644 --- a/litellm/responses/litellm_completion_transformation/tool_search.py +++ b/litellm/responses/litellm_completion_transformation/tool_search.py @@ -1,4 +1,5 @@ from collections.abc import Iterable, Iterator, Mapping, Sequence, Set +from dataclasses import dataclass from typing import TYPE_CHECKING, Final, Literal, TypeAlias from pydantic import ConfigDict, TypeAdapter, ValidationError @@ -18,25 +19,31 @@ if TYPE_CHECKING: __all__ = [ "ANTHROPIC_TOOL_SEARCH_TOOL_NAME", "ANTHROPIC_TOOL_SEARCH_TOOL_TYPE", + "CLIENT_TOOL_SEARCH_FUNCTION_NAME", "CompletionRoute", "ToolSearchCallStatus", + "ToolSearchExecution", + "ToolSearchRouting", "answers_tool_search", "anthropic_tool_search_chat_tool", "build_tool_search_call", "chat_config_builds_anthropic_request", "chat_messages_without_tool_search_calls", "chat_route_rejects_unanswered_tool_calls", + "has_client_tool_search", "has_server_tool_search", "holds_tool_search_use", "input_without_unanswered_anthropic_tool_search_calls", "is_anthropic_tool_search_chat_tool", "is_anthropic_tool_search_use", + "is_client_tool_search_tool", "is_openai_tool_search_tool", "is_server_tool_search_tool", "provider_builds_anthropic_request", "provider_chat_config", "runs_tool_search_on_the_server", "tool_search_call_ids", + "tool_search_routing", "tools_with_anthropic_tool_search_mapped", "tools_without_anthropic_tool_search", "unanswered_tool_search_call_ids", @@ -46,7 +53,9 @@ __all__ = [ ANTHROPIC_TOOL_SEARCH_TOOL_TYPE: Final = "tool_search_tool_regex_20251119" ANTHROPIC_TOOL_SEARCH_TOOL_NAME: Final = "tool_search_tool_regex" +CLIENT_TOOL_SEARCH_FUNCTION_NAME: Final = "tool_search" ToolSearchCallStatus: TypeAlias = Literal["in_progress", "completed", "incomplete"] +ToolSearchExecution: TypeAlias = Literal["server", "client"] _ARGUMENTS_ADAPTER: Final = TypeAdapter(dict[str, object]) _MESSAGE_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) @@ -79,6 +88,35 @@ def has_server_tool_search(tools: Sequence[object] | None) -> bool: return any(is_server_tool_search_tool(tool) for tool in tools or ()) +def is_client_tool_search_tool(tool: object) -> bool: + try: + parsed: Final = _ToolSearchFields.model_validate(tool) + except ValidationError: + return False + return parsed.type == "tool_search" and parsed.execution == "client" + + +def has_client_tool_search(tools: Sequence[object] | None) -> bool: + return any(is_client_tool_search_tool(tool) for tool in tools or ()) + + +@dataclass(frozen=True, slots=True) +class ToolSearchRouting: + server: bool + client: bool + + def execution_of(self, tool_id: str, tool_name: str) -> ToolSearchExecution | None: + if self.server and is_anthropic_tool_search_use(tool_id, tool_name): + return "server" + if self.client and tool_name == CLIENT_TOOL_SEARCH_FUNCTION_NAME: + return "client" + return None + + +def tool_search_routing(tools: Sequence[object] | None) -> ToolSearchRouting: + return ToolSearchRouting(server=has_server_tool_search(tools), client=has_client_tool_search(tools)) + + def anthropic_tool_search_chat_tool() -> AnthropicToolSearchToolRegex: return AnthropicToolSearchToolRegex(type=ANTHROPIC_TOOL_SEARCH_TOOL_TYPE, name=ANTHROPIC_TOOL_SEARCH_TOOL_NAME) @@ -346,13 +384,18 @@ def _parsed_arguments(arguments_json: str) -> Mapping[str, object] | None: return None -def build_tool_search_call(tool_id: str, arguments_json: str, status: ToolSearchCallStatus) -> OutputToolSearchCall: +def build_tool_search_call( + tool_id: str, + arguments_json: str, + status: ToolSearchCallStatus, + execution: ToolSearchExecution = "server", +) -> OutputToolSearchCall: arguments: Final = _parsed_arguments(arguments_json) return OutputToolSearchCall( id=openai_shaped_tool_call_item_id("tool_search_call", tool_id), call_id=tool_id, arguments={} if arguments is None else arguments, - execution="server", + execution=execution, status="incomplete" if arguments is None else status, type="tool_search_call", ) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index bc92bc9cb3a..ce8841c9190 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -64,6 +64,7 @@ from litellm.llms.snowflake.chat.transformation import is_snowflake_claude_model from litellm.responses.bridge_check import chat_target_accepts_reasoning_summary from litellm.responses.litellm_completion_transformation.session_handler import ( ResponsesSessionHandler, + StoredTurns, ) from litellm.types.llms.anthropic import AnthropicToolSearchToolRegex from litellm.types.llms.base import CachedTokensDetails @@ -124,6 +125,7 @@ from litellm.types.utils import ( Usage, ) +from .client_tool_search import client_tool_search_chat_tool from .compaction_items import ( carries_only_compaction, compaction_output_item, @@ -151,16 +153,17 @@ from .custom_tools import ( from .reasoning_items import decode_thinking_blocks, encode_thinking_blocks, mint_reasoning_item_id from .tool_search import ( CompletionRoute, + ToolSearchRouting, answers_tool_search, anthropic_tool_search_chat_tool, build_tool_search_call, has_server_tool_search, holds_tool_search_use, is_anthropic_tool_search_chat_tool, - is_anthropic_tool_search_use, is_server_tool_search_tool, provider_builds_anthropic_request, tool_search_call_ids, + tool_search_routing, without_tool_search_calls, without_tool_search_uses, ) @@ -321,7 +324,7 @@ def _responses_file_part(item: Mapping[str, object]) -> _ResponsesFilePart: ) -def _decode_responses_media(item: Mapping[str, object]) -> _ResponsesMediaPart | None: +def decode_responses_media(item: Mapping[str, object]) -> _ResponsesMediaPart | None: kind: Final = item.get("type") if kind in ("input_image", "image_url", "computer_screenshot") and any(key in item for key in ("image_url", "url")): image: Final = _responses_image_part(item) @@ -383,6 +386,52 @@ def _attribute_fields(value: object) -> dict[str, object]: return dict(cast("Iterable[tuple[str, object]]", value)) # cast-ok: dict() raises on non-pair values, as before +def tool_output_to_chat_content( + output: object, +) -> str | list[ToolMessageContentPart]: + if output is None: + return "" + if isinstance(output, str): + return output + + single_media: Final = decode_responses_media(output) if isinstance(output, Mapping) else None + if single_media is not None: + return [single_media] + if isinstance(output, list): + normalized_blocks: Final[list[ToolMessageContentPart]] = [] + text_acc: Final[list[str]] = [] + for part in output: + if not isinstance(part, dict): + continue + part_type = part.get("type") + if part_type in ("input_text", "output_text", "text"): + txt = part.get("text") + if isinstance(txt, str) and txt: + text_acc.append(txt) + normalized_blocks.append({"type": "text", "text": txt}) + elif (media := decode_responses_media(part)) is not None: + normalized_blocks.append(media) + + if any(b.get("type") != "text" for b in normalized_blocks): + return normalized_blocks + if text_acc: + return "".join(text_acc) + try: + return json.dumps(output) + except Exception: + return str(output) + + try: + return json.dumps(output) + except Exception: + return str(output) + + +TOOL_CALL_OUTPUT_ITEM_TYPES: Final = frozenset( + {"function_call_output", "custom_tool_call_output", "computer_call_output", "tool_result"} +) + + def _input_item_role(input_item: Mapping[str, object]) -> str: return cast(str, input_item.get("role") or "user") # cast-ok: client-supplied role forwarded verbatim, unvalidated @@ -1296,6 +1345,7 @@ class LiteLLMCompletionResponsesConfig: request_kwargs: Mapping[str, object] = MappingProxyType({}), previous_response_client_ids: tuple[str, ...] = (), input_replaces_history: bool = False, + stored_turns: StoredTurns | None = None, ) -> dict[str, object]: """ Async hook to get the chain of previous input and output pairs and return a list of Chat Completion messages @@ -1311,6 +1361,7 @@ class LiteLLMCompletionResponsesConfig: ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id, item_cache_control=keeps_item_marks, client_response_ids=previous_response_client_ids, + stored_turns=stored_turns, ) chat_completion_session: Final = ( await lookup(previous_response_id) @@ -2371,12 +2422,7 @@ class LiteLLMCompletionResponsesConfig: """ Check if the input item is a tool call output """ - return input_item.get("type") in [ - "function_call_output", - "custom_tool_call_output", - "computer_call_output", - "tool_result", # Anthropic/MCP format - ] + return input_item.get("type") in TOOL_CALL_OUTPUT_ITEM_TYPES @staticmethod def _is_input_item_function_call(input_item: Mapping[str, object]) -> bool: @@ -2400,58 +2446,7 @@ class LiteLLMCompletionResponsesConfig: if not call_id: return [] - def _normalize_function_call_output_to_tool_content( - output: object, - ) -> str | list[ToolMessageContentPart]: - """ - Preserve structured media for downstream chat adapters and token estimation. - Text-only outputs retain their existing string representation. - """ - if output is None: - return "" - if isinstance(output, str): - return output - - single_media: Final = _decode_responses_media(output) if isinstance(output, Mapping) else None - if single_media is not None: - return [single_media] - # Some adapters represent tool output as a list of "input_*" parts - if isinstance(output, list): - normalized_blocks: Final[list[ToolMessageContentPart]] = [] - text_acc: Final[list[str]] = [] - for part in output: - if not isinstance(part, dict): - continue - part_type = part.get("type") - if part_type in ("input_text", "output_text", "text"): - txt = part.get("text") - if isinstance(txt, str) and txt: - text_acc.append(txt) - normalized_blocks.append({"type": "text", "text": txt}) - elif (media := _decode_responses_media(part)) is not None: - normalized_blocks.append(media) - - if any(b.get("type") != "text" for b in normalized_blocks): - return normalized_blocks - if text_acc: - return "".join(text_acc) - try: - # last resort: keep something meaningful for providers that require a string - import json as _json - - return _json.dumps(output) - except Exception: - return str(output) - - # Fallback for dict/number/etc. - try: - import json as _json - - return _json.dumps(output) - except Exception: - return str(output) - - tool_content: Final = _normalize_function_call_output_to_tool_content(tool_call_output.get("output")) + tool_content: Final = tool_output_to_chat_content(tool_call_output.get("output")) output_cache_control: Final = _input_item_cache_control(tool_call_output) tool_output_message: Final = ( ChatCompletionToolMessage(role="tool", content=tool_content, tool_call_id=str(call_id)) @@ -2606,7 +2601,7 @@ class LiteLLMCompletionResponsesConfig: content_list.append(item) elif isinstance(item, dict): if ( - media := _decode_responses_media( + media := decode_responses_media( fields := _validate_responses_media(_STR_KEY_DICT_ADAPTER, item) ) ) is not None: @@ -2828,9 +2823,11 @@ class LiteLLMCompletionResponsesConfig: if tool_type == "custom": converted: Final = convert_custom_tool_to_function_tool(tool) return ResponsesToolChatForm(chat_tools=() if converted is None else (converted,), web_search_options=None) - if tool_type == "tool_search" and validated_tool_search_execution(tool.get("execution")) != "client": + if tool_type == "tool_search" and validated_tool_search_execution(tool.get("execution")) == "client": + return ResponsesToolChatForm(chat_tools=(client_tool_search_chat_tool(tool),), web_search_options=None) + if tool_type == "tool_search": return ResponsesToolChatForm(chat_tools=(anthropic_tool_search_chat_tool(),), web_search_options=None) - if tool_type in ("computer_use", "image_generation", "local_shell", "shell", "tool_search"): + if tool_type in ("computer_use", "image_generation", "local_shell", "shell"): verbose_logger.warning( "Dropping Responses API tool of type '%s': it has no Chat Completions " "equivalent and the target provider would reject the request.", @@ -2970,7 +2967,7 @@ class LiteLLMCompletionResponsesConfig: request_tools: Final = responses_api_request.get("tools") if responses_api_request is not None else None custom_tool_names: Final = extract_custom_tool_names(request_tools) namespace_tool_names: Final = LiteLLMCompletionResponsesConfig.namespace_tool_name_map(request_tools) - tool_search_requested: Final = has_server_tool_search(request_tools) + tool_search: Final = tool_search_routing(request_tools) web_search_calls: Final = LiteLLMCompletionResponsesConfig._web_search_calls_by_call_id( chat_completion_response @@ -2987,7 +2984,7 @@ class LiteLLMCompletionResponsesConfig: if ( server_tool_call := LiteLLMCompletionResponsesConfig._server_tool_call_item( - tool_id, tool_name, tool_arguments, web_search_calls, tool_search_requested + tool_id, tool_name, tool_arguments, web_search_calls, tool_search ) ) is not None: responses_tools.append(server_tool_call) @@ -3052,14 +3049,15 @@ class LiteLLMCompletionResponsesConfig: tool_name: str, tool_arguments: str, web_search_calls: Mapping[str, ResponseFunctionWebSearch], - tool_search_requested: bool, + tool_search: ToolSearchRouting, ) -> ResponseFunctionWebSearch | OutputToolSearchCall | None: web_search_call: Final = web_search_calls.get(tool_id) if web_search_call is not None: return web_search_call - if tool_search_requested and is_anthropic_tool_search_use(tool_id, tool_name): - return build_tool_search_call(tool_id, tool_arguments, "completed") - return None + execution: Final = tool_search.execution_of(tool_id, tool_name) + if execution is None: + return None + return build_tool_search_call(tool_id, tool_arguments, "completed", execution) @staticmethod def _web_search_calls_by_call_id( @@ -3565,7 +3563,7 @@ class LiteLLMCompletionResponsesConfig: choice=choice, ) message_output_items.extend(image_generation_items) - elif choice.message.content is not None: + elif choice.message.content is not None or getattr(choice.message, "annotations", None): message_output_items.append( GenericResponseOutputItem( type="message", @@ -3639,13 +3637,13 @@ class LiteLLMCompletionResponsesConfig: return OutputText( type="output_text", - text=message.content, + text=message.content or "", annotations=transformed_annotations, ) @staticmethod def transform_chat_completion_annotations_to_response_output_annotations( - annotations: list[ChatCompletionAnnotation] | None, + annotations: Sequence[ChatCompletionAnnotation] | None, ) -> list[GenericResponseOutputItemContentAnnotation]: response_output_annotations: Final[list[GenericResponseOutputItemContentAnnotation]] = [] diff --git a/litellm/responses/main.py b/litellm/responses/main.py index d474ed3991f..98356e9a5d4 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -90,9 +90,9 @@ from litellm.types.responses.streaming_websocket import ResponsesWebSocketReques from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( ADDRESSED_RESPONSE_ID_FIELD, + AgenticLoopParams, CallTypes, CostPerToken, - CustomPricingLiteLLMParams, all_litellm_params, ) from litellm.utils import ( @@ -106,7 +106,17 @@ if TYPE_CHECKING: else: MCPTool = Any -from .streaming_iterator import BaseResponsesAPIStreamingIterator +from .retrieve_stream_query import ( + RETRIEVE_INCLUDE_QUERY_KEYS, + InvalidRetrieveStreamQuery, + RetrieveStreamQuery, + parse_retrieve_stream_query, +) +from .streaming_iterator import ( + BaseResponsesAPIStreamingIterator, + ResponsesAPIStreamingIterator, + SyncResponsesAPIStreamingIterator, +) __all__ = ( "acancel_responses", @@ -192,6 +202,7 @@ def mock_responses_api_response( _STR_MAPPING: Final = TypeAdapter(Mapping[str, object]) +_STR_DICT: Final = TypeAdapter(dict[str, object]) _FORWARDED_VALUE: Final = TypeAdapter(object) @@ -554,12 +565,6 @@ async def aresponses_api_with_mcp( return response -def _per_call_custom_pricing(litellm_params: GenericLiteLLMParams) -> Mapping[str, float]: - return MappingProxyType( - litellm_params.model_dump(include=set(CustomPricingLiteLLMParams.model_fields), exclude_none=True) - ) - - class _DeploymentModelInfo(BaseModel): model_config = ConfigDict(frozen=True) id: object = None @@ -1569,6 +1574,7 @@ def responses( local_vars=local_vars, use_chat_completions_api=use_chat_completions_api, ) + litellm_logging_obj.record_agentic_loop_params(AgenticLoopParams(model=model)) from litellm.litellm_core_utils.core_helpers import attribute_or_none @@ -1787,7 +1793,7 @@ def responses( optional_params=dict(responses_api_request_params), litellm_params={ **responses_api_request_params, - **_per_call_custom_pricing(litellm_params), + **litellm_params.custom_pricing_overrides(), "aresponses": _is_async, "litellm_call_id": litellm_call_id, "model_info": kwargs.get("model_info"), @@ -2013,6 +2019,53 @@ def delete_responses( ) +def _requested_retrieve_stream_query( + stream: bool | None, + starting_after: int | None, + extra_query: Mapping[str, object] | None, + custom_llm_provider: str | None, + drop_malformed: bool, +) -> RetrieveStreamQuery: + explicit: Final = { + key: value for key, value in (("stream", stream), ("starting_after", starting_after)) if value is not None + } + requested: Final = {**explicit, **(extra_query or {})} + parsed: Final = parse_retrieve_stream_query( + requested.get("stream"), requested.get("starting_after"), drop_malformed=drop_malformed + ) + if isinstance(parsed, InvalidRetrieveStreamQuery): + raise litellm.BadRequestError( + message=parsed.message, + model=None, + llm_provider=custom_llm_provider, + body={"message": parsed.message, "type": "invalid_request_error", "param": parsed.param}, + ) + return parsed + + +def _supported_retrieve_stream_query( + requested: RetrieveStreamQuery, + responses_api_provider_config: BaseResponsesAPIConfig, + custom_llm_provider: str, + drop_params: object, +) -> RetrieveStreamQuery: + if not requested.stream: + return RetrieveStreamQuery(stream=False, starting_after=None) + if responses_api_provider_config.supports_streamed_retrieval(): + return requested + if drops_unsupported_params(drop_params): + return RetrieveStreamQuery(stream=False, starting_after=None) + raise UnsupportedParamsError( + status_code=400, + llm_provider=custom_llm_provider, + message=( + f"{custom_llm_provider} does not support streaming a stored response (stream=true on " + "GET /v1/responses/{response_id}). To drop it, set `litellm.drop_params=True` or for proxy:\n\n" + "`litellm_settings:\n drop_params: true`\n" + ), + ) + + @client async def aget_responses( response_id: str, @@ -2024,8 +2077,10 @@ async def aget_responses( timeout: float | httpx.Timeout | None = None, # LiteLLM specific params, custom_llm_provider: str | None = None, + stream: bool | None = None, + starting_after: int | None = None, **kwargs, -) -> ResponsesAPIResponse: +) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator: """ Async: Fetch a response by its ID. @@ -2058,6 +2113,8 @@ async def aget_responses( extra_query=extra_query, extra_body=extra_body, timeout=timeout, + stream=stream, + starting_after=starting_after, **kwargs, ) @@ -2071,7 +2128,7 @@ async def aget_responses( response = init_response # Update the responses_api_response_id with the model_id - if not isinstance(response, ResponsesAPIResponse): # pyright: ignore[reportUnnecessaryIsInstance] # handlers can return non-ResponsesAPIResponse objects at runtime + if not isinstance(response, ResponsesAPIResponse): return response return ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id( responses_api_response=response, @@ -2101,8 +2158,14 @@ def get_responses( timeout: float | httpx.Timeout | None = None, # LiteLLM specific params, custom_llm_provider: str | None = None, + stream: bool | None = None, + starting_after: int | None = None, **kwargs, -) -> ResponsesAPIResponse | Coroutine[object, object, ResponsesAPIResponse]: +) -> ( + ResponsesAPIResponse + | SyncResponsesAPIStreamingIterator + | Coroutine[object, object, ResponsesAPIResponse | ResponsesAPIStreamingIterator] +): """ Fetch a response by its ID. @@ -2146,6 +2209,20 @@ def get_responses( if responses_api_provider_config is None: raise ValueError(f"GET responses is not supported for {custom_llm_provider}") + stream_query: Final = _supported_retrieve_stream_query( + requested=_requested_retrieve_stream_query( + stream=stream, + starting_after=starting_after, + extra_query=extra_query, + custom_llm_provider=custom_llm_provider, + drop_malformed=drops_unsupported_params(litellm_params.drop_params), + ), + responses_api_provider_config=responses_api_provider_config, + custom_llm_provider=custom_llm_provider, + drop_params=litellm_params.drop_params, + ) + litellm_metadata: Final = _STR_DICT.validate_python(kwargs.get("litellm_metadata") or {}) + local_vars.update(kwargs) # Pre Call logging @@ -2169,7 +2246,13 @@ def get_responses( litellm_params=litellm_params, logging_obj=litellm_logging_obj, extra_headers=extra_headers, + extra_query={ + key: value for key, value in (extra_query or {}).items() if key in RETRIEVE_INCLUDE_QUERY_KEYS + }, extra_body=extra_body, + stream=stream_query.stream, + starting_after=stream_query.starting_after, + litellm_metadata=litellm_metadata, timeout=timeout or request_timeout, _is_async=_is_async, client=kwargs.get("client"), @@ -2180,7 +2263,7 @@ def get_responses( if isinstance(response, ResponsesAPIResponse): response = ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id( responses_api_response=response, - litellm_metadata=kwargs.get("litellm_metadata", {}), + litellm_metadata=litellm_metadata, custom_llm_provider=custom_llm_provider, ) diff --git a/litellm/responses/retrieve_stream_query.py b/litellm/responses/retrieve_stream_query.py new file mode 100644 index 00000000000..086aa39efc6 --- /dev/null +++ b/litellm/responses/retrieve_stream_query.py @@ -0,0 +1,83 @@ +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Final, Literal + +from pydantic import TypeAdapter + +RETRIEVE_STREAM_QUERY_KEYS: Final = frozenset({"stream", "starting_after"}) +RETRIEVE_INCLUDE_QUERY_KEYS: Final = frozenset({"include", "include[]", "include_obfuscation"}) +_QUERY_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_MAX_STARTING_AFTER_DIGITS: Final = len(str(2**63 - 1)) + + +@dataclass(frozen=True, slots=True) +class RetrieveStreamQuery: + stream: bool + starting_after: int | None + + +@dataclass(frozen=True, slots=True) +class InvalidRetrieveStreamQuery: + param: Literal["stream", "starting_after"] + message: str + + +_NO_STREAM: Final = RetrieveStreamQuery(stream=False, starting_after=None) +_FULL_REPLAY: Final = RetrieveStreamQuery(stream=True, starting_after=None) + + +def _parsed_stream(value: object) -> bool | InvalidRetrieveStreamQuery: + match value: + case None: + return False + case bool(): + return value + case str() if value.lower() in ("true", "false"): + return value.lower() == "true" + case _: + return InvalidRetrieveStreamQuery("stream", f"Invalid 'stream': expected true or false, got {value!r}") + + +def _parsed_starting_after(value: object) -> int | None | InvalidRetrieveStreamQuery: + match value: + case None: + return None + case int() if not isinstance(value, bool) and value >= 0: + return value + case str() if value.isascii() and value.isdigit() and len(value) <= _MAX_STARTING_AFTER_DIGITS: + return int(value) + case _: + return InvalidRetrieveStreamQuery( + "starting_after", f"Invalid 'starting_after': expected an integer >= 0, got {value!r}" + ) + + +def parse_retrieve_stream_query( + stream: object, starting_after: object, drop_malformed: bool +) -> RetrieveStreamQuery | InvalidRetrieveStreamQuery: + parsed_stream: Final = _parsed_stream(stream) + if isinstance(parsed_stream, InvalidRetrieveStreamQuery): + return _NO_STREAM if drop_malformed else parsed_stream + if not parsed_stream: + return _NO_STREAM + parsed_starting_after: Final = _parsed_starting_after(starting_after) + if isinstance(parsed_starting_after, InvalidRetrieveStreamQuery): + return _FULL_REPLAY if drop_malformed else parsed_starting_after + return RetrieveStreamQuery(stream=True, starting_after=parsed_starting_after) + + +def requests_streamed_retrieval(request: Mapping[str, object]) -> bool: + extra_query: Final = request.get("extra_query") + return ( + "response_id" in request + and isinstance(extra_query, Mapping) + and _parsed_stream(_QUERY_ADAPTER.validate_python(extra_query).get("stream")) is True + ) + + +def without_retrieve_stream_defaults( + deployment_params: Mapping[str, object], request: Mapping[str, object] +) -> Mapping[str, object]: + if "response_id" not in request: + return deployment_params + return {key: value for key, value in deployment_params.items() if key not in RETRIEVE_STREAM_QUERY_KEYS} diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 4e418a91156..8959c32fff2 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -39,7 +39,11 @@ from litellm.constants import ( ) from litellm.exceptions import MidStreamFallbackError, RateLimitError from litellm.litellm_core_utils.asyncify import run_async_function -from litellm.litellm_core_utils.core_helpers import process_response_headers +from litellm.litellm_core_utils.core_helpers import ( + process_response_headers, + provider_reported_usage_cost, + set_response_cost_in_hidden_params, +) from litellm.litellm_core_utils.exception_mapping_utils import with_deployment_api_base from litellm.litellm_core_utils.internal_call_metadata import ( is_background_response, @@ -67,6 +71,7 @@ from litellm.types.llms.openai import ( ResponsesAPIResponse, ResponsesAPIStreamEvents, ResponsesAPIStreamingResponse, + replayable_responses_input, ) from litellm.types.responses.main import GenericResponseOutputItem, OutputText from litellm.types.utils import CallTypes, ModelResponse @@ -371,6 +376,7 @@ class BaseResponsesAPIStreamingIterator: _pending_logging_tasks: tuple[asyncio.Task[object], ...] = () _started_tool_call: bool = False _earlier_rounds_spend: BilledRoundsSpend | None = None + _replays_stored_stream: bool = False def __init__( self, @@ -569,7 +575,7 @@ class BaseResponsesAPIStreamingIterator: openai_types.ResponsesAPIStreamEvents.RESPONSE_FAILED, ): _response_obj: Final[object] = getattr(openai_responses_api_chunk, "response", None) - _estimate_wanted: Final[bool] = _chunk_type in ( + _estimate_wanted: Final[bool] = not self._replays_stored_stream and _chunk_type in ( openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED, openai_types.ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, ) @@ -990,6 +996,7 @@ class BaseResponsesAPIStreamingIterator: generated_text: Final = self._generated_content + self._generated_tool_arguments if ( not generated_text + or self._replays_stored_stream or requests_background(self.request_data) or is_background_response(self._created_response) or self._completed_response_logged @@ -1105,7 +1112,7 @@ class BaseResponsesAPIStreamingIterator: """ created: Final = self._created_response generated: Final = self._generated_content + self._generated_tool_arguments - if created is None or not generated or self._billed_without_disconnect(): + if self._replays_stored_stream or created is None or not generated or self._billed_without_disconnect(): return None usage: Final = _estimate_usage_safely( self.model or "", @@ -1157,6 +1164,8 @@ class BaseResponsesAPIStreamingIterator: def _raise_if_ended_without_terminal_event(self) -> None: if self.completed_response is not None: return + if self._replays_stored_stream and not self._yielded_first_chunk: + return error: Final = self._stream_ended_early_error() self._handle_failure(error) if self._output_started: @@ -1204,6 +1213,7 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): custom_llm_provider: str | None = None, request_data: dict[str, object] | None = None, call_type: str | None = None, + replays_stored_stream: bool = False, ): super().__init__( response, @@ -1215,6 +1225,7 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): request_data, call_type, ) + self._replays_stored_stream = replays_stored_stream self.stream_iterator = SSEDecoder().aiter_bytes(response.aiter_bytes()) def __aiter__(self): @@ -1227,6 +1238,9 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): await self._await_pending_logging() raise + async def aclose(self) -> None: + await self.response.aclose() + async def _next_event(self) -> ResponsesAPIStreamingResponse: try: self._check_max_streaming_duration() @@ -1299,6 +1313,7 @@ class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): custom_llm_provider: str | None = None, request_data: dict[str, object] | None = None, call_type: str | None = None, + replays_stored_stream: bool = False, ): super().__init__( response, @@ -1310,11 +1325,15 @@ class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): request_data, call_type, ) + self._replays_stored_stream = replays_stored_stream self.stream_iterator = SSEDecoder().iter_bytes(response.iter_bytes()) def __iter__(self): return self + def close(self) -> None: + self.response.close() + def __next__(self): try: self._check_max_streaming_duration() @@ -1825,20 +1844,32 @@ def _estimate_usage_safely( def stamp_responses_usage_cost( response_obj: ResponsesAPIResponse | None, logging_obj: LiteLLMLoggingObj | None ) -> None: - if response_obj is None or logging_obj is None: + """The terminal response's ``usage.cost`` is the bill litellm writes to the spend row. It is priced on a copy + shaped the way the success handler shapes the spend row's result (the reported ``usage.cost`` in the + response-cost header, the usage in chat shape), so the client and the row read the same number.""" + if response_obj is None: return usage_obj: Final[ResponseAPIUsage | None] = _usage_as_model(getattr(response_obj, "usage", None)) if usage_obj is None: return - response_obj.usage = usage_obj # rebind-ok: the stamped cost has to ride on the response the client receives - if isinstance(getattr(usage_obj, "cost", None), (int, float)): + response_obj.usage = usage_obj # rebind-ok: the typed usage has to ride on the response the client receives + if logging_obj is None: return + priced: Final = response_obj.model_copy( + update={"usage": ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(usage_obj)} + ) + priced.hidden_params = {} + set_response_cost_in_hidden_params( + priced, provider_reported_usage_cost(usage_obj, logging_obj.model_call_details.get("custom_llm_provider")) + ) try: - cost: Final[float | None] = logging_obj.response_cost_calculator(result=response_obj) - except Exception: + bill: Final = logging_obj.response_cost_calculator(result=priced) + except Exception: # noqa: BLE001 # a pricing failure leaves the client usage as the upstream sent it return - if isinstance(cost, (int, float)) and cost > 0: - setattr(usage_obj, "cost", cost) + if isinstance(bill, bool) or not isinstance(bill, (int, float)): + return + if bill > 0 or isinstance(getattr(usage_obj, "cost", None), (int, float)): + setattr(usage_obj, "cost", float(bill)) def opening_item_payload(payload: Mapping[str, object]) -> Mapping[str, object]: @@ -2349,15 +2380,17 @@ def _ws_event_error(event: Mapping[str, object]) -> object: return response.get("error") if _is_json_object(response) else None -def _restore_input_item_ids(items: Sequence[object]) -> Sequence[object]: - return ResponsesAPIRequestUtils.restore_encrypted_content_item_ids_in_input(copy.deepcopy(list(items))) +def _replayable_input_items(items: Sequence[object]) -> object: + return replayable_responses_input( + ResponsesAPIRequestUtils.restore_encrypted_content_item_ids_in_input(copy.deepcopy(list(items))) + ) def _restored_container_fields(container: Mapping[str, object]) -> Mapping[str, object]: input_items: Final = container.get("input") previous_response_id: Final = container.get("previous_response_id") restored: Final = { - "input": _restore_input_item_ids(input_items) if _is_json_array(input_items) else input_items, + "input": _replayable_input_items(input_items) if _is_json_array(input_items) else input_items, "previous_response_id": ( ResponsesAPIRequestUtils.decode_previous_response_id_to_original_previous_response_id(previous_response_id) if isinstance(previous_response_id, str) diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index cf993c69814..b0db0516658 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -131,14 +131,14 @@ def _provider_mapped_params( params: ResponsesAPIOptionalRequestParams, model: str, drop_params: bool, - declared_reasoning_efforts: Collection[str], + declared_reasoning_efforts: Collection[str] | None, ) -> Mapping[str, object]: from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig if ( - declared_reasoning_efforts + declared_reasoning_efforts is not None and isinstance(config, OpenAIResponsesAPIConfig) - and type(config).map_openai_params is OpenAIResponsesAPIConfig.map_openai_params + and config.honors_declared_reasoning_efforts() ): return config.map_openai_params_for_deployment( params, model, drop_params, declared_reasoning_efforts=declared_reasoning_efforts @@ -287,7 +287,7 @@ class ResponsesAPIRequestUtils: response_api_optional_params: ResponsesAPIOptionalRequestParams, allowed_openai_params: list[str] | None = None, drop_params: bool | None = None, - declared_reasoning_efforts: Collection[str] = (), + declared_reasoning_efforts: Collection[str] | None = None, ) -> dict: """ Get optional parameters for the responses API. @@ -766,6 +766,12 @@ class ResponsesAPIRequestUtils: model_id=model_id, response_id=decoded_response_id, ) + except ValueError: + return DecodedResponseId( + custom_llm_provider=None, + model_id=None, + response_id=response_id, + ) except Exception as e: verbose_logger.debug("Error decoding response_id '%s': %s", response_id, e) return DecodedResponseId( diff --git a/litellm/router.py b/litellm/router.py index fc888495e99..ff31f48c49e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -75,7 +75,7 @@ from typing_extensions import assert_never, overload import litellm import litellm.litellm_core_utils.exception_mapping_utils from litellm import get_secret_str -from litellm._internal_context import service_target, with_service_target +from litellm._internal_context import in_mirrored_call, mirrored_call, service_target, with_service_target from litellm._logging import verbose_router_logger from litellm._uuid import uuid from litellm.caching.caching import ( @@ -162,7 +162,11 @@ from litellm.litellm_core_utils.sensitive_data_masker import ( SensitiveDataMasker, mask_sensitive_structure, ) -from litellm.litellm_core_utils.token_counter import offload_token_count +from litellm.litellm_core_utils.token_counter import ( + offload_token_count, + served_openai_models, + single_encoding_openai_model, +) from litellm.litellm_core_utils.tool_fields import TOOL_FIELDS, drop_policy from litellm.llms.anthropic.common_utils import AnthropicModelInfo, anthropic_error_frame_exception from litellm.llms.base_llm.passthrough.transformation import replace_path_segment @@ -180,6 +184,7 @@ from litellm.llms.openai_like.model_info import ( get_openai_compatible_model_info, ) from litellm.responses.bridge_check import keep_registered_mode +from litellm.responses.retrieve_stream_query import requests_streamed_retrieval, without_retrieve_stream_defaults from litellm.router_strategy.auto_router.auto_router_model_naming import ( AUTO_ROUTER_MODEL_PREFIX, STRATEGY_ROUTER_PARAM_FIELDS, @@ -719,6 +724,29 @@ def _configured_model_info(deployment: Mapping[str, object]) -> Mapping[str, obj return _MODEL_INFO_ADAPTER.validate_python(deployment.get("model_info") or {}) +def _search_tool_support(deployment: DeploymentTypedDict) -> bool | None: + from litellm.responses.litellm_completion_transformation.tool_search import chat_config_builds_anthropic_request + + configured: Final = _configured_model_info(deployment).get("supports_search_tool") + if isinstance(configured, bool): + return configured + params: Final = _LITELLM_PARAMS_ADAPTER.validate_python(deployment["litellm_params"]) + model: Final = params.get("model") + provider: Final = params.get("custom_llm_provider") + if not isinstance(model, str): + return None + try: + resolved_model, resolved_provider = resolve_model_and_provider_for_metadata( + model=model, custom_llm_provider=provider if isinstance(provider, str) else None + ) + chat_config: Final = ProviderConfigManager.get_provider_chat_config( + model=resolved_model, provider=LlmProviders(resolved_provider) + ) + except (litellm.BadRequestError, ValueError): + return None + return True if chat_config_builds_anthropic_request(chat_config) else None + + def _every_deployment_refused( deployments: Sequence[Mapping[str, object]], refused_deployment_ids: tuple[str, ...] ) -> bool: @@ -726,7 +754,7 @@ def _every_deployment_refused( def _silent_experiment_targets(silent_model: object) -> tuple[str, ...]: - if silent_model is None: + if silent_model is None or in_mirrored_call(): return () try: targets: Final = _SILENT_MODEL_ADAPTER.validate_python(silent_model) @@ -1533,6 +1561,7 @@ class Router: self._cached_deployment_tracks_usage = lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)( self._deployment_tracks_usage ) + self.served_openai_models = lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)(self._served_openai_models) self._discovered_model_info_cache: InMemoryCache = InMemoryCache( max_size_in_memory=max(len(model_list or ()), 1), default_ttl=2 * MODEL_INFO_REFRESH_SECONDS, @@ -3343,16 +3372,17 @@ class Router: remaining_kwargs: Final = MappingProxyType( {key: value for key, value in silent_kwargs.items() if key != "stream"} ) - response: Final = await self.acompletion( - model=silent_model, - messages=cast(list[AllMessageValues], messages), - stream=bool(silent_kwargs.get("stream", False)), - **remaining_kwargs, - ) - if not isinstance(response, CustomStreamWrapper): - return - async for _ in response: - pass + with mirrored_call(): + response: Final = await self.acompletion( + model=silent_model, + messages=cast(list[AllMessageValues], messages), + stream=bool(silent_kwargs.get("stream", False)), + **remaining_kwargs, + ) + if not isinstance(response, CustomStreamWrapper): + return + async for _ in response: + pass def _silent_experiment_completion(self, silent_model: str, messages: Sequence[Mapping[str, str]], **kwargs): """ @@ -4832,7 +4862,7 @@ class Router: Handles inserting this as either "metadata" or "litellm_metadata" depending on the metadata_variable_name """ # 1) copy your defaults and pull out metadata - defaults: Final = self.default_litellm_params.copy() + defaults: Final = dict(without_retrieve_stream_defaults(self.default_litellm_params, kwargs)) metadata_defaults: Final = defaults.pop("metadata", {}) or {} # 2) add any non-metadata defaults that aren't already in kwargs @@ -4921,6 +4951,7 @@ class Router: deployment=deployment, forwarded_keys=kwargs.pop(_ALIAS_MARKER_FORWARDED_PARAMS_KWARG, ()) ): kwargs.pop(key, None) + kwargs.pop("daily_reset_time", None) # pyright: ignore[reportUnknownMemberType] # kwargs is an untyped dict defaults: Final = _LITELLM_PARAMS_ADAPTER.validate_python(self.default_litellm_params) self._merge_tools_from_deployment(deployment=deployment, kwargs=kwargs, defaults=defaults) @@ -5072,12 +5103,13 @@ class Router: router_timeout=_router_timeout, router_stream_timeout=_router_stream_timeout, ) + timeout_request: Final = {**kwargs, "stream": True} if requests_streamed_retrieval(kwargs) else kwargs kwargs.update( CompletionTimeout.stream_request_kwargs( - passthrough_timeout(kwargs={**kwargs, "stream": False}), passthrough_timeout(kwargs=kwargs) + passthrough_timeout(kwargs={**kwargs, "stream": False}), passthrough_timeout(kwargs=timeout_request) ) - if splits_stream_timeout and kwargs.get("stream") - else {"timeout": passthrough_timeout(kwargs=kwargs)} + if splits_stream_timeout and timeout_request.get("stream") + else {"timeout": passthrough_timeout(kwargs=timeout_request)} ) elif splits_stream_timeout: request_kwargs: Final = _REQUEST_KWARGS_ADAPTER.validate_python(kwargs) @@ -5668,7 +5700,7 @@ class Router: def _image_generation(self, prompt: str, model: str, **kwargs): model_name: Final = "" try: - verbose_router_logger.debug("Inside _image_generation()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside _image_generation()- model: %s", model) deployment: Final = self.get_available_deployment( model=model, messages=[{"role": "user", "content": "prompt"}], @@ -5730,7 +5762,7 @@ class Router: async def _aimage_generation(self, prompt: str, model: str, **kwargs): model_name = model try: - verbose_router_logger.debug("Inside _image_generation()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside _image_generation()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) deployment: Final = await self.async_get_available_deployment( model=model, @@ -5815,7 +5847,7 @@ class Router: async def _atranscription(self, file: FileTypes, model: str, **kwargs): model_name: Final = model try: - verbose_router_logger.debug("Inside _atranscription()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside _atranscription()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) deployment: Final = await self.async_get_available_deployment( model=model, @@ -5908,7 +5940,7 @@ class Router: async def _aspeech(self, model: str, input: str, voice: str | None = None, **kwargs): model_name: Final = model try: - verbose_router_logger.debug("Inside _aspeech()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside _aspeech()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) deployment: Final = await self.async_get_available_deployment( model=model, @@ -5970,7 +6002,7 @@ class Router: async def _arerank(self, model: str, **kwargs): model_name = None try: - verbose_router_logger.debug("Inside _rerank()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside _rerank()- model: %s", model) deployment: Final = await self.async_get_available_deployment( model=model, specific_deployment=kwargs.pop("specific_deployment", None), @@ -6079,7 +6111,7 @@ class Router: async def _atext_completion(self, model: str, prompt: str, **kwargs): try: - verbose_router_logger.debug("Inside _atext_completion()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside _atext_completion()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) deployment: Final = await self.async_get_available_deployment( model=model, @@ -6149,7 +6181,7 @@ class Router: async def _aadapter_completion(self, adapter_id: str, model: str, **kwargs): try: - verbose_router_logger.debug("Inside _aadapter_completion()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside _aadapter_completion()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) deployment: Final = await self.async_get_available_deployment( model=model, @@ -6243,7 +6275,7 @@ class Router: kwargs=kwargs, metadata_variable_name="litellm_metadata", ) - verbose_router_logger.debug("Inside aguardrail() - guardrail_name: %s; kwargs: %s", guardrail_name, kwargs) + verbose_router_logger.debug("Inside aguardrail() - guardrail_name: %s", guardrail_name) response: Final = await self.async_function_with_fallbacks(**kwargs) return response @@ -6324,9 +6356,7 @@ class Router: controls: Final = per_request_fallback_controls(kwargs) kwargs[MID_STREAM_FALLBACK_CONTROLS_KEY] = controls # rebind-ok: forwarded to every hop self._update_kwargs_before_fallbacks(model=model, kwargs=kwargs, metadata_variable_name="litellm_metadata") - verbose_router_logger.debug( - "Inside ageneric_api_call_with_fallbacks() - model: %s; kwargs: %s", model, kwargs - ) + verbose_router_logger.debug("Inside ageneric_api_call_with_fallbacks() - model: %s", model) response: Final = await self.async_function_with_fallbacks(**kwargs) return response except Exception as e: @@ -6413,7 +6443,7 @@ class Router: generic_function_name: Final[str] = getattr(original_generic_function, "__name__", "") response_kwargs: Final = { - **data, + **without_retrieve_stream_defaults(data, kwargs), "caching": self.cache_responses, **kwargs, "model": model_name, @@ -7146,9 +7176,7 @@ class Router: handler_name: Final = original_function.__name__ metadata_variable_name: Final = get_router_metadata_variable_name(function_name="generic_api_call") try: - verbose_router_logger.debug( - "Inside _generic_api_call() - handler: %s, model: %s; kwargs: %s", handler_name, model, kwargs - ) + verbose_router_logger.debug("Inside _generic_api_call() - handler: %s, model: %s", handler_name, model) self._update_kwargs_before_fallbacks( model=model, kwargs=kwargs, @@ -7223,7 +7251,7 @@ class Router: def _embedding(self, input: str | list[object], model: str, **kwargs): model_name = None try: - verbose_router_logger.debug("Inside embedding()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside embedding()- model: %s", model) with embedding_deployment_selection(): deployment: Final = self.get_available_deployment( model=model, @@ -7312,7 +7340,7 @@ class Router: async def _aembedding(self, input: str | list[object], model: str, **kwargs): model_name = None try: - verbose_router_logger.debug("Inside _aembedding()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside _aembedding()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) with embedding_deployment_selection(): deployment: Final = await self.async_get_available_deployment( @@ -7396,7 +7424,7 @@ class Router: try: from litellm.router_utils.common_utils import add_model_file_id_mappings - verbose_router_logger.debug("Inside _atext_completion()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside _atext_completion()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) healthy_deployments: Final = await self.async_get_healthy_deployments( model=model, @@ -7634,7 +7662,7 @@ class Router: **kwargs, ) -> LiteLLMBatch: try: - verbose_router_logger.debug("Inside _acreate_batch()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside _acreate_batch()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) with batch_create_routing(): deployment: Final = await self.async_get_available_deployment( @@ -7833,7 +7861,7 @@ class Router: **kwargs, ) -> LiteLLMBatch: try: - verbose_router_logger.debug("Inside _acancel_batch()- model: %s; kwargs: %s", model, kwargs) + verbose_router_logger.debug("Inside _acancel_batch()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) deployment: Final = await self.async_get_available_deployment( model=model, @@ -11893,6 +11921,9 @@ class Router: indices: Final = self.model_name_to_deployment_indices.get(model_name) return None if indices is None else tuple(self.model_list[idx] for idx in indices) + def _served_openai_models(self, model_name: str) -> tuple[str | None, ...]: + return served_openai_models(self.deployments_named(model_name) or ()) + def team_public_model_names_of(self, team_id: str) -> frozenset[str]: """The public names `team_id` serves its own deployments under.""" with self._deployment_index_lock: @@ -12229,6 +12260,15 @@ class Router: for deployment in self._routable_deployments(model_name, team_id) ) + def get_search_tool_support(self, model_name: str, team_id: str | None = None) -> tuple[bool | None, ...]: + """ + Whether each deployment a request from team_id to model_name can route + to (see _routable_deployments) answers Codex's client `tool_search`: + its model_info.supports_search_tool when that is a bool, else True when + its route builds an Anthropic Messages request, else None (unknown). + """ + return tuple(_search_tool_support(deployment) for deployment in self._routable_deployments(model_name, team_id)) + def get_routable_upstream_model(self, model_name: str, team_id: str | None = None) -> str | None: """ Return the `litellm_params.model` of the first deployment a request @@ -14047,6 +14087,7 @@ class Router: self._ptu_share_teams.cache_clear() self._cached_team_ptu_ceiling.cache_clear() self._cached_deployment_tracks_usage.cache_clear() + self.served_openai_models.cache_clear() self._zero_cost_cache.clear() self._routing_group_rows = None @@ -14306,6 +14347,7 @@ class Router: messages: Sequence[Mapping[str, object]] | None, input: str | list[object] | None, request_kwargs: Mapping[str, object] | None = None, + tokenizer_model: str = "", ) -> int: """ Count input tokens for context-window pre-call checks. @@ -14336,7 +14378,7 @@ class Router: system_message: Final = anthropic_system_to_openai_message(extras.get("system")) if messages is not None: counted_messages: Final = (system_message, *messages) if system_message is not None else messages - return litellm.token_counter(messages=counted_messages, tools=tools) + return litellm.token_counter(model=tokenizer_model, messages=counted_messages, tools=tools) if input is not None: from openai.types.responses.response_create_params import ResponseInputParam @@ -14351,6 +14393,7 @@ class Router: only_since_compaction=True, ) return litellm.token_counter( + model=tokenizer_model, messages=cast(list, input_messages), # cast-ok: transformed chat messages tools=tools, ) @@ -14407,6 +14450,10 @@ class Router: messages=cast(list[dict[str, str]] | None, messages), # cast-ok: forwarded to the sync counter input=cast(str | list | None, input), # cast-ok: forwarded to the sync counter request_kwargs=request_kwargs, + tokenizer_model=single_encoding_openai_model( + self.served_openai_models(model) or served_openai_models(healthy_deployments) + ) + or "", ) except Exception as e: # noqa: BLE001 # best-effort: an uncountable prompt must not fail the request verbose_router_logger.error( @@ -14421,6 +14468,8 @@ class Router: request_kwargs: Mapping[str, object] | None, input_token_count: int | None, count_inline: bool, + model: str, + deployments: Sequence[Mapping[str, object]], ) -> int | None: if input_token_count is not None: return input_token_count @@ -14431,6 +14480,10 @@ class Router: messages=cast(list[dict[str, str]] | None, messages), # cast-ok: forwarded to the sync counter input=cast(str | list[object] | None, input), # cast-ok: forwarded to the sync counter request_kwargs=request_kwargs, + tokenizer_model=single_encoding_openai_model( + self.served_openai_models(model) or served_openai_models(deployments) + ) + or "", ) except Exception as e: verbose_router_logger.error( @@ -14625,6 +14678,8 @@ class Router: count_inline=has_countable_input and not skip_inline_token_count and any(limit is not None for limit in deployment_max_input_tokens), + model=model, + deployments=_returned_deployments, ) request_settings: Final[Mapping[str, object]] = request_kwargs if request_kwargs is not None else {} requested_params: Final[frozenset[str]] = ( diff --git a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py index 2e350a2f87b..0b3d48c9e1d 100644 --- a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py @@ -22,7 +22,7 @@ from litellm import token_counter from litellm._internal_context import with_service_target from litellm._logging import verbose_router_logger from litellm.caching.dual_cache import DualCache -from litellm.litellm_core_utils.token_counter import offload_token_count +from litellm.litellm_core_utils.token_counter import deployment_openai_model, offload_token_count from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET from litellm.types.router import RouterCacheEnum, RouterErrors @@ -127,6 +127,12 @@ def _get_cache_keys(deployment: dict, current_minute: str) -> tuple[str, str] | return itpm_key, otpm_key +def _tokenizer_model(deployment: object) -> str: + params: Final = deployment.get("litellm_params") if is_str_keyed_mapping(deployment) else None + served: Final = params.get("model") if is_str_keyed_mapping(params) else None + return deployment_openai_model(deployment) or (served if isinstance(served, str) else "") + + def _estimate_input_tokens(request_kwargs: dict[str, Any] | None, model: str = "") -> int: if not request_kwargs: return 0 @@ -427,7 +433,7 @@ def io_token_pre_call_check( return deployment request_kwargs: Final = get_io_token_rate_limit_request_kwargs() - _model: Final = (deployment.get("litellm_params") or {}).get("model") or "" + _model: Final = _tokenizer_model(deployment) estimated_input: Final = _estimate_input_tokens(request_kwargs, model=_model) max_tokens: Final = _resolve_max_tokens(request_kwargs, deployment) @@ -491,7 +497,7 @@ async def async_io_token_pre_call_check( return deployment request_kwargs: Final = get_io_token_rate_limit_request_kwargs() - _model: Final = (deployment.get("litellm_params") or {}).get("model") or "" + _model: Final = _tokenizer_model(deployment) estimated_input: Final = await offload_token_count(_estimate_input_tokens)(request_kwargs, model=_model) max_tokens: Final = _resolve_max_tokens(request_kwargs, deployment) diff --git a/litellm/router_utils/pre_call_checks/model_rate_limit_check.py b/litellm/router_utils/pre_call_checks/model_rate_limit_check.py index 9eebcf39515..f4b41211c6e 100644 --- a/litellm/router_utils/pre_call_checks/model_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/model_rate_limit_check.py @@ -12,7 +12,7 @@ is logged the first time such a deployment is seen. """ import contextlib -from collections.abc import Callable +from collections.abc import Callable, Mapping from datetime import datetime, timedelta from typing import TYPE_CHECKING, Any, Final, Literal @@ -38,7 +38,7 @@ from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import ( ) from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET from litellm.types.caching import RedisPipelineIncrementOperation -from litellm.types.router import RouterErrors +from litellm.types.router import RouterErrors, parse_daily_reset_time from litellm.types.utils import StandardLoggingPayload from litellm.utils import get_utc_datetime @@ -58,42 +58,49 @@ class RoutingArgs: DAY_FORMAT: Final = "%Y-%m-%d" -def seconds_until_utc_midnight(now: datetime) -> int: - next_midnight: Final = (now + timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0) - return max(1, int((next_midnight - now).total_seconds())) +def daily_window_day(now: datetime, reset_offset: timedelta) -> str: + return (now - reset_offset).strftime(DAY_FORMAT) -def _first_set(*values: int | None) -> int | None: - return next((value for value in values if value is not None), None) +def seconds_until_daily_reset(now: datetime, reset_offset: timedelta = timedelta(0)) -> int: + window_start: Final = (now - reset_offset).replace(hour=0, minute=0, second=0, microsecond=0) + next_reset: Final = window_start + timedelta(days=1) + reset_offset + return max(1, int((next_reset - now).total_seconds())) -class _DailyLimitFields(BaseModel): +class _DailyLimitParams(BaseModel): + """rpd, tpd and daily_reset_time live in litellm_params only.""" + model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True) + model: str | None = None tpd: int | None = None rpd: int | None = None + daily_reset_time: str | None = None + + def reset_offset(self) -> timedelta: + return parse_daily_reset_time(self.daily_reset_time) -class _DailyLimitParams(_DailyLimitFields): - model: str | None = None - - -class _DailyLimitModelInfo(_DailyLimitFields): +class _DailyLimitModelInfo(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True) id: str | None = None -class _DailyLimitDeployment(_DailyLimitFields): +class _DailyLimitDeployment(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True) model_name: str = "" litellm_params: _DailyLimitParams = _DailyLimitParams() model_info: _DailyLimitModelInfo = _DailyLimitModelInfo() def limits(self) -> tuple[int | None, int | None]: - return ( - _first_set(self.tpd, self.litellm_params.tpd, self.model_info.tpd), - _first_set(self.rpd, self.litellm_params.rpd, self.model_info.rpd), - ) + return self.litellm_params.tpd, self.litellm_params.rpd - def cache_keys(self, current_day: str) -> tuple[str, str]: + def reset_offset(self) -> timedelta: + return self.litellm_params.reset_offset() + + def cache_keys(self, now: datetime) -> tuple[str, str]: prefix: Final = f"{self.model_info.id}:{self.litellm_params.model}" + current_day: Final = daily_window_day(now, self.reset_offset()) return f"{prefix}:tpd:{current_day}", f"{prefix}:rpd:{current_day}" def limit_error(self, kind: Literal["tpd", "rpd"], limit: int, usage: int, now: datetime) -> litellm.RateLimitError: @@ -104,12 +111,19 @@ class _DailyLimitDeployment(_DailyLimitFields): response=httpx.Response( status_code=429, content=f"{RouterErrors.user_defined_ratelimit_error.value} {kind} limit={limit}. current usage={usage}. id={self.model_info.id}, model_group={self.model_name}", - headers={"retry-after": str(seconds_until_utc_midnight(now))}, + headers={"retry-after": str(seconds_until_daily_reset(now, self.reset_offset()))}, request=httpx.Request(method="model_rate_limit_check", url="https://github.com/BerriAI/litellm"), ), ) +def _reset_offset_from_kwargs(kwargs: Mapping[str, object]) -> timedelta: + litellm_params: Final = kwargs.get("litellm_params") + if litellm_params is None: + return timedelta(0) + return _DailyLimitParams.model_validate(litellm_params).reset_offset() + + class ModelRateLimitingCheck(CustomLogger): """ Pre-call check that enforces TPM/RPM or ITPM/OTPM limits on model deployments. @@ -242,7 +256,7 @@ class ModelRateLimitingCheck(CustomLogger): tpd_limit, rpd_limit = deployment.limits() if tpd_limit is None and rpd_limit is None: return - tpd_key, rpd_key = deployment.cache_keys(now.strftime(DAY_FORMAT)) + tpd_key, rpd_key = deployment.cache_keys(now) if tpd_limit is not None: current_tpd: Final = self._get_current_tpm(tpd_key, tpd_limit) if current_tpd is not None and current_tpd >= tpd_limit: @@ -394,7 +408,7 @@ class ModelRateLimitingCheck(CustomLogger): current_minute: Final = dt.strftime("%H-%M") tpm_key, rpm_key = self._get_cache_keys(deployment, current_minute) - tpd_key, rpd_key = daily.cache_keys(dt.strftime(DAY_FORMAT)) + tpd_key, rpd_key = daily.cache_keys(dt) model_id: Final = deployment.get("model_info", {}).get("id") model_name: Final = deployment.get("litellm_params", {}).get("model") @@ -510,7 +524,8 @@ class ModelRateLimitingCheck(CustomLogger): dt: Final = self.clock() current_minute: Final = dt.strftime("%H-%M") tpm_key: Final = f"{model_id}:{model}:tpm:{current_minute}" - tpd_key: Final = f"{model_id}:{model}:tpd:{dt.strftime(DAY_FORMAT)}" + reset_offset: Final = _reset_offset_from_kwargs(kwargs) # pyright: ignore[reportUnknownArgumentType] # CustomLogger passes an untyped kwargs dict + tpd_key: Final = f"{model_id}:{model}:tpd:{daily_window_day(dt, reset_offset)}" verbose_router_logger.debug("[TPM TRACKING] Incrementing %s by %s", tpm_key, total_tokens) @@ -570,7 +585,8 @@ class ModelRateLimitingCheck(CustomLogger): dt: Final = self.clock() current_minute: Final = dt.strftime("%H-%M") tpm_key: Final = f"{model_id}:{model}:tpm:{current_minute}" - tpd_key: Final = f"{model_id}:{model}:tpd:{dt.strftime(DAY_FORMAT)}" + reset_offset: Final = _reset_offset_from_kwargs(kwargs) # pyright: ignore[reportUnknownArgumentType] # CustomLogger passes an untyped kwargs dict + tpd_key: Final = f"{model_id}:{model}:tpd:{daily_window_day(dt, reset_offset)}" tokens: Final = int(total_tokens) self._increment(tpm_key, tokens, RoutingArgs.ttl) diff --git a/litellm/router_utils/reasoning_effort_capability.py b/litellm/router_utils/reasoning_effort_capability.py index 16b8a22a40f..3c3fad93579 100644 --- a/litellm/router_utils/reasoning_effort_capability.py +++ b/litellm/router_utils/reasoning_effort_capability.py @@ -426,6 +426,27 @@ def _log_remap( logged_deployment_id: Final = _single_line(deployment_id) logged_model_group: Final = _single_line(model_group) logged_deployment_model: Final = _single_line(deployment_model) + extra: Final = MappingProxyType( + { + "reasoning_effort_deployment_id": logged_deployment_id, + "reasoning_effort_model_group": logged_model_group, + "reasoning_effort_deployment_model": logged_deployment_model, + "reasoning_effort_requested": logged_requested_level, + "reasoning_effort_sent": logged_sent_level, + "reasoning_effort_rounding": rounding, + } + ) + if sent_level is None and requested_level != _DEFAULT_EFFORT: + verbose_logger.warning( + "Dropping reasoning level %s (drop_params=True) for deployment %s, model group %s, model %s because its " + "model_info.reasoning_effort_levels does not list that level. List every level the deployment supports.", + logged_requested_level, + logged_deployment_id, + logged_model_group, + logged_deployment_model, + extra=extra, + ) + return verbose_logger.info( "reasoning_effort %s sent as %s (rounding %s) to deployment %s, model group %s, model %s", logged_requested_level, @@ -434,14 +455,7 @@ def _log_remap( logged_deployment_id, logged_model_group, logged_deployment_model, - extra={ - "reasoning_effort_deployment_id": logged_deployment_id, - "reasoning_effort_model_group": logged_model_group, - "reasoning_effort_deployment_model": logged_deployment_model, - "reasoning_effort_requested": logged_requested_level, - "reasoning_effort_sent": logged_sent_level, - "reasoning_effort_rounding": rounding, - }, + extra=extra, ) diff --git a/litellm/router_utils/search_api_router.py b/litellm/router_utils/search_api_router.py index 1cfb311d796..26326a7e6d4 100644 --- a/litellm/router_utils/search_api_router.py +++ b/litellm/router_utils/search_api_router.py @@ -155,10 +155,9 @@ class SearchAPIRouter: available_search_tool_names: Final = [tool.get("search_tool_name") for tool in router_instance.search_tools] verbose_router_logger.debug( - "Inside SearchAPIRouter.async_search_with_fallbacks() - search_tool_name: %s, Available Search Tools: %s, kwargs: %s", + "Inside SearchAPIRouter.async_search_with_fallbacks() - search_tool_name: %s, Available Search Tools: %s", search_tool_name, available_search_tool_names, - kwargs, ) # Use the existing retry/fallback infrastructure diff --git a/litellm/rust_bridge/catalog.py b/litellm/rust_bridge/catalog.py index 74cd28161a4..fe52f8416d0 100644 --- a/litellm/rust_bridge/catalog.py +++ b/litellm/rust_bridge/catalog.py @@ -96,6 +96,8 @@ class Feature(str, Enum): """litellm.include_cost_in_streaming_usage, where Python adds cost to streamed usage.""" THINKING_SIGNATURE_RETRY = "thinking_signature_retry" """Replayed thinking blocks, which Python strips and retries once if Anthropic rejects their signature.""" + TOOL_CUSTOM_FIELD = "tool_custom_field" + """Tools carrying a non-spec custom object, which Python drops or hoists for the provider.""" CLIENT_HEADERS = "client_headers" """Proxy-forwarded headers that Python merges into extra_headers.""" DEPLOYMENT_HOOK = "deployment_hook" @@ -176,6 +178,7 @@ MESSAGES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | { Feature.CACHE_CONTROL_INJECTION, Feature.STREAM_COST_INJECTION, Feature.THINKING_SIGNATURE_RETRY, + Feature.TOOL_CUSTOM_FIELD, } RESPONSES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | { diff --git a/litellm/rust_bridge/host/routes/ocr.py b/litellm/rust_bridge/host/routes/ocr.py index 8f563a381ef..827d0ecbbe0 100644 --- a/litellm/rust_bridge/host/routes/ocr.py +++ b/litellm/rust_bridge/host/routes/ocr.py @@ -7,12 +7,13 @@ from typing import Final from pydantic import TypeAdapter import litellm +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.llms.base_llm.ocr.transformation import PROVIDER_NATIVE_RESPONSE_KEY, OCRResponse from litellm.rust_bridge import failures from litellm.rust_bridge.failures import UpstreamFailure from litellm.rust_bridge.public_call import optional_str -__all__ = ("UpstreamFailure", "map_failure", "response") +__all__ = ("UpstreamFailure", "drops_unsupported_params", "map_failure", "response") _RESPONSE_ADAPTER: Final = TypeAdapter(dict[str, object]) diff --git a/litellm/rust_bridge/messages/detectors.py b/litellm/rust_bridge/messages/detectors.py index 5f75858d350..52917126b3b 100644 --- a/litellm/rust_bridge/messages/detectors.py +++ b/litellm/rust_bridge/messages/detectors.py @@ -43,6 +43,11 @@ def thinking_signature_retry(request: NativeCall) -> bool: return any(_replays_thinking(message) for message in optional_sequence(request.resolved.get("messages")) or ()) +def tool_custom_field(request: NativeCall) -> bool: + tools: Final = optional_sequence(request.resolved.get("tools")) or () + return any("custom" in (optional_mapping(tool) or {}) for tool in tools) + + def stream_cost_injection(request: NativeCall) -> bool: """Python adds response cost to streaming usage when the global setting is truthy.""" return streaming(request) and bool(litellm.include_cost_in_streaming_usage) diff --git a/litellm/tool_loop.py b/litellm/tool_loop.py index aaa0d62eea7..d0b2850bb5b 100644 --- a/litellm/tool_loop.py +++ b/litellm/tool_loop.py @@ -63,6 +63,8 @@ def _assistant_tool_call(tool_call: ChatCompletionMessageToolCall) -> ChatComple def _assistant_message( message: Message, tool_calls: tuple[ChatCompletionMessageToolCall, ...], replays_provider_specific_fields: bool ) -> ChatCompletionAssistantMessage: + from litellm.completion_extras.litellm_responses_transformation.history import retained_replay_fields + return cast( # cast-ok: dict literal with the optional message fields spread in only when set "ChatCompletionAssistantMessage", { @@ -78,7 +80,7 @@ def _assistant_message( "provider_specific_fields", getattr(message, "provider_specific_fields", None) if replays_provider_specific_fields - else None, + else retained_replay_fields(getattr(message, "provider_specific_fields", None)), ), ) if value is not None diff --git a/litellm/types/litellm_params.py b/litellm/types/litellm_params.py index 9196b7a226f..2065093f295 100644 --- a/litellm/types/litellm_params.py +++ b/litellm/types/litellm_params.py @@ -191,6 +191,7 @@ class DeploymentOptions: otpm: int | None = None rpd: int | None = None tpd: int | None = None + daily_reset_time: str | None = None kubernetes_pod_discovery: bool | None = None default_api_key_rpm_limit: int | None = None default_api_key_tpm_limit: int | None = None diff --git a/litellm/types/llms/azure_ai.py b/litellm/types/llms/azure_ai.py index 722bc53f429..419e6c0cec5 100644 --- a/litellm/types/llms/azure_ai.py +++ b/litellm/types/llms/azure_ai.py @@ -1,6 +1,6 @@ from typing import Literal -from typing_extensions import Required, TypedDict +from typing_extensions import NotRequired, ReadOnly, Required, TypedDict class ImageEmbeddingInput(TypedDict, total=False): @@ -15,3 +15,9 @@ class ImageEmbeddingRequest(TypedDict, total=False): input: Required[list[ImageEmbeddingInput]] dimensions: int encoding_format: EncodingFormat + + +class FoundryAgentReference(TypedDict): + type: ReadOnly[Literal["agent_reference"]] + name: ReadOnly[str] + version: ReadOnly[NotRequired[str]] diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 8e1db87c16f..6ce51abcd8a 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1,5 +1,5 @@ import builtins -from collections.abc import Iterable, Mapping +from collections.abc import Iterable, Mapping, Sequence from enum import Enum from os import PathLike from typing import IO, TYPE_CHECKING, Any, Final, Generic, Literal, Optional, TypeAlias, Union @@ -66,6 +66,7 @@ from pydantic import ( ConfigDict, Discriminator, Field, + FieldSerializationInfo, NonNegativeInt, PrivateAttr, SerializerFunctionWrapHandler, @@ -1487,15 +1488,66 @@ One of: completed, failed, in_progress, cancelled, queued, or incomplete. """ -_NULL_DROPPED_OUTPUT_ITEM_KEYS: Final = frozenset( - ( - ("reasoning", "status"), - ("reasoning", "content"), - ("reasoning", "encrypted_content"), - ("tool_search_call", "created_by"), - ("tool_search_output", "created_by"), - ) -) +_NULL_DROPPED_REASONING_ITEM_KEYS: Final = frozenset(("status", "content", "encrypted_content")) + + +def _as_json_object(value: object) -> Mapping[str, object] | None: + return value if isinstance(value, dict) else None + + +def _as_json_array(value: object) -> Sequence[object] | None: + return value if isinstance(value, list) else None + + +def _drops_created_by(value: object, *, drops_set_created_by: bool) -> bool: + return value is None or drops_set_created_by + + +def _drops(item_type: object, key: object, value: object, *, drops_set_created_by: bool) -> bool: + if key == "created_by": + return _drops_created_by(value, drops_set_created_by=drops_set_created_by) + return value is None and item_type == "reasoning" and key in _NULL_DROPPED_REASONING_ITEM_KEYS + + +def _without_created_by(entry: object, *, drops_set_created_by: bool) -> object: + entry_object: Final = _as_json_object(entry) + if entry_object is None: + return entry + return { + k: v + for k, v in entry_object.items() + if k != "created_by" or not _drops_created_by(v, drops_set_created_by=drops_set_created_by) + } + + +def _replayable_output_value(item_type: object, key: object, value: object, *, drops_set_created_by: bool) -> object: + entries: Final = _as_json_array(value) + if item_type != "shell_call_output" or key != "output" or entries is None: + return value + return [_without_created_by(entry, drops_set_created_by=drops_set_created_by) for entry in entries] + + +def replayable_responses_item( + item: Mapping[str, object], *, drops_set_created_by: bool = False +) -> Mapping[str, object]: + item_type: Final = item.get("type") + return { + k: _replayable_output_value(item_type, k, v, drops_set_created_by=drops_set_created_by) + for k, v in item.items() + if not _drops(item_type, k, v, drops_set_created_by=drops_set_created_by) + } + + +def replayable_responses_input(value: object, *, drops_set_created_by: bool = False) -> object: + items: Final = _as_json_array(value) + if items is None: + return value + return [ + replayable_responses_item(obj, drops_set_created_by=drops_set_created_by) + if (obj := _as_json_object(item)) is not None + else item + for item in items + ] class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject): @@ -1570,34 +1622,16 @@ class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject): @field_serializer("output", mode="wrap") @classmethod - def _serialize_output_filter_reasoning_nulls(cls, value, handler, _info): + def _serialize_output_as_replayable( + cls, value: builtins.object, handler: SerializerFunctionWrapHandler, _info: FieldSerializationInfo + ): """ - Filter null status/content/encrypted_content from reasoning output items. - - Mirrors the request-side filtering in - OpenAIResponsesAPIConfig._handle_reasoning_item() which filters these - same fields before sending requests to providers. - - Without this, reasoning items include null fields that cause SDK errors - (e.g., the OpenAI C# SDK crashes on status=null). - - Issue: https://github.com/BerriAI/litellm/issues/16824 + Drop the nulls the SDK types default in but providers never send: created_by on every output item + (OpenAI rejects it when `output` is replayed as `input`), and status/content/encrypted_content on + reasoning items (the OpenAI C# SDK crashes on status=null, issue #16824). """ - serialized: Final = handler(value) - if not isinstance(serialized, list): - return serialized - return [ - ( - { - k: v - for k, v in item.items() - if v is not None or (item.get("type"), k) not in _NULL_DROPPED_OUTPUT_ITEM_KEYS - } - if isinstance(item, dict) - else item - ) - for item in serialized - ] + serialized: Final[object] = handler(value) + return replayable_responses_input(serialized) @property def output_text(self) -> str: diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index f1176f777c7..7cb5432d009 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -376,7 +376,7 @@ class RequestBody(TypedDict, total=False): class CachedContentRequestBody(TypedDict, total=False): contents: Required[list[ContentType]] system_instruction: SystemInstructions - tools: Tools + tools: Tools | None toolConfig: ToolConfig model: Required[str] # Format: models/{model} ttl: str # ending in 's' - Example: "3.5s". diff --git a/litellm/types/proxy/discovery_endpoints/ui_discovery_endpoints.py b/litellm/types/proxy/discovery_endpoints/ui_discovery_endpoints.py index d396c79b844..ca6ada0d1ec 100644 --- a/litellm/types/proxy/discovery_endpoints/ui_discovery_endpoints.py +++ b/litellm/types/proxy/discovery_endpoints/ui_discovery_endpoints.py @@ -15,4 +15,5 @@ class UiDiscoveryEndpoints(LiteLLMBaseModel): is_control_plane: bool = False mcp_stdio_enabled: bool = False use_si_terminology: bool = False + claude_code_marketplace_auth_required: bool = False workers: list[WorkerRegistryEntry] = Field(default=[]) diff --git a/litellm/types/router.py b/litellm/types/router.py index f8de2128a80..db8be9e91a3 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -4,6 +4,7 @@ litellm.Router Types - includes RouterConfig, UpdateRouterConfig, ModelInfo etc import datetime import enum +import re from collections.abc import Container, Mapping, Sequence from dataclasses import dataclass, field from typing import ( @@ -66,6 +67,10 @@ from .utils import ( AllowedModelRegion: TypeAlias = Literal["eu", "us"] +class SilentExperimentFlag(TypedDict, total=False): + is_silent_experiment: ReadOnly[bool] + + class FallbackSkipReason(str, enum.Enum): NOT_AUTHORIZED = "not_authorized" OVER_BUDGET = "over_budget" @@ -518,6 +523,19 @@ def holds_secret_pointer(param_name: str) -> bool: _RESERVED_INIT_KEYS: Final = frozenset({"self", "params", "__class__"}) +_DAILY_RESET_TIME_PATTERN: Final = re.compile(r"\d{2}:\d{2}") + + +def parse_daily_reset_time(value: str | None) -> datetime.timedelta: + """`daily_reset_time` is "HH:MM" in UTC, when the rpd/tpd window starts over; unset means midnight UTC.""" + if value is None: + return datetime.timedelta(0) + if _DAILY_RESET_TIME_PATTERN.fullmatch(value) is None: + raise ValueError(f"daily_reset_time must be HH:MM in UTC, got {value!r}") + parsed: Final = datetime.time.fromisoformat(value) + return datetime.timedelta(hours=parsed.hour, minutes=parsed.minute) + + class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): """ LiteLLM Params without 'model' arg (used across completion / assistants api) @@ -535,6 +553,7 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): otpm: int | None = None tpd: int | None = None rpd: int | None = None + daily_reset_time: str | None = None timeout: float | str | httpx.Timeout | SDKTimeout | None = None # if str, pass in as os.environ/ # read timeout of a live stream, a faked stream takes the longer of this and timeout. If str, pass in as os.environ/ stream_timeout: float | str | None = None @@ -626,6 +645,13 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): valkey_text_field: str | None = None valkey_embedding_field: str | None = None + @field_validator("daily_reset_time") + @classmethod + def _daily_reset_time_is_utc_hh_mm(cls, value: str | None) -> str | None: + if value is not None: + parse_daily_reset_time(value) + return value + @field_validator("timeout") @classmethod def normalize_timeout( @@ -742,6 +768,7 @@ class LiteLLMParamsTypedDict(TypedDict, total=False): otpm: int | None tpd: ReadOnly[int | None] rpd: ReadOnly[int | None] + daily_reset_time: ReadOnly[str | None] order: int | None weight: int | None max_parallel_requests: int | None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index b21bf0e14d4..e9984a35ad1 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -263,10 +263,10 @@ class AgenticLoopParams(TypedDict, total=False): """ model: str - """The model string with provider prefix (e.g., 'bedrock/invoke/...')""" + """The model as the caller requested it, usually with its provider prefix (e.g., 'bedrock/invoke/...')""" custom_llm_provider: str - """The LLM provider name (e.g., 'bedrock', 'anthropic')""" + """The LLM provider name (e.g., 'bedrock', 'anthropic'), when the recorder resolved one""" class OffPeakWindow(TypedDict, total=False): @@ -2878,7 +2878,7 @@ class TranscriptionUsageTokensObject(LiteLLMBaseModel): input_tokens: int output_tokens: int total_tokens: int - input_token_details: TranscriptionUsageInputTokenDetailsObject + input_token_details: TranscriptionUsageInputTokenDetailsObject | None = None class TranscriptionResponse(OpenAIObject): @@ -3374,6 +3374,7 @@ class StandardLoggingMetadata(StandardLoggingUserAPIKeyMetadata): team_alias: str | None team_id: str | None used_client_oauth_token: ReadOnly[bool | None] + is_silent_experiment: ReadOnly[NotRequired[bool]] class AzureSpillover(TypedDict): @@ -4130,6 +4131,18 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): """ return {k: v for k, v in model_info.items() if k not in cls.model_fields} + def custom_pricing_overrides(self) -> Mapping[str, object]: + return MappingProxyType( + self.model_dump(include=set(CustomPricingLiteLLMParams.model_fields), exclude_none=True) + ) + + def per_second_pricing(self) -> Mapping[str, float]: + return MappingProxyType( + self.model_dump( + include={"cost_per_second", "input_cost_per_second", "output_cost_per_second"}, exclude_none=True + ) + ) + DEPLOYMENT_SCOPED_PRICING_FIELDS: Final[frozenset[str]] = frozenset({"off_peak_pricing"}) @@ -4424,6 +4437,7 @@ class LlmProviders(str, Enum): JINA_AI = "jina_ai" XAI = "xai" ZAI = "zai" + BIGMODEL = "bigmodel" CUSTOM_OPENAI = "custom_openai" TEXT_COMPLETION_OPENAI = "text-completion-openai" COHERE = "cohere" @@ -4593,10 +4607,12 @@ class LlmProviders(str, Enum): META = "meta" SAIL = "sail" REKA = "reka" + STREAMLAKE = "streamlake" LITELLM_AGENT = "litellm_agent" CURSOR = "cursor" BEDROCK_MANTLE = "bedrock_mantle" GDC = "gdc" + KYMA = "kyma" # Create a set of all provider values for quick lookup diff --git a/litellm/utils.py b/litellm/utils.py index be1e11dd39f..a656d8ebca3 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -29,8 +29,8 @@ import threading import time import traceback from contextlib import nullcontext -from dataclasses import dataclass, field -from functools import lru_cache, wraps +from dataclasses import dataclass, field, replace +from functools import lru_cache, partial, wraps from importlib import resources from inspect import iscoroutine from io import StringIO @@ -91,7 +91,7 @@ from litellm.litellm_core_utils.fallback_generalizations import ( match_capability_generalizations, match_fill_missing_generalizations, ) -from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry +from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry, GetModelCostMap from litellm.litellm_core_utils.model_param_helper import peek_reasoning_summary_aliases from litellm.litellm_core_utils.offline_mode import is_offline_mode from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload @@ -220,6 +220,7 @@ def _get_cached_audio_utils(): from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper as _TextCompletionStreamWrapper +from litellm.types.llms.base_model import LiteLLMBaseModel from litellm.types.llms.openai import ( AllMessageValues, AllPromptValues, @@ -304,7 +305,7 @@ except (ImportError, AttributeError, TypeError): # Convert to str (if necessary) claude_json_str = json.dumps(json_data) import importlib.metadata -from collections.abc import AsyncIterator, Callable, Collection, Iterable, Iterator, Mapping, Sequence +from collections.abc import AsyncIterator, Callable, Collection, Iterable, Iterator, Mapping, Sequence, Sized from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypeVar, cast, runtime_checkable from typing_extensions import assert_never @@ -606,9 +607,23 @@ def custom_llm_setup(): litellm.provider_list.append(custom_llm["provider"]) if custom_llm["provider"] not in litellm._custom_providers: + _warn_if_built_in_provider_shadows_custom_handler(custom_llm["provider"]) litellm._custom_providers.append(custom_llm["provider"]) +def _warn_if_built_in_provider_shadows_custom_handler(provider: str) -> None: + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + if provider not in LlmProvidersSet and not JSONProviderRegistry.exists(provider): + return + verbose_logger.warning( + "custom_provider_map provider '%s' has the same name as a built-in LiteLLM provider, which handles " + "'%s/' models instead of your custom_handler. Rename the provider and its models to use your handler", + provider, + provider, + ) + + def add_custom_logger_callback_to_specific_event(callback: str, logging_event: Literal["success", "failure"]) -> None: """ Add a custom logger callback to the specific event @@ -2018,16 +2033,15 @@ def client(original_function): # MODEL CALL result = original_function(*args, **kwargs) end_time = datetime.datetime.now() - if _is_streaming_request( - kwargs=kwargs, - call_type=call_type, - ): + if _is_streaming_request(kwargs=kwargs, call_type=call_type) or _is_converted_stream_result(result): if "complete_response" in kwargs and kwargs["complete_response"] is True: chunks = [] for idx, chunk in enumerate(result): chunks.append(chunk) return litellm.stream_chunk_builder(chunks, messages=kwargs.get("messages", None)) else: + logging_obj.stream = True + logging_obj.model_call_details["stream"] = True # RETURN RESULT update_response_metadata: _ResponseMetadataUpdater = litellm_utils.update_response_metadata update_response_metadata( @@ -2564,6 +2578,8 @@ _NEVER_STREAMING_CALL_TYPES: Final = frozenset( } ) +_RESULT_DECIDES_STREAMING_CALL_TYPES: Final = frozenset({"get_responses", "aget_responses"}) + def _is_streaming_request( kwargs: dict[str, object], @@ -2575,7 +2591,7 @@ def _is_streaming_request( - if "stream=True" in kwargs (litellm chat completion, litellm text completion, litellm messages) - if call_type is generate_content_stream or agenerate_content_stream (litellm google genai) """ - if call_type in _NEVER_STREAMING_CALL_TYPES: + if call_type in _NEVER_STREAMING_CALL_TYPES or call_type in _RESULT_DECIDES_STREAMING_CALL_TYPES: return False if "stream" in kwargs and kwargs["stream"] is True: return True @@ -2668,9 +2684,99 @@ def huggingface_tokenizer_kind(model: str) -> HuggingFaceTokenizerKind | None: return None +@dataclass(frozen=True, slots=True) +class _HuggingFaceTokenizerLoaded: + tokenizer: HuggingFace + + @dataclass(frozen=True, slots=True) class _HuggingFaceTokenizerLoadFailure: reason: str + retry_at: float + retry_pid: int | None = None + + +_HuggingFaceTokenizerKey: TypeAlias = tuple[HuggingFaceTokenizerKind, Decision] +_HuggingFaceTokenizerLoadOutcome: TypeAlias = _HuggingFaceTokenizerLoaded | _HuggingFaceTokenizerLoadFailure + + +def _start_in_thread_joined_at_exit(work: Callable[[], None]) -> None: + threading.Thread(target=work, name="litellm-huggingface-tokenizer-retry", daemon=False).start() + + +@dataclass(slots=True) +class _HuggingFaceTokenizerLoadOutcomes: + clock: Callable[[], float] = time.monotonic + start_retry: Callable[[Callable[[], None]], None] = _start_in_thread_joined_at_exit + pid: Callable[[], int] = os.getpid + lock: threading.Lock = field(default_factory=threading.Lock) + by_tokenizer: Mapping[_HuggingFaceTokenizerKey, _HuggingFaceTokenizerLoadOutcome] = field( + default_factory=lambda: MappingProxyType({}) + ) + + def known( + self, key: _HuggingFaceTokenizerKey, retry_when_due: Callable[[], None] + ) -> _HuggingFaceTokenizerLoadOutcome | None: + with self.lock: + known: Final = self.by_tokenizer.get(key) + if ( + not isinstance(known, _HuggingFaceTokenizerLoadFailure) + or known.retry_pid == self.pid() + or self.clock() < known.retry_at + ): + return known + retrying: Final = replace(known, retry_pid=self.pid()) + self._remember(key, retrying) + try: + self.start_retry(retry_when_due) + except RuntimeError as error: + return self.remember_failed_retry(key, str(error)) + return retrying + + def remember_loaded(self, key: _HuggingFaceTokenizerKey, tokenizer: HuggingFace) -> _HuggingFaceTokenizerLoaded: + loaded: Final = _HuggingFaceTokenizerLoaded(tokenizer) + with self.lock: + self._remember(key, loaded) + return loaded + + def remember_first_failure(self, key: _HuggingFaceTokenizerKey, reason: str) -> _HuggingFaceTokenizerLoadOutcome: + with self.lock: + known: Final = self.by_tokenizer.get(key) + if known is not None: + return known + failure: Final = self._failure(reason) + self._remember(key, failure) + return failure + + def remember_failed_retry(self, key: _HuggingFaceTokenizerKey, reason: str) -> _HuggingFaceTokenizerLoadOutcome: + with self.lock: + known: Final = self.by_tokenizer.get(key) + if isinstance(known, _HuggingFaceTokenizerLoaded): + return known + failure: Final = self._failure(reason) + self._remember(key, failure) + return failure + + def end_retry(self, key: _HuggingFaceTokenizerKey) -> None: + with self.lock: + known: Final = self.by_tokenizer.get(key) + if isinstance(known, _HuggingFaceTokenizerLoadFailure) and known.retry_pid is not None: + self._remember(key, self._failure(known.reason)) + + def flush_cache(self) -> None: + with self.lock: + self.by_tokenizer = MappingProxyType({}) + + def _failure(self, reason: str) -> _HuggingFaceTokenizerLoadFailure: + return _HuggingFaceTokenizerLoadFailure( + reason=reason, retry_at=self.clock() + HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS + ) + + def _remember(self, key: _HuggingFaceTokenizerKey, outcome: _HuggingFaceTokenizerLoadOutcome) -> None: + self.by_tokenizer = MappingProxyType({**self.by_tokenizer, key: outcome}) + + +_HUGGINGFACE_TOKENIZER_LOAD_OUTCOMES: Final = _HuggingFaceTokenizerLoadOutcomes() def _warn_huggingface_tokenizer_fallback(model: str, reason: str) -> None: @@ -2682,30 +2788,28 @@ def _warn_huggingface_tokenizer_fallback(model: str, reason: str) -> None: ) -def _return_huggingface_tokenizer(model: str) -> SelectTokenizerResponse | _HuggingFaceTokenizerLoadFailure | None: +def _return_huggingface_tokenizer( + model: str, outcomes: _HuggingFaceTokenizerLoadOutcomes = _HUGGINGFACE_TOKENIZER_LOAD_OUTCOMES +) -> SelectTokenizerResponse | _HuggingFaceTokenizerLoadFailure | None: kind: Final = huggingface_tokenizer_kind(model) if kind is None or (kind not in _BUNDLED_TOKENIZER_LOAD_LOCKS and is_offline_mode()): return None - backend: Final = _huggingface_tokenizer_backend() - key: Final = _huggingface_tokenizer_failure_key(kind, backend) + key: Final = (kind, _huggingface_tokenizer_backend()) with _BUNDLED_TOKENIZER_LOAD_LOCKS.get(kind, nullcontext()): - cached_failure: Final = _HUGGINGFACE_TOKENIZER_LOAD_FAILURES.get_cache(key) - if isinstance(cached_failure, _HuggingFaceTokenizerLoadFailure): - return cached_failure - tokenizer: Final = _load_huggingface_tokenizer_recording_failure(kind, backend, key) - if isinstance(tokenizer, _HuggingFaceTokenizerLoadFailure): - _warn_huggingface_tokenizer_fallback(model, tokenizer.reason) - return tokenizer - return {"type": "huggingface_tokenizer", "tokenizer": tokenizer} + known: Final = outcomes.known(key, partial(_retry_huggingface_tokenizer_load, model, key, outcomes)) + outcome: Final = ( + known + if known is not None + else _load_huggingface_tokenizer_remembering_outcome(model, key, outcomes, outcomes.remember_first_failure) + ) + match outcome: + case _HuggingFaceTokenizerLoaded(tokenizer=tokenizer): + return {"type": "huggingface_tokenizer", "tokenizer": tokenizer} + case _HuggingFaceTokenizerLoadFailure(): + return outcome -def _huggingface_tokenizer_failure_key(kind: HuggingFaceTokenizerKind, backend: Decision) -> str: - return f"{kind}:{backend.name}" - - -@lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE) -def _load_huggingface_tokenizer(kind: HuggingFaceTokenizerKind, backend: Decision) -> HuggingFace: - """One tokenizer per kind and backend; `backend` is the cache key, the dispatch re-derives it.""" +def _load_huggingface_tokenizer(kind: HuggingFaceTokenizerKind) -> HuggingFace: match kind: case "cohere": return tokenizer_dispatch.from_pretrained("Xenova/c4ai-command-r-v01-tokenizer") @@ -2725,19 +2829,34 @@ def _load_huggingface_tokenizer(kind: HuggingFaceTokenizerKind, backend: Decisio assert_never(kind) -def _load_huggingface_tokenizer_recording_failure( - kind: HuggingFaceTokenizerKind, backend: Decision, key: str -) -> HuggingFace | _HuggingFaceTokenizerLoadFailure: +def _load_huggingface_tokenizer_remembering_outcome( + model: str, + key: _HuggingFaceTokenizerKey, + outcomes: _HuggingFaceTokenizerLoadOutcomes, + remember_failure: Callable[[_HuggingFaceTokenizerKey, str], _HuggingFaceTokenizerLoadOutcome], +) -> _HuggingFaceTokenizerLoadOutcome: + kind, _ = key try: - return _load_huggingface_tokenizer(kind, backend) + tokenizer: Final = _load_huggingface_tokenizer(kind) except Exception as error: from litellm.rust_bridge.fork_guard import ForkedAfterNativeRuntimeStarted, ProcessReservedForForking if isinstance(error, (ForkedAfterNativeRuntimeStarted, ProcessReservedForForking)): raise - failure: Final = _HuggingFaceTokenizerLoadFailure(reason=str(error)) - _HUGGINGFACE_TOKENIZER_LOAD_FAILURES.set_cache(key, failure, ttl=HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS) - return failure + outcome: Final = remember_failure(key, str(error)) + if isinstance(outcome, _HuggingFaceTokenizerLoadFailure): + _warn_huggingface_tokenizer_fallback(model, outcome.reason) + return outcome + return outcomes.remember_loaded(key, tokenizer) + + +def _retry_huggingface_tokenizer_load( + model: str, key: _HuggingFaceTokenizerKey, outcomes: _HuggingFaceTokenizerLoadOutcomes +) -> None: + try: + _load_huggingface_tokenizer_remembering_outcome(model, key, outcomes, outcomes.remember_failed_retry) + finally: + outcomes.end_retry(key) def encode(model="", text="", custom_tokenizer: dict | None = None): @@ -3383,6 +3502,42 @@ def supports_mid_conversation_system(model: str, custom_llm_provider: str | None ) +class _DeploymentModelInfo(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + + model_info: Mapping[str, object] | None = None + + +def deployment_model_info(litellm_params: object) -> Mapping[str, object] | None: + """The deployment's own ``model_info`` read off its litellm params.""" + return _DeploymentModelInfo.model_validate(litellm_params).model_info + + +def deployment_declared_mid_conversation_system_support(model_info: object) -> bool | None: + match model_info: + case {"supports_mid_conversation_system": bool(declared)}: + return declared + case _: + return None + + +def declared_mid_conversation_system_support( + model: str, custom_llm_provider: str | None, model_info: object +) -> bool | None: + """ + The `supports_mid_conversation_system` the deployment's own model_info declares, else the one the + cost map declares for the model, else None when neither sets it. + """ + deployment_declared: Final = deployment_declared_mid_conversation_system_support(model_info) + if deployment_declared is not None: + return deployment_declared + if is_explicitly_disabled_factory(model, custom_llm_provider, "supports_mid_conversation_system"): + return False + if supports_mid_conversation_system(model, custom_llm_provider): + return True + return None + + def supports_native_structured_output(model: str, custom_llm_provider: str | None = None) -> bool: """ Check if the given model supports native structured outputs and return a boolean value. @@ -3697,7 +3852,7 @@ def _resolve_builtin_model_cost_entry(key: str, provider: str) -> _BuiltinCostMa return None -def is_generalized_model_info(model_info: ModelInfo) -> bool: +def is_generalized_model_info(model_info: ModelInfoBase) -> bool: """Whether ``model_info`` came from a fallback-generalization capability rule. Detected as the resolved key missing ``litellm.model_cost`` while matching a @@ -3713,10 +3868,12 @@ def is_generalized_model_info(model_info: ModelInfo) -> bool: _REGISTRATION_ATTEMPTS: Final = 3 +_ModelInfoT: Final = TypeVar("_ModelInfoT", bound=ModelInfoBase) -def lookup_model_info_for_registration(model: str, custom_llm_provider: str | None) -> ModelInfo | ModelInfoLookupMiss: + +def _lookup_mapped_model_info(lookup: Callable[[], _ModelInfoT]) -> _ModelInfoT | ModelInfoLookupMiss: try: - info: Final = get_model_info(model=model, custom_llm_provider=custom_llm_provider) + info: Final = lookup() except ModelNotMappedError: return ModelInfoLookupMiss(tripped=False) except Exception: @@ -3724,6 +3881,16 @@ def lookup_model_info_for_registration(model: str, custom_llm_provider: str | No return ModelInfoLookupMiss(tripped=False) if is_generalized_model_info(info) else info +def lookup_model_info_for_registration(model: str, custom_llm_provider: str | None) -> ModelInfo | ModelInfoLookupMiss: + return _lookup_mapped_model_info(lambda: get_model_info(model=model, custom_llm_provider=custom_llm_provider)) + + +def lookup_catalog_model_info(model: str, custom_llm_provider: str | None) -> ModelInfoBase | ModelInfoLookupMiss: + return _lookup_mapped_model_info( + lambda: get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider, catalog_only=True) + ) + + def _get_builtin_model_info_for_registration( model: str, custom_llm_provider: str | None ) -> ModelInfo | ModelInfoLookupMiss: @@ -4067,7 +4234,9 @@ def get_optional_params_transcription( "timestamp_granularities": None, } - non_default_params = {k: v for k, v in passed_params.items() if (k in default_params and v != default_params[k])} + non_default_params: Final = { + k: v for k, v in passed_params.items() if (k in default_params and v != default_params[k]) + } optional_params = {} ## raise exception if non-default value passed for non-openai/azure embedding calls @@ -4104,6 +4273,19 @@ def get_optional_params_transcription( model=model, drop_params=drop_params if drop_params is not None else False, ) + elif custom_llm_provider == "bedrock": + from litellm.llms.bedrock.audio_transcription import ( + UnsupportedTranscriptionParams, + map_openai_transcription_params, + ) + + match map_openai_transcription_params( + non_default_params, drop_params=drop_params is True or litellm.drop_params is True + ): + case UnsupportedTranscriptionParams(message=message): + raise UnsupportedParamsError(message=message, llm_provider="bedrock") + case mapped_params: + optional_params = dict(mapped_params) elif provider_config is not None: # custom audio transcription config supported_params = provider_config.get_supported_openai_params(model=model) _check_valid_arg(supported_params=supported_params) @@ -4118,7 +4300,7 @@ def get_optional_params_transcription( optional_params=optional_params, passed_params=passed_params, custom_llm_provider=custom_llm_provider, - openai_params=OPENAI_TRANSCRIPTION_PARAMS, + openai_params=list(default_params) if custom_llm_provider == "bedrock" else OPENAI_TRANSCRIPTION_PARAMS, additional_drop_params=kwargs.get("additional_drop_params", None), ) @@ -4974,6 +5156,46 @@ def _declared_reasoning_efforts(model_info: Mapping[str, object] | None) -> tupl return deployment_reasoning_effort_levels(model_info) or () +_DROPPED_REASONING_PARAMS: Final = ("reasoning_effort", "thinking") + + +def _turns_reasoning_off(value: object) -> bool: + match value: + case "none" | {"effort": "none"} | {"type": "disabled"}: + return True + case _: + return False + + +def is_known_non_reasoning_model(model: str, custom_llm_provider: str | None) -> bool: + info: Final = lookup_catalog_model_info(model, custom_llm_provider) + if isinstance(info, ModelInfoLookupMiss) or info.get("supports_reasoning") is True: + return False + return info.get("supports_reasoning") is False or info["key"] in GetModelCostMap.loaded_model_cost_map() + + +def has_catalog_entry(model: str, custom_llm_provider: str | None) -> bool: + info: Final = lookup_catalog_model_info(model, custom_llm_provider) + return not isinstance(info, ModelInfoLookupMiss) and info["key"] in GetModelCostMap.loaded_model_cost_map() + + +def _dropped_reasoning_warning(model: str, custom_llm_provider: str | None) -> str: + return "".join( + ( + "Dropping %s=%r for model=%r (drop_params=True) because %s does not support %s for it.", + ( + "" + if has_catalog_entry(model, custom_llm_provider) + else " If the deployment is a reasoning model, set model_info.base_model to its underlying model." + ), + ) + ) + + +def _warns_on_dropped_reasoning(value: object, model: str, custom_llm_provider: str | None) -> bool: + return not (_turns_reasoning_off(value) and is_known_non_reasoning_model(model, custom_llm_provider)) + + def get_optional_params( # use the openai defaults # https://platform.openai.com/docs/api-reference/chat/create @@ -5059,21 +5281,34 @@ def get_optional_params( supported_params: List[str] - supported params from the litellm config """ verbose_logger.info("\nLiteLLM completion() model= %s; provider = %s", model, custom_llm_provider) - verbose_logger.debug("\nLiteLLM: Params passed to completion() %s", passed_params) - verbose_logger.debug("\nLiteLLM: Non-Default params passed to completion() %s", non_default_params) - unsupported_params: Final = {} - for k in non_default_params: - if k not in supported_params: - if k in PROVIDER_UNVALIDATED_PARAMS: - continue - if k == "n" and n == 1: # langchain sends n=1 as a default value - continue # skip this param - unsupported_params[k] = non_default_params[k] + if verbose_logger.isEnabledFor(logging.DEBUG): + verbose_logger.debug( + "\nLiteLLM: Non-Default params passed to completion() %s", _debug_params_repr(non_default_params) + ) + unsupported_params: Final[MappingProxyType[str, object]] = MappingProxyType( + { + k: non_default_params[k] + for k in non_default_params + if k not in supported_params and k not in PROVIDER_UNVALIDATED_PARAMS and not (k == "n" and n == 1) + } + ) if unsupported_params: if litellm.drop_params is True or (drop_params is not None and drop_params is True): for k in unsupported_params: non_default_params.pop(k, None) + for reasoning_param in _DROPPED_REASONING_PARAMS: + if reasoning_param in unsupported_params and _warns_on_dropped_reasoning( + unsupported_params[reasoning_param], base_model or model, custom_llm_provider + ): + verbose_logger.warning( + _dropped_reasoning_warning(base_model or model, custom_llm_provider), + reasoning_param, + unsupported_params[reasoning_param], + model, + custom_llm_provider, + reasoning_param, + ) else: raise UnsupportedParamsError( status_code=500, @@ -5601,7 +5836,9 @@ def get_optional_params( additional_drop_params=additional_drop_params, ) if _print_verbose_is_active(): - print_verbose(f"Final returned optional params: {redact_credentials_in_payload(optional_params)}") + print_verbose( + f"Final returned optional params: {_debug_params_repr(redact_credentials_in_payload(optional_params))}" + ) optional_params = apply_openai_param_overrides( optional_params=optional_params, non_default_params=non_default_params, @@ -5641,6 +5878,7 @@ def add_provider_specific_params_to_optional_params( "openai", "azure", "text-completion-openai", + "bigmodel", *litellm.openai_compatible_providers, ] or JSONProviderRegistry.exists(custom_llm_provider) if calls_through_openai_sdk: @@ -6586,6 +6824,8 @@ def get_model_info_helper( custom_llm_provider: str | None = None, api_base: str | None = None, api_key: str | None = None, + *, + catalog_only: bool = False, ) -> ModelInfoBase: """ Helper for 'get_model_info'. Separated out to avoid infinite loop caused by returning 'supported_openai_param's @@ -6610,8 +6850,6 @@ def get_model_info_helper( model=model, custom_llm_provider=custom_llm_provider or declared_authenticating_provider(model) ) - verbose_logger.debug("checking potential_model_names in litellm.model_cost: %s", potential_model_names) - combined_model_name: Final = potential_model_names["combined_model_name"] stripped_model_name: Final = potential_model_names["stripped_model_name"] combined_stripped_model_name: Final = potential_model_names["combined_stripped_model_name"] @@ -6621,7 +6859,7 @@ def get_model_info_helper( model_cost_custom_llm_provider: Final = custom_llm_provider ######################### provider_config: BaseLLMModelInfo | None = None - if custom_llm_provider and custom_llm_provider in LlmProvidersSet: + if not catalog_only and custom_llm_provider and custom_llm_provider in LlmProvidersSet: provider_config = ProviderConfigManager.get_provider_model_info( model=model, provider=LlmProviders(custom_llm_provider) ) @@ -6645,7 +6883,7 @@ def get_model_info_helper( e, ) - if custom_llm_provider == "huggingface": + if not catalog_only and custom_llm_provider == "huggingface": max_tokens: Final = _get_max_position_embeddings(model_name=model) return ModelInfoBase( key=model, @@ -6714,6 +6952,7 @@ def get_model_info_helper( key, _model_info = generalization if _model_info is None or key is None: + verbose_logger.debug("model not found in litellm.model_cost, tried: %s", potential_model_names) raise ModelNotMappedError(_model_not_mapped_message(model, custom_llm_provider)) _input_cost_per_token: float | None = _model_info.get("input_cost_per_token") if _input_cost_per_token is None: @@ -8312,26 +8551,6 @@ def trim_messages( from litellm.caching.in_memory_cache import InMemoryCache -class _HuggingFaceTokenizerLoadFailureCache(Protocol): - def get_cache(self, key: str) -> object | None: ... - - def set_cache(self, key: str, value: object, ttl: float | None = None) -> None: ... - - def flush_cache(self) -> None: ... - - -_HUGGINGFACE_TOKENIZER_LOAD_FAILURES: Final[_HuggingFaceTokenizerLoadFailureCache] = ( - cast( # cast-ok: cache methods are untyped - _HuggingFaceTokenizerLoadFailureCache, - InMemoryCache( - max_size_in_memory=16, - default_ttl=HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS, - clock=lambda: time.monotonic(), - ), - ) -) - - class AvailableModelsCache(InMemoryCache): def __init__(self, ttl_seconds: int = 300, max_size: int = 1000): super().__init__(ttl_seconds, max_size) @@ -8533,6 +8752,23 @@ def get_valid_models( return [] # NON-Blocking +_MAX_INLINE_DEBUG_ARG_CHARS: Final = 500 + + +def _debug_arg_repr(value: object) -> str: + """Short form of a call argument, since the outgoing provider request log already prints the request body.""" + rendered: Final = repr(value) + if len(rendered) <= _MAX_INLINE_DEBUG_ARG_CHARS: + return rendered + if isinstance(value, Sized): + return f"<{type(value).__name__} len={len(value)}>" + return f"<{type(value).__name__}>" + + +def _debug_params_repr(params: Mapping[str, object]) -> str: + return "{" + ", ".join(f"{key!r}: {_debug_arg_repr(value)}" for key, value in params.items()) + "}" + + def print_args_passed_to_litellm(original_function, args, kwargs): if not is_debugging_on(): return @@ -8553,9 +8789,9 @@ def print_args_passed_to_litellm(original_function, args, kwargs): ): return - args_str: Final = ", ".join(map(repr, args)) + args_str: Final = ", ".join(map(_debug_arg_repr, args)) redacted_kwargs: Final = redact_credentials_in_payload(kwargs) - kwargs_str: Final = ", ".join(f"{key}={value!r}" for key, value in redacted_kwargs.items()) + kwargs_str: Final = ", ".join(f"{key}={_debug_arg_repr(value)}" for key, value in redacted_kwargs.items()) print_verbose( "\n", ) # new line before @@ -9134,6 +9370,7 @@ class ProviderConfigManager: LlmProviders.DATABRICKS: (lambda: litellm.DatabricksConfig(), False), LlmProviders.XAI: (lambda: litellm.XAIChatConfig(), False), LlmProviders.ZAI: (lambda: litellm.ZAIChatConfig(), False), + LlmProviders.BIGMODEL: (ProviderConfigManager._get_bigmodel_chat_config, False), LlmProviders.OPENCODE_GO: (ProviderConfigManager.get_opencode_go_config, True), LlmProviders.LAMBDA_AI: (lambda: litellm.LambdaAIChatConfig(), False), LlmProviders.INCEPTION: (lambda: litellm.InceptionChatConfig(), False), @@ -9326,6 +9563,12 @@ class ProviderConfigManager: return get_bedrock_chat_config(model=model) + @staticmethod + def _get_bigmodel_chat_config() -> BaseConfig: + from litellm.llms.bigmodel.chat.transformation import BigModelChatConfig + + return BigModelChatConfig() + @staticmethod def get_opencode_go_config(model: str) -> BaseConfig: from litellm.llms.opencode.chat.transformation import go_chat_config @@ -9647,6 +9890,10 @@ class ProviderConfigManager: model: str, provider: LlmProviders, ) -> BaseAnthropicMessagesConfig | None: + if provider == LlmProviders.BIGMODEL: + from litellm.llms.bigmodel.messages.transformation import BigModelMessagesConfig + + return BigModelMessagesConfig() if provider == LlmProviders.OPENCODE_GO: from litellm.llms.opencode.common_utils import go_endpoint from litellm.llms.opencode.messages.transformation import OpenCodeGoMessagesConfig @@ -9878,6 +10125,10 @@ class ProviderConfigManager: if provider is None: return None + if provider == LlmProviders.BIGMODEL: + from litellm.llms.bigmodel.responses.transformation import BigModelResponsesAPIConfig + + return BigModelResponsesAPIConfig() if provider == LlmProviders.OPENCODE_GO: from litellm.llms.opencode.common_utils import go_endpoint from litellm.llms.opencode.responses.transformation import OpenCodeGoResponsesConfig @@ -9899,8 +10150,14 @@ class ProviderConfigManager: elif litellm.LlmProviders.AZURE_AI == provider: from litellm.llms.azure_ai.common_utils import ( azure_ai_supports_native_responses, + is_foundry_project_agent, + ) + from litellm.llms.azure_ai.responses.transformation import ( + AzureAIAgentResponsesAPIConfig, ) + if is_foundry_project_agent(model): + return AzureAIAgentResponsesAPIConfig() if azure_ai_supports_native_responses(model, api_base): return litellm.AzureAIResponsesAPIConfig() return None diff --git a/litellm/vector_stores/vector_store_registry.py b/litellm/vector_stores/vector_store_registry.py index b93d9874dbc..598b56f4a79 100644 --- a/litellm/vector_stores/vector_store_registry.py +++ b/litellm/vector_stores/vector_store_registry.py @@ -36,7 +36,7 @@ else: _DB_ROW: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True)) _ITEMS: Final = TypeAdapter(Sequence[object], config=ConfigDict(strict=True)) _TOOL: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(strict=True)) -_STORE_ID_ITEMS: Final = TypeAdapter(list[object], config=ConfigDict(strict=True)) +_STORE_ID_ITEMS: Final = TypeAdapter(list[object] | tuple[object, ...], config=ConfigDict(strict=True)) def _named_store_ids(store_ids: object) -> tuple[str, ...]: @@ -49,7 +49,11 @@ def _named_store_ids(store_ids: object) -> tuple[str, ...]: return tuple(store_id for store_id in items if isinstance(store_id, str)) -def _tool_vector_store_ids(tool: object) -> tuple[str, ...]: +def listed_vector_store_ids(store_ids: object) -> tuple[str, ...]: + return _named_store_ids(store_ids) if isinstance(store_ids, (list, tuple)) else () + + +def vector_store_ids_in_tool(tool: object) -> tuple[str, ...]: try: return _named_store_ids(_TOOL.validate_python(tool).get("vector_store_ids")) except ValidationError: @@ -195,7 +199,7 @@ class VectorStoreRegistry: tools_to_remove: Final[list[int]] = [] for i, tool in enumerate(tools): - tool_vector_store_ids = _tool_vector_store_ids(tool) + tool_vector_store_ids = vector_store_ids_in_tool(tool) if not tool_vector_store_ids: continue @@ -305,7 +309,7 @@ class VectorStoreRegistry: List of vector stores with tool parameters merged into litellm_params """ # Pop vector_store_ids from params - vector_store_ids: Final[list[str]] = non_default_params.pop("vector_store_ids", None) or [] + vector_store_ids: Final = [*listed_vector_store_ids(non_default_params.pop("vector_store_ids", None))] # Extract params from tools and collect IDs params_by_id = self.get_and_pop_recognised_vector_store_tools(tools=tools, vector_store_ids=vector_store_ids) @@ -353,7 +357,7 @@ class VectorStoreRegistry: List of vector stores with tool parameters merged into litellm_params """ # Pop vector_store_ids from params - vector_store_ids: Final[list[str]] = non_default_params.pop("vector_store_ids", None) or [] + vector_store_ids: Final = [*listed_vector_store_ids(non_default_params.pop("vector_store_ids", None))] # Extract params from tools and collect IDs params_by_id = self.get_and_pop_recognised_vector_store_tools(tools=tools, vector_store_ids=vector_store_ids) @@ -423,7 +427,7 @@ class VectorStoreRegistry: validated_tools: Final = _ITEMS.validate_python(tools) except ValidationError: return vector_store_ids - return [*vector_store_ids, *chain.from_iterable(_tool_vector_store_ids(tool) for tool in validated_tools)] + return [*vector_store_ids, *chain.from_iterable(vector_store_ids_in_tool(tool) for tool in validated_tools)] def load_vector_stores_from_config(self, vector_stores_config: list[dict]): """ diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 8719565a96f..4fad4857c60 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -6101,6 +6101,7 @@ "text", "audio" ], + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, @@ -6133,6 +6134,7 @@ "text", "audio" ], + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, @@ -6164,6 +6166,7 @@ "text", "audio" ], + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, @@ -6230,6 +6233,7 @@ "text", "audio" ], + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, @@ -6293,6 +6297,7 @@ "text", "audio" ], + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, @@ -6369,6 +6374,7 @@ "text", "audio" ], + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, @@ -28065,7 +28071,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": true }, "fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct-hf": { "input_cost_per_token": 1.2e-06, @@ -28339,7 +28345,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": true }, "fireworks_ai/qwen3p7-plus": { "cache_read_input_token_cost": 8e-08, @@ -41933,6 +41939,120 @@ "supports_system_messages": true, "supports_native_structured_output": true }, + "bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 4.12e-08, + "input_cost_per_token_batches": 2.06e-08, + "input_cost_per_token_flex": 2.06e-08, + "input_cost_per_token_priority": 7.21e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 4.12e-08, + "output_cost_per_token_batches": 2.06e-08, + "output_cost_per_token_flex": 2.06e-08, + "output_cost_per_token_priority": 7.21e-08, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/ap-southeast-2/mistral.voxtral-small-24b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 1.03e-07, + "input_cost_per_token_batches": 5.15e-08, + "input_cost_per_token_flex": 5.15e-08, + "input_cost_per_token_priority": 1.8025e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 3.09e-07, + "output_cost_per_token_batches": 1.545e-07, + "output_cost_per_token_flex": 1.545e-07, + "output_cost_per_token_priority": 5.4075e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/ap-northeast-1/mistral.voxtral-mini-3b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 5e-08, + "input_cost_per_token_batches": 3e-08, + "input_cost_per_token_flex": 3e-08, + "input_cost_per_token_priority": 9e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 5e-08, + "output_cost_per_token_batches": 3e-08, + "output_cost_per_token_flex": 3e-08, + "output_cost_per_token_priority": 9e-08, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-northeast-1/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/ap-northeast-1/mistral.voxtral-small-24b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 1.2e-07, + "input_cost_per_token_batches": 6e-08, + "input_cost_per_token_flex": 6e-08, + "input_cost_per_token_priority": 2.1e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 3.6e-07, + "output_cost_per_token_batches": 1.8e-07, + "output_cost_per_token_flex": 1.8e-07, + "output_cost_per_token_priority": 6.3e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-northeast-1/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/eu-west-2/mistral.voxtral-mini-3b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 6e-08, + "input_cost_per_token_batches": 3e-08, + "input_cost_per_token_flex": 3e-08, + "input_cost_per_token_priority": 1.1e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 6e-08, + "output_cost_per_token_batches": 3e-08, + "output_cost_per_token_flex": 3e-08, + "output_cost_per_token_priority": 1.1e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/eu-west-2/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/eu-west-2/mistral.voxtral-small-24b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 1.6e-07, + "input_cost_per_token_batches": 8e-08, + "input_cost_per_token_flex": 8e-08, + "input_cost_per_token_priority": 2.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 4.7e-07, + "output_cost_per_token_batches": 2.3e-07, + "output_cost_per_token_flex": 2.3e-07, + "output_cost_per_token_priority": 8.2e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/eu-west-2/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, "mistral/codestral-2508": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 3e-07, @@ -61269,6 +61389,359 @@ "supports_tool_choice": true, "supports_vision": true }, + "streamlake/deepseek-v4-flash-0731": { + "cache_read_input_token_cost": 1.4e-08, + "input_cost_per_token": 4.4e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 1.32e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/DeepSeek-V4-Pro-0813": { + "cache_read_input_token_cost": 4.4e-08, + "input_cost_per_token": 1.32e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 1048566, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 3.96e-06, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/DeepSeek-V3": { + "cache_read_input_token_cost": 1.14e-07, + "input_cost_per_token": 2.86e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 56000, + "max_output_tokens": 8000, + "mode": "chat", + "output_cost_per_token": 1.143e-06, + "supports_function_calling": true, + "supports_reasoning": false, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/GLM-5.3": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_assistant_prefill": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/GLM-5.3-Flash": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_assistant_prefill": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/Kimi-K2-Instruct-0905": { + "cache_read_input_token_cost": 1.43e-07, + "input_cost_per_token": 5.71e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 224000, + "max_output_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 2.286e-06, + "supports_function_calling": true, + "supports_reasoning": false, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-r1": { + "cache_read_input_token_cost": 2.29e-07, + "input_cost_per_token": 5.71e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 98304, + "max_output_tokens": 16000, + "mode": "chat", + "output_cost_per_token": 2.286e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-v3.1": { + "cache_read_input_token_cost": 2.29e-07, + "input_cost_per_token": 5.71e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 98304, + "max_output_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 1.714e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-v4-flash": { + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 1.4e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 2.8e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-v4-pro": { + "cache_read_input_token_cost": 1.45e-07, + "input_cost_per_token": 1.74e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 3.48e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-v4.1-flash-1017": { + "cache_read_input_token_cost": 6e-09, + "input_cost_per_token": 3e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/deepseek-v4.1-flash-260910": { + "cache_read_input_token_cost": 6e-09, + "input_cost_per_token": 3e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 1048568, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/glm-5": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 202752, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3.2e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/glm-5.1": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 202752, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/glm-5.2": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 1048566, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/kimi-k2.6": { + "cache_read_input_token_cost": 1.6e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 262144, + "max_output_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 4e-06, + "supports_function_calling": true, + "supports_vision": true, + "supports_reasoning": false, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/kimi-k2.7-code": { + "cache_read_input_token_cost": 1.9e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 262144, + "max_output_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 4e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/mimo-v2.6-flash": { + "cache_read_input_token_cost": 2.8e-09, + "input_cost_per_token": 1.4e-07, + "litellm_provider": "streamlake", + "mode": "chat", + "output_cost_per_token": 2.8e-07, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/mimo-v2.6-pro": { + "cache_read_input_token_cost": 3.6e-09, + "input_cost_per_token": 4.35e-07, + "litellm_provider": "streamlake", + "mode": "chat", + "output_cost_per_token": 8.7e-07, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/minimax-m2.5": { + "cache_creation_input_token_cost": 3.75e-07, + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 196608, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/minimax-m3": { + "cache_read_input_token_cost": 1.2e-07, + "cache_read_input_token_cost_above_512k_tokens": 2.4e-07, + "input_cost_per_token": 6e-07, + "input_cost_per_token_above_512k_tokens": 1.2e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 1000000, + "max_output_tokens": 512000, + "mode": "chat", + "output_cost_per_token": 2.4e-06, + "output_cost_per_token_above_512k_tokens": 4.8e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/qwen3.5-397b-a17b": { + "cache_read_input_token_cost": 1.2e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 258048, + "max_output_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/qwen3.6-27b": { + "input_cost_per_token": 6e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 229376, + "max_output_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "supports_function_calling": true, + "supports_vision": true, + "supports_reasoning": false, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/qwen3.6-35b-a3b": { + "input_cost_per_token": 3.75e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 229376, + "max_output_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.25e-06, + "supports_function_calling": true, + "supports_vision": true, + "supports_reasoning": false, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/qwen3.8-2.4t-a95b": { + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 983616, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 6e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "cache_creation_input_token_cost": 2.5e-06, + "explicit_cache_read_input_token_cost": 1.7e-07, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/qwen3.8-27b": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 5e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 983616, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "cache_creation_input_token_cost": 6.25e-07, + "explicit_cache_read_input_token_cost": 5e-08, + "supports_prompt_caching": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, "deepseek-v4-flash": { "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 6e-09, @@ -62584,6 +63057,14 @@ "litellm_provider": "anthropic" } }, + { + "name": "typesafe-jev-ids", + "pattern": "^jev-(?:latest|preview|1\\.13\\.0|1\\.14\\.0)$", + "description": "A bare Jev id that has a typesafe/ price entry (jev-latest, jev-preview, jev-1.13.0, jev-1.14.0), the name the /typesafe pass-through forwards to TypeSafe. Listed by name, matched case-insensitively like every rule here, so foo/jev-latest or a model group named like a Jev id TypeSafe does not serve, such as jev-1.0 or jev-internal-router, never resolves to typesafe. A new Jev version goes here together with its price entry. Provider inference feeds the proxy's provider/* wildcard access checks, so this rule is what lets typesafe/* on a key, team or project allowlist cover a jev-latest request.", + "model_info": { + "litellm_provider": "typesafe" + } + }, { "name": "claude-family-baseline", "pattern": "claude-[a-z]+-\\d+(?:[-.]\\d+)?(?:-\\d{8})?", @@ -63174,6 +63655,7 @@ "output_cost_per_token_priority": 9e-06, "source": "https://api.fireworks.ai/v1/serverless/models", "supports_function_calling": true, + "supports_mid_conversation_system": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -63320,6 +63802,7 @@ "output_cost_per_token_priority": 9e-06, "source": "https://api.fireworks.ai/v1/serverless/models", "supports_function_calling": true, + "supports_mid_conversation_system": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -77858,6 +78341,1254 @@ "supports_xhigh_reasoning_effort": true, "use_openai_responses_path": true }, + "kyma/claude-fable-5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-05, + "output_cost_per_token": 6.75e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true + }, + "kyma/claude-fable-5.1": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-05, + "output_cost_per_token": 6.75e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true + }, + "kyma/claude-haiku-4-5": { + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "input_cost_per_token": 1.35e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 6.75e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false, + "prompt_cache_min_tokens": 4096 + }, + "kyma/claude-haiku-5.5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-07, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 6.75e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": false, + "prompt_cache_min_tokens": 512 + }, + "kyma/claude-opus-4-7": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 6.75e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 3.375e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 2048 + }, + "kyma/claude-opus-5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 6.75e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 3.375e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 512 + }, + "kyma/claude-opus-5.5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512 + }, + "kyma/claude-sonnet-4-6": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 4.05e-06, + "cache_read_input_token_cost": 4.05e-07, + "output_cost_per_token": 2.025e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 1024 + }, + "kyma/claude-sonnet-5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.35e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 1024 + }, + "kyma/claude-sonnet-5.5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 1.35e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512 + }, + "kyma/deepseek-r1": { + "max_input_tokens": 64000, + "max_output_tokens": 32768, + "input_cost_per_token": 7.425e-07, + "output_cost_per_token": 2.957e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v3": { + "max_input_tokens": 160000, + "max_output_tokens": 8192, + "input_cost_per_token": 5.18e-07, + "output_cost_per_token": 7.571e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v4-flash": { + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "input_cost_per_token": 1.23e-07, + "cache_read_input_token_cost": 2.43e-08, + "output_cost_per_token": 2.458e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v4-flash-vision-exp": { + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "input_cost_per_token": 3.892e-07, + "output_cost_per_token": 1.168e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v4.1-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 2.7e-07, + "output_cost_per_token": 8.1e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 6.75e-07, + "cache_read_input_token_cost": 6.75e-08, + "output_cost_per_token": 4.05e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3.1-pro": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.62e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemini-3.5-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 2.025e-06, + "cache_read_input_token_cost": 2.025e-07, + "output_cost_per_token": 1.215e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3.5-flash-lite": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 4.05e-07, + "cache_read_input_token_cost": 4.05e-08, + "output_cost_per_token": 3.375e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3.6-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.013e-06, + "cache_read_input_token_cost": 1.0125e-07, + "output_cost_per_token": 5.063e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": false, + "supports_pdf_input": false + }, + "kyma/gemini-3.7-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 2.026e-06, + "cache_read_input_token_cost": 1.0125e-07, + "output_cost_per_token": 1.0126e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemini-3.8-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 2.026e-06, + "output_cost_per_token": 1.0126e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemini-4-argon": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemma-4-31b": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.642e-07, + "cache_read_input_token_cost": 1.642e-07, + "output_cost_per_token": 4.689e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-4.5-air": { + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "input_cost_per_token": 1.797e-07, + "cache_read_input_token_cost": 3.375e-08, + "output_cost_per_token": 1.175e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-4.7-flash": { + "max_input_tokens": 203000, + "max_output_tokens": 65536, + "input_cost_per_token": 8.1e-08, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 5.4e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.1": { + "max_input_tokens": 203000, + "max_output_tokens": 65536, + "input_cost_per_token": 1.89e-06, + "cache_read_input_token_cost": 2.7675e-07, + "output_cost_per_token": 5.94e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.2": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 9.029e-07, + "cache_read_input_token_cost": 2.9835e-07, + "output_cost_per_token": 3.083e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.3": { + "max_input_tokens": 1310720, + "max_output_tokens": 131072, + "input_cost_per_token": 1.178e-06, + "output_cost_per_token": 3.702e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.3-flash": { + "max_input_tokens": 1310720, + "max_output_tokens": 65536, + "input_cost_per_token": 1.72e-07, + "output_cost_per_token": 5.733e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gpt-5.6-luna": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.944e-07, + "cache_read_input_token_cost": 2.7e-08, + "output_cost_per_token": 1.767e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-luna-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.703e-07, + "cache_read_input_token_cost": 2.7e-08, + "output_cost_per_token": 1.621e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-sol": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-sol-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-terra": { + "max_input_tokens": 1050000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.62e-05, + "input_cost_per_token_above_272k_tokens": 5.4e-06, + "output_cost_per_token_above_272k_tokens": 2.43e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gpt-5.6-terra-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.62e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-6-astra": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-05, + "cache_read_input_token_cost": 1.35e-06, + "output_cost_per_token": 6.75e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gpt-6-luna": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.639e-07, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 8.195e-07, + "input_cost_per_token_above_272k_tokens": 2.7e-07, + "output_cost_per_token_above_272k_tokens": 1.0125e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-6.1-sol": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 1.35e-05, + "input_cost_per_token_above_272k_tokens": 5.4e-06, + "output_cost_per_token_above_272k_tokens": 2.025e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-oss-120b": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 4.73e-08, + "output_cost_per_token": 2.303e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/grok-4.20": { + "max_input_tokens": 2000000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.763e-06, + "output_cost_per_token": 3.528e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.20-multi-agent": { + "max_input_tokens": 2000000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.767e-06, + "output_cost_per_token": 3.534e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": false, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.3": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 1.92e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 3.838e-06, + "input_cost_per_token_above_200k_tokens": 3.375e-06, + "output_cost_per_token_above_200k_tokens": 6.75e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/grok-4.5": { + "max_input_tokens": 500000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.73e-06, + "output_cost_per_token": 8.189e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.6": { + "max_input_tokens": 500000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.758e-06, + "output_cost_per_token": 8.274e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.7": { + "max_input_tokens": 500000, + "max_output_tokens": 450000, + "input_cost_per_token": 2.16e-06, + "cache_read_input_token_cost": 5.4e-07, + "output_cost_per_token": 6.48e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-build": { + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "input_cost_per_token": 1.389e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 2.777e-06, + "input_cost_per_token_above_200k_tokens": 2.7e-06, + "output_cost_per_token_above_200k_tokens": 5.4e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/hy3": { + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "input_cost_per_token": 1.594e-07, + "output_cost_per_token": 6.576e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/hy4-preview": { + "max_input_tokens": 1048576, + "max_output_tokens": 64000, + "input_cost_per_token": 1.1259e-06, + "output_cost_per_token": 3.3764e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k2.6": { + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "input_cost_per_token": 5.909e-07, + "output_cost_per_token": 2.758e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k2.7-code": { + "max_input_tokens": 262144, + "max_output_tokens": 16384, + "input_cost_per_token": 1.009e-06, + "output_cost_per_token": 4.774e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k3": { + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "input_cost_per_token": 3.052e-06, + "cache_read_input_token_cost": 4.05e-07, + "output_cost_per_token": 1.5258e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k3.1-pro": { + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.35e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/llama-3.3-70b": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.35e-07, + "output_cost_per_token": 4.32e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/llama-4-maverick": { + "max_input_tokens": 1048576, + "max_output_tokens": 16384, + "input_cost_per_token": 3.344e-07, + "output_cost_per_token": 1.337e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/mimo-v2.5": { + "max_input_tokens": 1050000, + "max_output_tokens": 16384, + "input_cost_per_token": 1.859e-07, + "output_cost_per_token": 3.718e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/mimo-v2.6-flash": { + "max_input_tokens": 1050000, + "max_output_tokens": 131072, + "input_cost_per_token": 1.89e-07, + "output_cost_per_token": 3.78e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/mimo-v2.6-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 131072, + "input_cost_per_token": 5.805e-07, + "output_cost_per_token": 1.1745e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m2.5": { + "max_input_tokens": 196608, + "max_output_tokens": 32768, + "input_cost_per_token": 4.188e-07, + "output_cost_per_token": 1.675e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m2.7": { + "max_input_tokens": 204800, + "max_output_tokens": 32768, + "input_cost_per_token": 4.05e-07, + "output_cost_per_token": 1.62e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m3": { + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "input_cost_per_token": 3.956e-07, + "cache_read_input_token_cost": 7.56e-08, + "output_cost_per_token": 1.582e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m3.1-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 524288, + "input_cost_per_token": 2.025e-07, + "cache_read_input_token_cost": 2.025e-08, + "output_cost_per_token": 8.1e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/muse-glimmer-30b": { + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "input_cost_per_token": 4.554e-07, + "output_cost_per_token": 1.821e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/muse-spark-1.1": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.542e-06, + "output_cost_per_token": 5.243e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/muse-spark-1.2": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.773e-06, + "output_cost_per_token": 6.025e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/muse-spark-1.3": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.688e-06, + "output_cost_per_token": 5.738e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/muse-spark-open": { + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "input_cost_per_token": 1.688e-06, + "output_cost_per_token": 5.738e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/muse-wonder": { + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "input_cost_per_token": 5.4e-06, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/nemotron-3-ultra-550b": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 6.75e-07, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 3.375e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/nemotron-3.5-super-vl": { + "max_input_tokens": 1048576, + "max_output_tokens": 235929, + "input_cost_per_token": 8.1e-08, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 4.725e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3-32b": { + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "input_cost_per_token": 1.08e-07, + "output_cost_per_token": 3.78e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3-coder": { + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "input_cost_per_token": 3.546e-07, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 1.612e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.6-plus": { + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "input_cost_per_token": 4.388e-07, + "output_cost_per_token": 2.633e-06, + "input_cost_per_token_above_256k_tokens": 1.755e-06, + "output_cost_per_token_above_256k_tokens": 5.265e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.7-max": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 2.071e-06, + "cache_read_input_token_cost": 1.6875e-07, + "output_cost_per_token": 6.207e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.7-plus": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 4.325e-07, + "cache_read_input_token_cost": 8.64e-08, + "output_cost_per_token": 1.731e-06, + "input_cost_per_token_above_256k_tokens": 1.296e-06, + "output_cost_per_token_above_256k_tokens": 5.184e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.8-max": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 2.228e-06, + "cache_read_input_token_cost": 2.781e-07, + "output_cost_per_token": 6.684e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen3.7-flash": { + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "input_cost_per_token": 4.35e-08, + "cache_read_input_token_cost": 8.1e-09, + "output_cost_per_token": 1.886e-07, + "input_cost_per_token_above_32k_tokens": 1.35e-07, + "output_cost_per_token_above_32k_tokens": 5.4e-07, + "input_cost_per_token_above_256k_tokens": 2.7e-07, + "output_cost_per_token_above_256k_tokens": 1.08e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen3.8-27b": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 3.826e-07, + "output_cost_per_token": 2.732e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen3.8-flash": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 2.045e-07, + "output_cost_per_token": 6.41e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/sonar": { + "max_input_tokens": 127072, + "max_output_tokens": 4096, + "input_cost_per_token": 1.35e-06, + "output_cost_per_token": 1.35e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": false, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_pdf_input": false + }, + "kyma/sonar-pro": { + "max_input_tokens": 200000, + "max_output_tokens": 8000, + "input_cost_per_token": 4.05e-06, + "output_cost_per_token": 2.025e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": false, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_pdf_input": false + }, + "kyma/step-3.7-flash": { + "max_input_tokens": 256000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.229e-07, + "cache_read_input_token_cost": 5.4e-08, + "output_cost_per_token": 1.282e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/step-5-preview": { + "max_input_tokens": 1048576, + "max_output_tokens": 64000, + "input_cost_per_token": 1.35e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 3.645e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, "databricks/databricks-claude-haiku-5-5": { "cache_creation_input_token_cost": 1.2502e-07, "cache_creation_input_token_cost_above_100k_tokens": 6.2503e-07, @@ -78514,5 +80245,81 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true + }, + "bigmodel/glm-5.3": { + "input_cost_per_token": 1.1428571428571428e-06, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 2.857142857142857e-07, + "litellm_provider": "bigmodel", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "source": "https://docs.bigmodel.cn/cn/guide/start/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "thinking_always_on": true, + "metadata": { + "comment": "Official API list price checked 2026-10-27: input 8, output 28, cached input 2 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded." + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_native_streaming": true + }, + "bigmodel/glm-5.3-flash": { + "input_cost_per_token": 1.142857142857143e-07, + "output_cost_per_token": 4e-07, + "cache_read_input_token_cost": 3.2857142857142855e-08, + "litellm_provider": "bigmodel", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "source": "https://docs.bigmodel.cn/cn/guide/start/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "thinking_always_on": true, + "supports_vision": true, + "supports_video_input": true, + "metadata": { + "comment": "Official API list price checked 2026-10-27: input 0.8, output 2.8, cached input 0.23 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded." + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_native_streaming": true + }, + "bigmodel/glm-5.3-flashx": { + "input_cost_per_token": 2.857142857142857e-07, + "output_cost_per_token": 1e-06, + "cache_read_input_token_cost": 8.142857142857142e-08, + "litellm_provider": "bigmodel", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "source": "https://docs.bigmodel.cn/cn/guide/start/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "thinking_always_on": true, + "supports_vision": true, + "supports_video_input": true, + "metadata": { + "comment": "Official API list price checked 2026-10-27: input 2, output 7, cached input 0.57 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded." + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_native_streaming": true } } diff --git a/osv-scanner.toml b/osv-scanner.toml index 33e2f614edc..32a4f65fdf9 100644 --- a/osv-scanner.toml +++ b/osv-scanner.toml @@ -1,27 +1,102 @@ -[[IgnoredVulns]] -id = "GHSA-w8v5-vhqr-4h9v" -ignoreUntil = 2026-11-01 -reason = "diskcache has no fixed release published; remove this entry once one exists" - [[IgnoredVulns]] id = "GHSA-v4x8-3wr5-3wq8" ignoreUntil = 2026-11-20 -reason = "the only fixed filelock is 4.2.1, a new major that requires Python >= 3.11 while litellm still supports 3.10, and the advisory is in ReadWriteLock, which litellm never uses (only FileLock); bump filelock and remove this entry once 3.10 is dropped or a 3.x fix ships" +reason = "filelock ReadWriteLock advisory, which no package in the lock takes (litellm and huggingface_hub, filelock's only dependents, use just FileLock and SoftFileLock); fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped" [[IgnoredVulns]] -id = "GHSA-vfj7-8cjw-p6xm" -ignoreUntil = 2026-11-03 -reason = "braces has no fixed release published (3.0.3, 2024-05-21, is latest and last_affected); dev-only via knip > fast-glob > micromatch; remove this entry once one exists" +id = "GHSA-567q-5qvw-pf92" +ignoreUntil = 2026-11-20 +reason = "Windows-only local DoS (CVSS 2.5) in filelock soft locks; litellm never takes SoftFileLock, but huggingface_hub falls back to it in the Hugging Face cache when the filesystem has no flock, which litellm reaches only to download the Kimi K3 tokenizer for Bedrock token counting, and the attack needs another local user who can write that cache; fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped" [[IgnoredVulns]] -id = "GHSA-6q48-6c7w-9gw2" -ignoreUntil = 2026-10-23 -reason = "certifi 2026.10.19 (published 2026-10-20) is inside the 3-day uv exclude-newer window until 2026-10-23 02:24 UTC; bump certifi and remove this entry once it clears" +id = "GHSA-9vhv-jxww-xfm5" +ignoreUntil = 2026-11-20 +reason = "Windows-only local DoS (CVSS 3.3) in filelock ReadWriteLock, which no package in the lock takes (litellm and huggingface_hub, filelock's only dependents, use just FileLock and SoftFileLock); fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped" + +[[IgnoredVulns]] +id = "GHSA-6627-m3xw-93mp" +ignoreUntil = 2026-11-20 +reason = "filelock StrictSoftFileLock claim directories ignore the lock mode, which no package in the lock takes (litellm and huggingface_hub, filelock's only dependents, use just FileLock and SoftFileLock); fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped" + +[[IgnoredVulns]] +id = "GHSA-69hj-f68j-2879" +ignoreUntil = 2026-11-20 +reason = "filelock UnixFileLock truncates and chmods a file another user renames or hard-links in at the lock path; litellm's CLI takes FileLock only inside ~/.litellm, which it creates owner-only, and huggingface_hub takes it in the Hugging Face cache, which litellm reaches only to download the Kimi K3 tokenizer for Bedrock token counting; the attack needs another local user who can rename or link files in that cache; fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped" + +[[IgnoredVulns]] +id = "GHSA-8mx5-7m5m-r77g" +ignoreUntil = 2026-11-20 +reason = "filelock WindowsFileLock ignores mode; the proxy image is Linux, so only Windows installs run it; there litellm's CLI takes FileLock only inside the owner-only ~/.litellm, whose lock file holds no data, and huggingface_hub takes it in the Hugging Face cache, which litellm reaches only to download the Kimi K3 tokenizer for Bedrock token counting; fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped" + +[[IgnoredVulns]] +id = "GHSA-9c4x-gm8v-wgc4" +ignoreUntil = 2026-11-20 +reason = "filelock StrictSoftFileLock and SoftReadWriteLock adopt another user's protocol directory, which no package in the lock takes (litellm and huggingface_hub, filelock's only dependents, use just FileLock and SoftFileLock); fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped" + +[[IgnoredVulns]] +id = "GHSA-9j6q-wvj5-jgjg" +ignoreUntil = 2026-11-20 +reason = "filelock SoftFileLock symlink swap leaks whether a path exists; litellm never takes SoftFileLock, but huggingface_hub falls back to it in the Hugging Face cache when the filesystem has no flock, which litellm reaches only to download the Kimi K3 tokenizer for Bedrock token counting, and the attack needs another local user who can write that cache; fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped" + +[[IgnoredVulns]] +id = "GHSA-cwqr-hh33-gh44" +ignoreUntil = 2026-11-20 +reason = "filelock ReadWriteLock stalls on a crafted lock database, which no package in the lock takes (litellm and huggingface_hub, filelock's only dependents, use just FileLock and SoftFileLock); fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped" + +[[IgnoredVulns]] +id = "GHSA-fq56-3cx3-fjxp" +ignoreUntil = 2026-11-20 +reason = "filelock SoftFileLock stale-lock break removes a live lock on a reused inode; litellm never takes SoftFileLock, but huggingface_hub falls back to it in the Hugging Face cache when the filesystem has no flock, which litellm reaches only to download the Kimi K3 tokenizer for Bedrock token counting, and the attack needs another local user who can write that cache; fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped" + +[[IgnoredVulns]] +id = "GHSA-j8f7-rjxc-mr56" +ignoreUntil = 2026-11-20 +reason = "filelock ReadWriteLock follows a symlink into SQLite, which no package in the lock takes (litellm and huggingface_hub, filelock's only dependents, use just FileLock and SoftFileLock); fixed in 4.0.9, which litellm's own filelock<4.0 cap blocks on every Python until it is lifted (#48635, #48802); remove once that lands" + +[[IgnoredVulns]] +id = "GHSA-m462-7rxw-wvpg" +ignoreUntil = 2026-11-20 +reason = "filelock StrictSoftFileLock follows a symlink swapped in at .filelock, which no package in the lock takes (litellm and huggingface_hub, filelock's only dependents, use just FileLock and SoftFileLock); fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped" + +[[IgnoredVulns]] +id = "GHSA-rfpc-qcw5-3pc4" +ignoreUntil = 2026-11-20 +reason = "filelock SoftFileLock removes a live holder's marker when another user forges its PID; litellm never takes SoftFileLock, but huggingface_hub falls back to it in the Hugging Face cache when the filesystem has no flock, which litellm reaches only to download the Kimi K3 tokenizer for Bedrock token counting, and the attack needs another local user who can write that cache; fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped" + +[[IgnoredVulns]] +id = "GHSA-pcg2-g6cj-q92c" +ignoreUntil = 2026-11-27 +reason = "tornado 6.6 (published 2026-10-21) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 6.5.10 while 3.11+ locks 6.6; litellm starts no tornado server or client, since semantic-router and aurelio-sdk only declare it and tenacity and sentry-sdk import it only to wrap tornado code when it is present, so tornado's HTTP header parsing is never reached; remove this entry once 3.10 is dropped" + +[[IgnoredVulns]] +id = "GHSA-pw9p-v7r6-gx4p" +ignoreUntil = 2026-11-27 +reason = "tornado 6.6 (published 2026-10-21) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 6.5.10 while 3.11+ locks 6.6; litellm starts no tornado server or client, since semantic-router and aurelio-sdk only declare it and tenacity and sentry-sdk import it only to wrap tornado code when it is present, so tornado's RedirectHandler is never reached; remove this entry once 3.10 is dropped" + +[[IgnoredVulns]] +id = "GHSA-r4j5-9xpp-3985" +ignoreUntil = 2026-11-27 +reason = "tornado 6.6 (published 2026-10-21) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 6.5.10 while 3.11+ locks 6.6; litellm starts no tornado server or client, since semantic-router and aurelio-sdk only declare it and tenacity and sentry-sdk import it only to wrap tornado code when it is present, so tornado's chunked body reading is never reached; remove this entry once 3.10 is dropped" + +[[IgnoredVulns]] +id = "GHSA-w8v5-vhqr-4h9v" +ignoreUntil = 2026-11-27 +reason = "diskcache still has no fixed release published (checked 2026-10-28), and it is reached only through the optional caching extra; remove this entry once one exists" + +[[IgnoredVulns]] +id = "GHSA-45c3-73f7-pv4m" +ignoreUntil = 2026-11-27 +reason = "dnspython 2.9.0 (published 2026-10-09) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 2.8.0 while 3.11+ locks 2.9.0; the advisory is CPU use in dns.name.from_wire_parser and litellm only reaches dnspython through pydantic EmailStr, which never does DNS lookups; remove this entry once 3.10 is dropped or a 3.10-compatible fix ships" + +[[IgnoredVulns]] +id = "GHSA-xcx6-jx8p-x3h5" +ignoreUntil = 2026-11-24 +reason = "@azure/msal-node is fixed only in 7.1.0 (published 2026-10-13), but it is dev-only in vscode-extension via @vscode/vsce > @azure/identity, and the latest @azure/identity (4.14.0) still requires msal-node ^6.0.0, so the lock cannot move without forcing a major override; the advisory is in the auth_code flow, which extension packaging does not run; bump and remove this entry once @azure/identity accepts msal-node >= 7.1.0" [[PackageOverrides]] name = "pillow" version = "12.3.0" ecosystem = "PyPI" vulnerability.ignore = true -effectiveUntil = 2026-10-27 -reason = "pillow 13.0.0 requires Python >= 3.11, so the Python 3.10 resolution of the ci and dev groups stays on 12.3.0; 3.11+ is on 13.0.0" +effectiveUntil = 2026-11-27 +reason = "pillow 13.0.0 requires Python >= 3.11, so the Python 3.10 resolution of the ci and dev groups and of the mlflow extra stays on 12.3.0; 3.11+ is on 13.0.0; pillow has never shipped an older-major release after the next major (no 10.x after 11.0.0, no 11.x after 12.0.0), so 13.x is the only fix line; extended to 2026-11-27 with maintainer approval, same date as the dnspython 3.10 entry; remove once 3.10 is dropped" diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 0a6c2bcb2a9..4188866de08 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -3785,6 +3785,32 @@ "generateContent": true } }, + "streamlake": { + "display_name": "StreamLake (`streamlake`)", + "url": "https://docs.litellm.ai/docs/providers/openai_compatible", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, "tencent": { "display_name": "Tencent TokenHub (`tencent`)", "url": "https://docs.litellm.ai/docs/providers/tencent", @@ -5300,6 +5326,58 @@ "a2a": false, "generateContent": false } + }, + "kyma": { + "display_name": "Kyma API (`kyma`)", + "url": "https://docs.litellm.ai/docs/providers/kyma", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, + "bigmodel": { + "display_name": "BigModel (Zhipu AI) (`bigmodel`)", + "url": "https://docs.bigmodel.cn/cn/guide/develop/responses/introduction", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } } }, "endpoints": { diff --git a/pyproject.toml b/pyproject.toml index 00cdb1587aa..200bb463313 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -360,11 +360,13 @@ cache-keys = [ ] constraint-dependencies = [ "tornado>=6.5.8", + "tornado>=6.6; python_version >= '3.11'", "aiohttp>=3.14.2,<4.0", "packaging>=24.0", "soupsieve>=2.8.4", "httplib2>=0.32.0", "setuptools>=84.1.0", + "dnspython>=2.9.0; python_version >= '3.11'", ] override-dependencies = [ # a2a-sdk 1.x requires packaging>=24.0; lunary 1.4.x still caps at <24.0. diff --git a/tests/benchmarks/test_json_fragment_accumulator_benchmarks.py b/tests/benchmarks/test_json_fragment_accumulator_benchmarks.py new file mode 100644 index 00000000000..807dd8ff236 --- /dev/null +++ b/tests/benchmarks/test_json_fragment_accumulator_benchmarks.py @@ -0,0 +1,26 @@ +from collections.abc import Iterator +from typing import Final + +import pytest + +from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAccumulator + +CONCATENATED_VALUES: Final = 20_000 + + +def _popped_values(accumulator: JSONFragmentAccumulator) -> Iterator[object]: + while True: + found, value = accumulator.pop_next_value() + if not found: + return + yield value + + +@pytest.mark.benchmark +def test_draining_many_concatenated_values() -> None: + accumulator: Final = JSONFragmentAccumulator() + accumulator.append('{"a": 1}' * CONCATENATED_VALUES) + + drained: Final = sum(1 for _ in _popped_values(accumulator)) + + assert drained == CONCATENATED_VALUES diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index 9fb3633d53c..c7781819ae4 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -107,6 +107,7 @@ ignored_function_names = [ "_deployment_num_retries", # Tested through the deployment num_retries mid-stream budget test in test_router.py "_request_fallback_list", # Tested through every mid-stream retry test in test_router.py "_request_model_group", # Tested through test_anthropic_messages_retry_budget_precedence_direct_call + "_search_tool_support", # Tested through the test_get_search_tool_support_* tests "_mid_stream_retry_trigger", # Tested through the retry policy mid-stream budget test in test_router.py "_anthropic_messages_group_retry_policy", # Tested through the retry budget precedence test in test_router.py "_anthropic_messages_resolved_retry_policy", # Tested through the malformed retry policy tests in test_router.py @@ -175,6 +176,7 @@ ignored_function_names = [ "_responses_continuation", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py "_responses_continuation_ceilings", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py "_team_deployments_named", # Tested through the team lookups in the concurrent delete tests in test_router.py + "_served_openai_models", # Cached as served_openai_models, tested through the pre-call check token count tests in test_router.py "priority_header_update", # Tested through the priority header tests in test_common_request_processing.py "_drops_params", # Read by split_request_priority and priority_header_update, tested through both "_responses_event_announces_background_run", # Tested through the background Responses stream tests in test_router.py diff --git a/tests/code_coverage_tests/router_surface_allowlist.json b/tests/code_coverage_tests/router_surface_allowlist.json index 428fd9b4011..52e5bb43625 100644 --- a/tests/code_coverage_tests/router_surface_allowlist.json +++ b/tests/code_coverage_tests/router_surface_allowlist.json @@ -102,6 +102,7 @@ "get_routable_upstream_model", "get_router_deployment_budget_limiter", "get_routing_group", + "get_search_tool_support", "get_settings", "get_valid_args", "guardrail_list", diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml index c3c5330c440..0e450b08ebe 100644 --- a/tests/e2e/coverage_registry/llm_conversational.yaml +++ b/tests/e2e/coverage_registry/llm_conversational.yaml @@ -155,3 +155,4 @@ - {id: llm.responses.anthropic.basic.stream.works, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: basic, streaming: stream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic over /v1/responses streams Responses events"} - {id: llm.responses.anthropic.basic.nonstream.cost_logged, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic over /v1/responses: cost header and spend row agree"} - {id: llm.responses.anthropic.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic function_call_output round trip over /v1/responses"} +- {id: llm.responses.anthropic.tool_search.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: tool_search, streaming: nonstream, assertions: [works], source: "llm_translation/test_responses_codex_tool_search_e2e.py", rationale: "Codex client tool_search over /v1/responses finds and loads a deferred tool on Claude"} diff --git a/tests/e2e/llm_translation/test_responses_codex_tool_search_e2e.py b/tests/e2e/llm_translation/test_responses_codex_tool_search_e2e.py new file mode 100644 index 00000000000..79fd61e247a --- /dev/null +++ b/tests/e2e/llm_translation/test_responses_codex_tool_search_e2e.py @@ -0,0 +1,205 @@ +"""Live e2e: Codex's client tool_search over /v1/responses on Claude. + +Codex holds MCP tools back behind its client-executed `tool_search` tool when the model catalog says +`supports_search_tool`. Over the completion bridge the proxy offers that search to Claude as a plain function, +hands Claude's call back as a `tool_search_call` Codex runs itself, and loads the tools Codex returns in +`tool_search_output` so Claude can call them on the next turn +""" + +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_config import unique_marker +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta +from lifecycle import ResourceManager +from models import LiteLLMParamsBody +from openai.types.responses import ( + FunctionToolParam, + Response, + ResponseFunctionToolCall, + ResponseInputParam, + ResponseToolSearchCall, + ToolParam, + ToolSearchToolParam, +) +from proxy_client import ProxyClient +from pydantic import BaseModel, ConfigDict +from sdk_clients import NO_PROXY_CACHE, SdkClients + +pytestmark = pytest.mark.e2e + +ANTHROPIC_BACKEND: Final = "anthropic/claude-haiku-5-5" +DEFERRED_TOOL_COUNT: Final = 50 +CALENDAR_TOOL_NAME: Final = "mcp__codex_apps__calendar_create_event" + +CODEX_TOOL_SEARCH: Final[ToolSearchToolParam] = { + "type": "tool_search", + "execution": "client", + "description": ( + "# Tool discovery\n\nSearches over deferred tool metadata with BM25 and exposes matching tools for the next " + "model call. Some of the tools may not have been provided to you upfront, and you should use this tool " + "(`tool_search`) to search for the required tools." + ), + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string", "description": "Search query for deferred tools."}, + "limit": {"type": "number", "description": "Maximum number of tools to return. Defaults to 8."}, + }, + "required": ["query"], + "additionalProperties": False, + }, +} + +SHELL_TOOL: Final[FunctionToolParam] = { + "type": "function", + "name": "shell", + "description": "Runs a shell command.", + "parameters": {"type": "object", "properties": {"cmd": {"type": "string"}}, "required": ["cmd"]}, + "strict": False, +} + +CALENDAR_TOOL: Final[FunctionToolParam] = { + "type": "function", + "name": CALENDAR_TOOL_NAME, + "description": "Create a calendar event.", + "parameters": { + "type": "object", + "properties": {"title": {"type": "string"}}, + "required": ["title"], + "additionalProperties": False, + }, + "strict": False, + "defer_loading": True, +} + +PROMPT: Final = "Create a calendar event titled Standup. Find the calendar tool with tool_search first, then call it." + + +class _CalendarArguments(BaseModel): + model_config = ConfigDict(frozen=True) + + title: str + + +def _filler_tool(index: int) -> FunctionToolParam: + return { + "type": "function", + "name": f"mcp__codex_apps__records_{index:02d}", + "description": f"List, inspect, update and archive records of family {index} in the connected workspace.", + "parameters": { + "type": "object", + "properties": { + "title": {"type": "string", "description": "Title of the record to act on"}, + "notes": {"type": "string", "description": "Free-form notes and context for the operation"}, + "tags": {"type": "array", "items": {"type": "string"}}, + "priority": {"type": "integer", "minimum": 0, "maximum": 5}, + }, + "required": ["title"], + "additionalProperties": False, + }, + "strict": False, + "defer_loading": True, + } + + +DEFERRED_TOOLS: Final[tuple[FunctionToolParam, ...]] = ( + *(_filler_tool(index) for index in range(DEFERRED_TOOL_COUNT - 1)), + CALENDAR_TOOL, +) + + +def _eager(tool: FunctionToolParam) -> ToolParam: + return FunctionToolParam( + type="function", + name=tool["name"], + description=tool.get("description"), + parameters=tool.get("parameters"), + strict=False, + ) + + +def _input_tokens(response: Response) -> int: + assert response.usage is not None, f"response carried no usage: {response.id}" + return response.usage.input_tokens + + +def _search_call(response: Response) -> ResponseToolSearchCall: + calls: Final = tuple(item for item in response.output if isinstance(item, ResponseToolSearchCall)) + assert len(calls) == 1, f"expected one tool_search_call, got {response.output!r}" + return calls[0] + + +class TestResponsesCodexToolSearchOverAnthropic: + @pytest.mark.covers("llm.responses.anthropic.tool_search.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.ANTHROPIC,), + models=(ANTHROPIC_BACKEND,), + capabilities=(Capability.TOOL_SEARCH, Capability.FUNCTION_CALLING), + mode=Mode.NONSTREAM, + ) + ) + def test_claude_finds_a_deferred_tool_through_codex_tool_search_and_calls_it( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model: Final = f"e2e-codex-tool-search-{unique_marker()}" + model_id: Final = proxy.create_model( + model, LiteLLMParamsBody(model=ANTHROPIC_BACKEND, api_key="os.environ/ANTHROPIC_API_KEY") + ) + resources.defer(lambda: proxy.delete_model(model_id)) + client: Final = sdk.openai(resources.key()) + searchable_tools: Final[list[ToolParam]] = [SHELL_TOOL, CODEX_TOOL_SEARCH] + + eager: Final = client.responses.create( + model=model, + input=PROMPT, + tools=[SHELL_TOOL, *(_eager(tool) for tool in DEFERRED_TOOLS)], + extra_body=NO_PROXY_CACHE, + ) + first: Final = client.responses.create( + model=model, input=PROMPT, tools=searchable_tools, extra_body=NO_PROXY_CACHE + ) + + search: Final = _search_call(first) + assert search.execution == "client", f"tool_search ran on the wrong side: {search!r}" + assert search.call_id, f"tool_search_call had no call_id: {search!r}" + assert not any(isinstance(item, ResponseFunctionToolCall) for item in first.output), first.output + assert _input_tokens(first) * 4 < _input_tokens(eager), ( + f"deferred first turn used {_input_tokens(first)} input tokens against {_input_tokens(eager)} eager" + ) + + follow_up: Final[ResponseInputParam] = [ + {"role": "user", "content": PROMPT}, + { + "type": "tool_search_call", + "call_id": search.call_id, + "execution": "client", + "arguments": search.arguments, + }, + { + "type": "tool_search_output", + "call_id": search.call_id, + "status": "completed", + "execution": "client", + "tools": [CALENDAR_TOOL], + }, + ] + second: Final = client.responses.create( + model=model, input=follow_up, tools=searchable_tools, extra_body=NO_PROXY_CACHE + ) + + calendar_call: Final = next( + ( + item + for item in second.output + if isinstance(item, ResponseFunctionToolCall) and item.name == CALENDAR_TOOL_NAME + ), + None, + ) + assert calendar_call is not None, f"Claude did not call the loaded tool: {second.output!r}" + assert "standup" in _CalendarArguments.model_validate_json(calendar_call.arguments).title.lower() diff --git a/tests/integration/AGENTS.md b/tests/integration/AGENTS.md index a727b20596c..5bdc538b2d9 100644 --- a/tests/integration/AGENTS.md +++ b/tests/integration/AGENTS.md @@ -37,3 +37,14 @@ together with its dedicated job, and list a new dedicated GitHub workflow with t `DEDICATED_INTEGRATION_WORKFLOWS` in `.github/scripts/assert_ci_coverage.py`. The `ci-coverage` check fails a file here that a GitHub workflow runs unless it is in `GITHUB_FILES` and that workflow owns its directory (#46780) + +## Fixtures that boot a proxy + +Scope them to the module or class, or pin a session one to a single worker with an `xdist_group` mark +as `_llm_gateway` in `mcp/test_mcp_llm_endpoints.py` does, and request them as arguments, never through +`request.getfixturevalue`. The harness puts tests that request the same set of module or class fixtures +on one xdist worker, so the proxy they share boots there once and stops when that worker moves to another +file. A test that requests one more of those fixtures than its neighbors gets its own group and can boot +the proxy again on another worker, so keep that set the same across the tests that share a proxy. An +unpinned session fixture boots a copy on every worker that runs one of its tests and keeps them all until +the node ends, and those copies ran CircleCI integration-extensions nodes out of memory diff --git a/tests/integration/_support/client.py b/tests/integration/_support/client.py index b8ea11f3d2f..a759126fcd4 100644 --- a/tests/integration/_support/client.py +++ b/tests/integration/_support/client.py @@ -280,12 +280,14 @@ class Scenario: self.gateway.rows('SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = %s', (identity,)) == [] ) - def model(self, *, model_info: Mapping[str, JsonValue] | None = None, **parameters: JsonValue) -> str: - name: Final = f"integration-{uuid.uuid4().hex}" + def model( + self, *, name: str | None = None, model_info: Mapping[str, JsonValue] | None = None, **parameters: JsonValue + ) -> str: + model_name: Final = name if name is not None else f"integration-{uuid.uuid4().hex}" created: Final = self.gateway.post( "/model/new", { - "model_name": name, + "model_name": model_name, "litellm_params": { "model": "openai/gpt-4o-mini", "api_key": "integration-provider-key", @@ -298,8 +300,8 @@ class Scenario: identity: Final = string_value(object_value(created["model_info"])["id"]) self.cleanups.callback(self.delete_model, identity) if self.gateway.workers > 1: - self._await_every_worker(name, identity) - return name + self._await_every_worker(model_name, identity) + return model_name def _await_every_worker(self, name: str, identity: str) -> None: from litellm.constants import PROXY_CONFIG_RELOAD_INTERVAL_SECONDS diff --git a/tests/integration/_support/device_login.py b/tests/integration/_support/device_login.py index 9b990fd2167..55519678f40 100644 --- a/tests/integration/_support/device_login.py +++ b/tests/integration/_support/device_login.py @@ -73,6 +73,7 @@ COPILOT_USER_CODE: Final = "COPI-LEAK" COPILOT_ACCESS: Final = "device-login-github-access" COPILOT_REJECTED_ACCESS: Final = "device-login-github-access-rejected" COPILOT_FIRST_LOGIN_ACCESS: Final = "device-login-github-access-first-login" +COPILOT_UNSAVED_KEY_ACCESS: Final = "device-login-github-access-unsaved-key" COPILOT_STORED_KEY: Final = "tid=device-login;stored" COPILOT_MINTED_KEY: Final = "tid=device-login;minted" COPILOT_REJECTED_KEY: Final = "tid=device-login;rejected" @@ -132,7 +133,11 @@ def _oauth_token(request: Request, grant: threading.Event) -> Reply: def _copilot_api_key(request: Request, api_url: str) -> Reply: - if request.headers.get("authorization") not in (f"token {COPILOT_ACCESS}", f"token {COPILOT_FIRST_LOGIN_ACCESS}"): + if request.headers.get("authorization") not in ( + f"token {COPILOT_ACCESS}", + f"token {COPILOT_FIRST_LOGIN_ACCESS}", + f"token {COPILOT_UNSAVED_KEY_ACCESS}", + ): return _json(401, {"message": "Bad credentials"}) return _json( 200, @@ -380,4 +385,7 @@ def write_copilot_access(directory: Path, access_token: str) -> None: def clear_tokens(*directories: Path) -> None: for directory in directories: for path in directory.iterdir(): - path.unlink() + if path.is_dir(): + path.rmdir() + else: + path.unlink() diff --git a/tests/integration/_support/responses_replay.py b/tests/integration/_support/responses_replay.py new file mode 100644 index 00000000000..c4e73ffbf2b --- /dev/null +++ b/tests/integration/_support/responses_replay.py @@ -0,0 +1,214 @@ +import json +import uuid +from collections.abc import Callable, Sequence +from itertools import chain +from typing import Final + +from integration._support.client import list_value, object_value +from integration._support.wire import Reply, Request, Wire +from pydantic import JsonValue + +MODEL: Final = "gpt-6.1-sol" +PLAN: Final = "I am starting by choosing which city to check." +PICK: Final = "My pick is Tokyo, so I will check its weather." +PATCH_NOTE: Final = "I will create hello.txt containing hi." +ANSWER: Final = "It is raining in Tokyo." + +WEATHER_TOOL: Final[dict[str, JsonValue]] = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the weather for a city.", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + }, +} +PATCH_TOOL: Final[dict[str, JsonValue]] = { + "type": "custom", + "custom": {"name": "apply_patch", "description": "Apply a V4A patch to the workspace"}, +} + + +def _message(item_id: str, text: str, phase: str) -> dict[str, JsonValue]: + return { + "type": "message", + "id": item_id, + "status": "completed", + "role": "assistant", + "phase": phase, + "content": [{"type": "output_text", "text": text, "annotations": []}], + } + + +def _reasoning(item_id: str, summary: str) -> dict[str, JsonValue]: + return { + "type": "reasoning", + "id": item_id, + "summary": [{"type": "summary_text", "text": summary}], + "encrypted_content": f"enc_{item_id}", + } + + +_WEATHER_CALL: Final[dict[str, JsonValue]] = { + "type": "function_call", + "id": "fc_weather", + "call_id": "call_weather", + "name": "get_weather", + "arguments": '{"city":"Tokyo"}', + "status": "completed", +} +WEATHER_TURN: Final[tuple[dict[str, JsonValue], ...]] = ( + _message("msg_plan", PLAN, "commentary"), + _reasoning("rs_pick", "Tokyo is often rainy this time of year."), + _message("msg_pick", PICK, "commentary"), + _WEATHER_CALL, +) +TWO_THOUGHTS_TURN: Final[tuple[dict[str, JsonValue], ...]] = ( + _reasoning("rs_scan", "Paris and Tokyo are the two candidates."), + _reasoning("rs_pick", "Tokyo is often rainy this time of year."), + _message("msg_pick", PICK, "commentary"), + _WEATHER_CALL, +) +CALL_ONLY_TURN: Final[tuple[dict[str, JsonValue], ...]] = (_WEATHER_CALL,) +PATCH_TURN: Final[tuple[dict[str, JsonValue], ...]] = ( + _message("msg_patch", PATCH_NOTE, "commentary"), + { + "type": "custom_tool_call", + "id": "ctc_patch", + "call_id": "call_patch", + "name": "apply_patch", + "input": "*** Begin Patch\n*** Add File: hello.txt\n+hi\n*** End Patch", + "status": "completed", + }, +) +WEATHER_REPLAY: Final = ( + ("message", "assistant", "commentary", PLAN), + ("reasoning", "rs_pick"), + ("message", "assistant", "commentary", PICK), + ("function_call", "call_weather", "get_weather"), + ("function_call_output", "call_weather"), +) +TWO_THOUGHTS_REPLAY: Final = ( + ("reasoning", "rs_scan"), + ("reasoning", "rs_pick"), + ("message", "assistant", "commentary", PICK), + ("function_call", "call_weather", "get_weather"), + ("function_call_output", "call_weather"), +) + + +def _response(identity: str, output: Sequence[dict[str, JsonValue]]) -> dict[str, JsonValue]: + return { + "id": f"resp_{identity}", + "object": "response", + "created_at": 1789788253, + "status": "completed", + "model": MODEL, + "output": list(output), + "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15}, + } + + +def _event(payload: dict[str, JsonValue]) -> bytes: + return f"event: {payload['type']}\ndata: {json.dumps(payload)}\n\n".encode() + + +def _opened(item: dict[str, JsonValue]) -> dict[str, JsonValue]: + match item["type"]: + case "message": + return {**item, "status": "in_progress", "content": []} + case "function_call": + return {**item, "status": "in_progress", "arguments": ""} + case _: + return item + + +def _deltas(index: int, item: dict[str, JsonValue]) -> tuple[dict[str, JsonValue], ...]: + match item["type"]: + case "message": + text: Final = object_value(list_value(item["content"])[0])["text"] + return ( + { + "type": "response.output_text.delta", + "item_id": item["id"], + "output_index": index, + "content_index": 0, + "delta": text, + }, + ) + case "function_call": + return ( + { + "type": "response.function_call_arguments.delta", + "item_id": item["id"], + "output_index": index, + "delta": item["arguments"], + }, + ) + case _: + return () + + +def _item_events(index: int, item: dict[str, JsonValue]) -> tuple[bytes, ...]: + return ( + _event({"type": "response.output_item.added", "output_index": index, "item": _opened(item)}), + *(_event(delta) for delta in _deltas(index, item)), + _event({"type": "response.output_item.done", "output_index": index, "item": item}), + ) + + +def _stream(identity: str, output: Sequence[dict[str, JsonValue]]) -> tuple[bytes, ...]: + response: Final = _response(identity, output) + return ( + _event({"type": "response.created", "response": {**response, "status": "in_progress", "output": []}}), + *chain.from_iterable(_item_events(index, item) for index, item in enumerate(output)), + _event({"type": "response.completed", "response": response}), + ) + + +def _answered(body: dict[str, JsonValue]) -> bool: + return any( + object_value(item).get("type") in {"function_call_output", "custom_tool_call_output"} + for item in list_value(body["input"]) + ) + + +def responses_upstream(turn_one: Sequence[dict[str, JsonValue]]) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + if request.method == "GET": + return Reply(body=b'{"object":"list","data":[]}') + assert request.target == "/responses", request.target + body: Final = object_value(json.loads(request.body)) + identity: Final = uuid.uuid4().hex + output: Final = (_message("msg_answer", ANSWER, "final_answer"),) if _answered(body) else turn_one + if body.get("stream"): + return Reply(content_type="text/event-stream", chunks=_stream(identity, output)) + return Reply(body=json.dumps(_response(identity, output)).encode()) + + return respond + + +def _text(content: JsonValue) -> JsonValue: + if isinstance(content, str): + return content + return "".join(str(object_value(part).get("text")) for part in list_value(content)) + + +def _shape(item: JsonValue) -> tuple[JsonValue, ...]: + fields: Final = object_value(item) + match fields.get("type"): + case "message": + return ("message", fields.get("role"), fields.get("phase"), _text(fields.get("content"))) + case "reasoning": + return ("reasoning", fields.get("id")) + case "function_call" | "custom_tool_call" as kind: + return (kind, fields.get("call_id"), fields.get("name")) + case kind: + return (kind, fields.get("call_id")) + + +def posted_inputs(wire: Wire) -> tuple[tuple[tuple[JsonValue, ...], ...], ...]: + return tuple( + tuple(_shape(item) for item in list_value(object_value(json.loads(request.body))["input"])) + for request in wire.drain() + if request.method == "POST" + ) diff --git a/tests/integration/authorization/test_claude_code_gateway_oauth.py b/tests/integration/authorization/test_claude_code_gateway_oauth.py index a80eeacad46..4dedbdcd8e4 100644 --- a/tests/integration/authorization/test_claude_code_gateway_oauth.py +++ b/tests/integration/authorization/test_claude_code_gateway_oauth.py @@ -545,3 +545,94 @@ def test_device_authorization_polls_complete_sso_and_mints_a_model_restricted_be assert set(token_form) == {"code", "code_verifier", "grant_type", "redirect_uri"}, token_form assert userinfo_request.method == "GET" and urlsplit(userinfo_request.target).path == "/userinfo" assert user_id == _USER_ID + + +_TEAM_OPTION: Final = re.compile(r'