diff --git a/.github/scripts/smoke_test_native_wheel.py b/.github/scripts/smoke_test_native_wheel.py index 577bb32fcf0..af4054ea92e 100644 --- a/.github/scripts/smoke_test_native_wheel.py +++ b/.github/scripts/smoke_test_native_wheel.py @@ -7,6 +7,8 @@ import zipfile from pathlib import Path from typing import Final +COST_MAP: Final = Path(__file__).resolve().parents[2] / "model_prices_and_context_window.json" + CHILD_SCRIPT: Final = """ from importlib.util import module_from_spec, spec_from_file_location from pathlib import Path @@ -34,6 +36,9 @@ else: after = module.gil_stats() if not isinstance(after.get("releases"), int): raise AssertionError(f"native module unusable after panic: {after!r}") + +if module.bundled_model_cost_map() != Path(sys.argv[2]).read_bytes(): + raise AssertionError("the wheel embeds a different model_prices_and_context_window.json than this checkout") """ @@ -55,7 +60,9 @@ def main() -> int: native_path: Final = Path(temporary_directory) / Path(native_members[0].filename).name native_path.write_bytes(archive.read(native_members[0])) - result: Final = subprocess.run((sys.executable, "-c", CHILD_SCRIPT, str(native_path)), check=False) + result: Final = subprocess.run( + (sys.executable, "-c", CHILD_SCRIPT, str(native_path), str(COST_MAP)), check=False + ) if result.returncode != 0: sys.stderr.write(f"native wheel smoke test exited with status {result.returncode}\n") diff --git a/.github/workflows/cost-map-guard.yml b/.github/workflows/cost-map-guard.yml index 6519b65e71b..fb86237e504 100644 --- a/.github/workflows/cost-map-guard.yml +++ b/.github/workflows/cost-map-guard.yml @@ -46,3 +46,24 @@ jobs: HEAD_REF: ${{ github.event.pull_request.head.ref }} run: | uv run --no-project python ci_cd/cost_map_guard.py --base "$MERGE_BASE" --head "$HEAD_SHA" --head-ref "$HEAD_REF" + - name: Check whether the pull request changes the cost map + id: changed + env: + MERGE_BASE: ${{ steps.revisions.outputs.merge_base }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + if git diff --quiet "$MERGE_BASE" "$HEAD_SHA" -- model_prices_and_context_window.json; then + echo "cost_map=false" >> "$GITHUB_OUTPUT" + else + echo "cost_map=true" >> "$GITHUB_OUTPUT" + fi + - if: steps.changed.outputs.cost_map == 'true' + run: rustup toolchain install --no-self-update + - name: Load the cost map through model-catalog + if: steps.changed.outputs.cost_map == 'true' + env: + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + git show "${HEAD_SHA}:model_prices_and_context_window.json" > "$RUNNER_TEMP/model_prices_and_context_window.json" + cargo run --locked --quiet --manifest-path litellm-rust/Cargo.toml -p litellm-model-catalog --bin check-registry \ + -- "$RUNNER_TEMP/model_prices_and_context_window.json" diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index 89ca60ad9d1..bd7343ea222 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -376,10 +376,10 @@ jobs: exit 0 fi files="$(git diff --name-only "$base" HEAD)" - if grep -Eq '^(litellm/(proxy|types)/|ui/litellm-dashboard/(src/lib/http/schema\.d\.ts|scripts/gen-api-types\.mjs|package(-lock)?\.json)$|\.github/workflows/test-linting\.yml$)' <<< "$files"; then + if grep -Eq '^(litellm/|enterprise/|(pyproject\.toml|uv\.lock)$|ui/litellm-dashboard/(src/lib/http/schema\.d\.ts|scripts/gen-api-types\.mjs|package(-lock)?\.json)$|\.github/workflows/test-linting\.yml$)' <<< "$files"; then echo "relevant=true" >> "$GITHUB_OUTPUT" else - echo "No proxy, types or generator changes in this pull request, nothing to verify." + echo "No litellm, enterprise, dependency or generator changes in this pull request, nothing to verify." echo "relevant=false" >> "$GITHUB_OUTPUT" fi diff --git a/.github/workflows/test-terraform-provider.yml b/.github/workflows/test-terraform-provider.yml index ff9db13bd25..6aeb66ed37c 100644 --- a/.github/workflows/test-terraform-provider.yml +++ b/.github/workflows/test-terraform-provider.yml @@ -12,6 +12,7 @@ on: paths: - "terraform/provider/**" - "litellm/proxy/**" + - "enterprise/**" - ".github/workflows/test-terraform-provider.yml" permissions: diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 4768eeb633e..509271cbf54 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -236,6 +236,7 @@ jobs: tests/unit/llms/azure_ai tests/unit/llms/base_llm tests/unit/llms/baseten + tests/unit/llms/bigmodel tests/unit/llms/black_forest_labs tests/unit/llms/brave tests/unit/llms/bytez @@ -321,6 +322,7 @@ jobs: tests/unit/llms/openai_like tests/unit/llms/opencode tests/unit/llms/opensandbox + tests/unit/llms/opensearch tests/unit/llms/openrouter tests/unit/llms/ovhcloud tests/unit/llms/parallel_ai diff --git a/backend/Dockerfile b/backend/Dockerfile index 1c22f914100..aad59ba17c6 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -24,6 +24,7 @@ ENV UV_PYTHON_DOWNLOADS=0 COPY pyproject.toml README.md rust-toolchain.toml ./ COPY .cargo ./.cargo COPY litellm-rust ./litellm-rust +COPY model_prices_and_context_window.json ./ RUN --mount=type=cache,target=/root/.cache/uv \ mkdir -p litellm/rust_bridge /wheels && \ diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py index 07048cf2c2f..7baef40e86b 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py @@ -30,7 +30,7 @@ from collections.abc import Awaitable, Callable, Mapping from dataclasses import dataclass from datetime import datetime, timedelta, timezone from types import MappingProxyType -from typing import TYPE_CHECKING, Final, cast +from typing import TYPE_CHECKING, Final from litellm_enterprise.proxy.common_utils.managed_object_attribution import ( ManagedObjectCreatorAttribution, @@ -290,12 +290,14 @@ class CheckResponsesCost: sees provider env vars, so it fails for every deployment whose credentials live in the config; the row then never leaves ``queued``. """ - if not via_router: - return await litellm.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata) - router_response = await self.llm_router.aget_responses( - response_id=response_id, litellm_metadata=litellm_metadata + retrieved: Final[object] = ( + await self.llm_router.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata) + if via_router + else await litellm.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata) ) - return cast(ResponsesAPIResponse, router_response) + if not isinstance(retrieved, ResponsesAPIResponse): + raise TypeError(f"Retrieving {response_id} did not return a response object") + return retrieved async def _fetch_unbilled(self, target: _PollTarget) -> ResponsesAPIResponse: return await self._get_response( diff --git a/examples/bigmodel.md b/examples/bigmodel.md new file mode 100644 index 00000000000..88a69796ebe --- /dev/null +++ b/examples/bigmodel.md @@ -0,0 +1,63 @@ +# BigModel (Zhipu AI) + +Use `bigmodel/` for the domestic BigModel platform and `zai/` for international Z.AI. Each platform uses its own credentials and default URL + +```yaml +model_list: + - model_name: glm + litellm_params: + model: bigmodel/glm-5.3 + api_key: os.environ/BIGMODEL_API_KEY + api_base: https://open.bigmodel.cn +``` + +One model configuration supports all three native APIs. The provider selects the upstream endpoint from the requested API + +| Proxy endpoint | BigModel endpoint | +| --- | --- | +| `/v1/chat/completions` | `/api/paas/v4/chat/completions` | +| `/v1/responses` | `/api/v1/responses` | +| `/v1/messages` | `/api/anthropic/v1/messages` | + +`api_base` is optional and defaults to `https://open.bigmodel.cn`. Set `BIGMODEL_API_BASE` to override the default, or pass an explicit `api_base` to override the environment value. Existing API-specific base URLs and complete endpoint URLs are normalized, preserving a custom proxy's path prefix + +A GLM Coding Plan key uses `https://open.bigmodel.cn/api/coding/paas/v4` as its `api_base`, since Coding Plan and standard keys do not work on each other's endpoints. Chat Completions then calls the Coding Plan endpoint, and Responses and Messages keep the endpoints in the table above + +The Admin UI lists BigModel (Zhipu AI), displays its official logo, and provides a required password API Key field and an optional API Base field with the default URL. Authentication uses an explicit `api_key` or `BIGMODEL_API_KEY` + +```python +import litellm + +chat = litellm.completion( + model="bigmodel/glm-5.3", + messages=[{"role": "user", "content": "Hello"}], + thinking={"type": "enabled"}, + reasoning_effort="low", +) + +response = litellm.responses( + model="bigmodel/glm-5.3", + input="Hello", + reasoning={"effort": "low"}, + store=True, +) + +async def messages(): + return await litellm.anthropic.messages.acreate( + model="bigmodel/glm-5.3", + max_tokens=1024, + messages=[{"role": "user", "content": "Hello"}], + thinking={"type": "adaptive"}, + output_config={"effort": "low"}, + ) +``` + +Each API retains its native response format and streaming events. The native Messages SDK entry point is asynchronous, as is the proxy's `/v1/messages` route + +GLM-5.3, GLM-5.3-Flash, and GLM-5.3-FlashX use always-on thinking. GLM-5.3 supports `low`, `high`, and `max` reasoning effort. Messages preserves native `thinking`, `output_config`, and `reasoning_effort` parameters + +Set `store=True` when creating a Responses request if you need continuation, retrieval, or deletion. Use the response ID returned by LiteLLM. BigModel documents a seven-day retention period and does not support cancel, compact, native WebSocket, or file_search + +The model catalog and packaged backup include GLM-5.3, GLM-5.3-Flash, and GLM-5.3-FlashX. Prices are expressed in USD/token using a static 7.0 CNY/USD conversion, with the source and verification date in each entry's metadata. Use existing `litellm_params` cost fields for contract-specific deployment prices + +API references: [Chat Completions](https://docs.bigmodel.cn/cn/guide/develop/openai/introduction), [Responses](https://docs.bigmodel.cn/cn/guide/develop/responses/introduction), [Messages](https://docs.bigmodel.cn/cn/guide/develop/claude/introduction), and [pricing](https://docs.bigmodel.cn/cn/guide/start/pricing), checked on 2026-10-27 diff --git a/gateway/Dockerfile b/gateway/Dockerfile index 6dc8bab45ce..9d228627543 100644 --- a/gateway/Dockerfile +++ b/gateway/Dockerfile @@ -40,6 +40,7 @@ ENV UV_PYTHON_DOWNLOADS=0 COPY pyproject.toml README.md rust-toolchain.toml ./ COPY .cargo ./.cargo COPY litellm-rust ./litellm-rust +COPY model_prices_and_context_window.json ./ RUN --mount=type=cache,target=/root/.cache/uv \ mkdir -p litellm/rust_bridge /wheels && \ diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index f7d7e5f4dab..e959f77917b 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -4345,6 +4345,7 @@ dependencies = [ "litellm-inference-transcription", "litellm-llms", "litellm-llms-types", + "litellm-model-catalog", "litellm-router-types", "litellm-secrets", "litellm-secrets-aws", diff --git a/litellm-rust/crates/core-utils/src/exception_mapping_utils/openai.rs b/litellm-rust/crates/core-utils/src/exception_mapping_utils/openai.rs index d45078c415e..1273fd4d7f1 100644 --- a/litellm-rust/crates/core-utils/src/exception_mapping_utils/openai.rs +++ b/litellm-rust/crates/core-utils/src/exception_mapping_utils/openai.rs @@ -53,6 +53,7 @@ pub(super) const RULES: &[Rule] = &[ |mapping| { mapping.error_str.contains("invalid_request_error") && !mapping.error_str.contains("Incorrect API key provided") + && !matches!(mapping.status, Some(401 | 404)) }, PublicError::BadRequest, ), @@ -132,6 +133,23 @@ mod tests { assert_eq!(classified(Some(400), text), Some(expected)); } + #[rstest::rstest] + #[case::unauthorized(Some(401), None)] + #[case::not_found(Some(404), None)] + #[case::forbidden(Some(403), Some(PublicError::BadRequest))] + #[case::rate_limited(Some(429), Some(PublicError::BadRequest))] + #[case::server_error(Some(500), Some(PublicError::BadRequest))] + #[case::no_status(None, Some(PublicError::BadRequest))] + fn invalid_request_errors_leave_401_and_404_to_the_status_mapping( + #[case] status: Option, + #[case] expected: Option, + ) { + assert_eq!( + classified(status, "invalid_request_error bad field"), + expected + ); + } + #[rstest::rstest] #[case::rate_limit_before_context_window( "rate limit and This model's maximum context length is 10", diff --git a/litellm-rust/crates/model-catalog/AGENTS.md b/litellm-rust/crates/model-catalog/AGENTS.md index fc4b28d4ef0..ff24fb81143 100644 --- a/litellm-rust/crates/model-catalog/AGENTS.md +++ b/litellm-rust/crates/model-catalog/AGENTS.md @@ -25,3 +25,5 @@ A new `N`, image size or resolution in an existing key shape parses with no Rust ## Tests `BUNDLED_REGISTRY` embeds the root `model_prices_and_context_window.json`, so the tests over it fail a JSON change with a bad value or a key no type reads + +`Catalog::problems` is the one definition of a mergeable cost map. The bundled test asserts it is empty, and the cost map guard workflow runs it on a pull request's JSON through `cargo run --bin check-registry ` diff --git a/litellm-rust/crates/model-catalog/Cargo.toml b/litellm-rust/crates/model-catalog/Cargo.toml index 45d16515dcd..63b9ad14768 100644 --- a/litellm-rust/crates/model-catalog/Cargo.toml +++ b/litellm-rust/crates/model-catalog/Cargo.toml @@ -10,6 +10,7 @@ schema = ["dep:schemars", "litellm-cost/schema"] [dependencies] litellm-cost = { workspace = true, features = ["serde"] } +litellm-error.workspace = true indexmap = { version = "2.14.0", features = ["serde"] } jiff = { version = "0.2.37", default-features = false, features = ["std", "tzdb-bundle-always"] } @@ -24,5 +25,4 @@ time.workspace = true [dev-dependencies] jsonschema = { version = "0.55.1", default-features = false } -litellm-error.workspace = true rstest.workspace = true diff --git a/litellm-rust/crates/model-catalog/src/bin/check-registry.rs b/litellm-rust/crates/model-catalog/src/bin/check-registry.rs new file mode 100644 index 00000000000..6fdc7621a0a --- /dev/null +++ b/litellm-rust/crates/model-catalog/src/bin/check-registry.rs @@ -0,0 +1,26 @@ +use std::{env, fs, path::PathBuf, process::ExitCode}; + +use litellm_model_catalog::Catalog; + +fn main() -> ExitCode { + let Some(path) = env::args_os().nth(1).map(PathBuf::from) else { + eprintln!("usage: check-registry "); + return ExitCode::from(2); + }; + let body = match fs::read(&path) { + Ok(body) => body, + Err(error) => { + eprintln!("{}: {error}", path.display()); + return ExitCode::FAILURE; + } + }; + let problems = Catalog::problems(&body); + for problem in &problems { + eprintln!("{}", litellm_error::Report::new(problem)); + } + if problems.is_empty() { + ExitCode::SUCCESS + } else { + ExitCode::FAILURE + } +} diff --git a/litellm-rust/crates/model-catalog/src/catalog.rs b/litellm-rust/crates/model-catalog/src/catalog.rs index ac5f4ffcf25..98af785b17e 100644 --- a/litellm-rust/crates/model-catalog/src/catalog.rs +++ b/litellm-rust/crates/model-catalog/src/catalog.rs @@ -5,7 +5,7 @@ use indexmap::IndexMap; use serde::Deserialize; use serde_json::Value; -use crate::error::{Error, Loaded, Rejects, Repr}; +use crate::error::{Error, Loaded, Rejects, Repr, pointer_child, pointer_for_key}; use crate::fallback::{FallbackGeneralizations, FallbackRule}; use crate::model::Model; use crate::store::{Rows, Store}; @@ -211,6 +211,36 @@ impl Catalog { }) } + pub fn problems(body: &[u8]) -> Vec { + let Loaded { + value: catalog, + errors, + } = match Self::parse(body) { + Ok(loaded) => loaded, + Err(fatal) => return vec![fatal], + }; + let unread = catalog.rows().entries.iter().flat_map(|(model, entry)| { + EntryWire::read(entry.fields()) + .unknown + .into_iter() + .map(move |(key, _)| { + Error::new( + pointer_child(&pointer_for_key(model), &key), + Repr::UnreadField, + ) + }) + }); + let no_fallback_rules = catalog + .fallback_rules() + .is_none_or(<[_]>::is_empty) + .then(|| Error::from_key(FALLBACK_GENERALIZATIONS, Repr::NoFallbackRules)); + errors + .into_iter() + .chain(unread) + .chain(no_fallback_rules) + .collect() + } + pub fn row_errors(model: &str, fields: &Fields) -> Vec { let (_, _, errors) = read_row(fields); errors @@ -404,3 +434,85 @@ fn unknown_timezone(wire: &EntryWire) -> Option { ) }) } + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + const RULES: &str = r#""fallback_generalizations":{"rules":[{"name":"family","pattern":"^new-","model_info":{"mode":"chat"}}]}"#; + + #[rstest] + #[case::clean(format!(r#"{{{RULES},"m":{{"litellm_provider":"test"}}}}"#), vec![])] + #[case::not_json("{".to_owned(), vec![("", "json")])] + #[case::row_error(format!(r#"{{{RULES},"m":{{}}}}"#), vec![("/m/litellm_provider", "missing_provider")])] + #[case::unread_key( + format!(r#"{{{RULES},"m":{{"litellm_provider":"test","typo_cost":1}}}}"#), + vec![("/m/typo_cost", "unread_field")] + )] + #[case::empty_model_key( + format!(r#"{{{RULES},"":{{"litellm_provider":"test","future_field":true}}}}"#), + vec![("//future_field", "unread_field")] + )] + #[case::unread_key_on_invalid_row( + format!(r#"{{{RULES},"m":{{"typo_cost":1}}}}"#), + vec![("/m/litellm_provider", "missing_provider"), ("/m/typo_cost", "unread_field")] + )] + #[case::escaped_unread_key_on_invalid_row( + format!(r#"{{{RULES},"open/router~v1":{{"typo/cost~":1}}}}"#), + vec![("/open~1router~0v1/litellm_provider", "missing_provider"), ("/open~1router~0v1/typo~1cost~0", "unread_field")] + )] + #[case::no_rules( + r#"{"m":{"litellm_provider":"test"}}"#.to_owned(), + vec![("/fallback_generalizations", "no_fallback_rules")] + )] + #[case::empty_rules( + r#"{"fallback_generalizations":{"rules":[]},"m":{"litellm_provider":"test"}}"#.to_owned(), + vec![("/fallback_generalizations", "no_fallback_rules")] + )] + fn problems_list_what_a_cost_map_change_must_fix( + #[case] body: String, + #[case] expected: Vec<(&str, &str)>, + ) { + let problems: Vec<_> = Catalog::problems(body.as_bytes()) + .iter() + .map(|problem| (problem.path().to_owned(), <&str>::from(problem.kind()))) + .collect(); + let expected: Vec<_> = expected + .into_iter() + .map(|(path, kind)| (path.to_owned(), kind)) + .collect(); + assert_eq!(problems, expected); + } + + #[rstest] + fn runtime_parsing_preserves_custom_fields_and_recovers_valid_rows() { + let body = br#"{ + "good":{"litellm_provider":"test","custom_metadata":true}, + "bad":{"litellm_provider":"test","input_cost_per_token":"wrong"} + }"#; + let loaded = Catalog::parse(body).unwrap(); + assert_eq!(loaded.errors.len(), 1); + assert_eq!(loaded.errors[0].path(), "/bad/input_cost_per_token"); + let (catalog, rejected) = loaded.value.without_rejected(&loaded.errors); + assert_eq!(rejected.len(), 1); + assert!(catalog.lookup("bad").is_none()); + let good = catalog.lookup("good").unwrap(); + assert!(good.entry.model().is_some()); + assert_eq!( + good.entry.field("custom_metadata"), + Some(&Value::Bool(true)) + ); + assert!( + Catalog::problems(body) + .iter() + .any(|problem| problem.kind() == crate::ErrorKind::UnreadField) + ); + assert!( + Catalog::problems(body) + .iter() + .any(|problem| problem.kind() == crate::ErrorKind::NoFallbackRules) + ); + } +} diff --git a/litellm-rust/crates/model-catalog/src/error.rs b/litellm-rust/crates/model-catalog/src/error.rs index 425ae47687f..89e27c05f46 100644 --- a/litellm-rust/crates/model-catalog/src/error.rs +++ b/litellm-rust/crates/model-catalog/src/error.rs @@ -109,6 +109,12 @@ pub(crate) enum Repr { #[source] cause: Option, }, + #[error("not a field litellm reads")] + #[strum_discriminants(strum(serialize = "unread_field"))] + UnreadField, + #[error("has no rules")] + #[strum_discriminants(strum(serialize = "no_fallback_rules"))] + NoFallbackRules, } /// What a problem takes out of the served map @@ -148,7 +154,7 @@ impl Error { | Repr::NoOffPeakHours | Repr::EmptyList | Repr::InvalidDate { .. } => Rejects::Row, - Repr::MissingProvider => Rejects::Nothing, + Repr::MissingProvider | Repr::UnreadField | Repr::NoFallbackRules => Rejects::Nothing, } } } @@ -334,6 +340,8 @@ mod tests { #[case::no_off_peak_hours(Repr::NoOffPeakHours, Rejects::Row)] #[case::empty_list(Repr::EmptyList, Rejects::Row)] #[case::invalid_date(Repr::InvalidDate { cause: None }, Rejects::Row)] + #[case::unread_field(Repr::UnreadField, Rejects::Nothing)] + #[case::no_fallback_rules(Repr::NoFallbackRules, Rejects::Nothing)] fn policy_classifies_each_error(#[case] error: Repr, #[case] expected: Rejects) { assert_eq!(Error::from(error).rejects(), expected); } diff --git a/litellm-rust/crates/model-catalog/tests/catalog.rs b/litellm-rust/crates/model-catalog/tests/catalog.rs index 30c070cfb52..31f907689d3 100644 --- a/litellm-rust/crates/model-catalog/tests/catalog.rs +++ b/litellm-rust/crates/model-catalog/tests/catalog.rs @@ -366,18 +366,14 @@ fn written_is_the_document_in_order_including_sections_and_bad_entries() { } #[rstest] -fn the_bundled_registry_parses_with_no_problems_and_every_key_understood() { - let loaded = Catalog::parse(BUNDLED_REGISTRY).unwrap(); - let catalog = loaded.value; - assert!(loaded.errors.is_empty(), "{:#?}", loaded.errors); - let unrecognized: Vec<_> = catalog.unrecognized().collect(); - assert!(unrecognized.is_empty(), "{unrecognized:#?}"); - assert!( - catalog - .fallback_rules() - .is_some_and(|rules| !rules.is_empty()) - ); +fn the_bundled_registry_has_no_problems() { + let problems: Vec<_> = Catalog::problems(BUNDLED_REGISTRY) + .iter() + .map(report) + .collect(); + assert!(problems.is_empty(), "{problems:#?}"); const REFERENCE_MODEL_COUNT: usize = 4303; + let catalog = Catalog::parse(BUNDLED_REGISTRY).unwrap().value; assert!(catalog.model_count() as f64 >= REFERENCE_MODEL_COUNT as f64 * 0.9); } diff --git a/litellm-rust/crates/python-bridge/AGENTS.md b/litellm-rust/crates/python-bridge/AGENTS.md index d90fe073b20..26858b2afe2 100644 --- a/litellm-rust/crates/python-bridge/AGENTS.md +++ b/litellm-rust/crates/python-bridge/AGENTS.md @@ -106,6 +106,18 @@ GIL handling to `litellm-host-python`. ## Data Handling +- Reuse large immutable bundled data and parsed catalogs across calls, including in + replacement APIs. Share Rust storage and cache repeated Python projections instead + of reparsing, serializing or copying the full payload on every access +- Cache owned Python objects with `PyOnceLock>`, then bind and clone references + while attached. Do not use blocking `OnceLock::get_or_init` while attached + ([PyO3 initialization](https://pyo3.rs/v0.29.2/faq.html), + [reference ownership](https://pyo3.rs/v0.29.2/types.html)) +- Never reuse Python objects across interpreters or interpreter reinitialization. + Once-only caches are for immutable data. Reloadable/custom data needs versioned + entries or invalidation, and caller-mutable Python containers must not be shared +- Test unchanged contents and allocation reuse, not just equal values. Keep this + coverage when replacing a bundled-data accessor with a catalog API - OCR payloads can contain personal data and large base64 images. Do not log payloads or provider responses. - Avoid copying large payloads more than needed. The current JSON round-trip is diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index 9d69cd8c14b..a4e1c57ab1e 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -49,6 +49,7 @@ litellm-secrets-types.workspace = true litellm-llms-types.workspace = true litellm-router-types.workspace = true litellm-host-python.workspace = true +litellm-model-catalog.workspace = true litellm-token-counter = { path = "../token-counter", default-features = false } pyo3.workspace = true prost.workspace = true diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index 12827b76ff3..683e90d02a2 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -8,6 +8,7 @@ mod execution; mod http; mod logger; mod marshal; +mod model_registry; mod preflight; mod python_settings; mod routes; @@ -30,6 +31,8 @@ mod _native { #[pymodule_export] use crate::logger::NativeDiagnosticProcessor; #[pymodule_export] + use crate::model_registry::bundled_model_cost_map; + #[pymodule_export] use crate::routes::audio_transcription::{atranscription, transcription}; #[pymodule_export] use crate::routes::chat_completions::{acompletion, completion}; @@ -110,6 +113,7 @@ mod tests { "NativeClickHouseSpendConfig", "NativeClickHouseSpendStorage", "trace_encode_error", + "bundled_model_cost_map", "TokenCounter", "Tokenizer", "gil_stats", diff --git a/litellm-rust/crates/python-bridge/src/model_registry.rs b/litellm-rust/crates/python-bridge/src/model_registry.rs new file mode 100644 index 00000000000..b7b7cd1fbb3 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/model_registry.rs @@ -0,0 +1,30 @@ +use pyo3::prelude::*; +use pyo3::sync::PyOnceLock; +use pyo3::types::PyBytes; + +#[pyfunction] +pub fn bundled_model_cost_map(py: Python<'_>) -> Bound<'_, PyBytes> { + static BYTES: PyOnceLock> = PyOnceLock::new(); + BYTES + .get_or_init(py, || { + PyBytes::new(py, litellm_model_catalog::BUNDLED_REGISTRY).unbind() + }) + .bind(py) + .clone() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[rstest::rstest] + fn repeated_calls_reuse_the_bundled_registry_bytes() { + Python::initialize(); + let first = Python::attach(|py| bundled_model_cost_map(py).unbind()); + Python::attach(|py| { + let second = bundled_model_cost_map(py); + assert!(first.bind(py).is(&second)); + assert_eq!(second.as_bytes(), litellm_model_catalog::BUNDLED_REGISTRY); + }); + } +} diff --git a/litellm/__init__.py b/litellm/__init__.py index b7d46c197f2..a1bda4aa9f7 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -39,6 +39,7 @@ from litellm._logging import ( turn_on_json, log_level, ) +from litellm._logging import suppress_startup_output as suppress_startup_output import re from litellm.constants import ( DEFAULT_BATCH_SIZE, @@ -619,6 +620,8 @@ vertex_openai_models: Set = set() vertex_minimax_models: Set = set() vertex_moonshot_models: Set = set() vertex_zai_models: Set = set() +vertex_qwen_models: Final[Set[str]] = set() +vertex_ai_models: Final[Set[str]] = set() ai21_models: Set = set() ai21_chat_models: Set = set() nlp_cloud_models: Set = set() @@ -635,6 +638,7 @@ watsonx_models: Set = set() gemini_models: Set = set() xai_models: Set = set() zai_models: Set = set() +bigmodel_models: Final[set[str]] = set() # mutable-ok: provider registry is populated and extended at runtime deepseek_models: Set = set() tencent_models: Set = set() runwayml_models: Set = set() @@ -681,6 +685,7 @@ moonshot_models: Set = set() publicai_models: Set = set() darkbloom_models: Set = set() openinfer_models: Final[Set[str]] = set() +kyma_models: Final[set[str]] = set() # mutable-ok: populated when the shared model catalog loads streamlake_models: Final[Set[str]] = set() zerogpu_models: Final[Set[str]] = set() v0_models: Set = set() @@ -833,6 +838,10 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje elif value.get("litellm_provider") == "vertex_ai-zai_models": key = key.replace("vertex_ai/", "") vertex_zai_models.add(key) + elif value.get("litellm_provider") == "vertex_ai-qwen_models": + vertex_qwen_models.add(key.removeprefix("vertex_ai/")) + elif value.get("litellm_provider") == "vertex_ai": + vertex_ai_models.add(key.removeprefix("vertex_ai/")) elif value.get("litellm_provider") == "ai21": if value.get("mode") == "chat": ai21_chat_models.add(key) @@ -877,6 +886,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje xai_models.add(key) elif value.get("litellm_provider") == "zai": zai_models.add(key) + elif value.get("litellm_provider") == "bigmodel": + bigmodel_models.add(key) elif value.get("litellm_provider") == "fal_ai": fal_ai_models.add(key) elif value.get("litellm_provider") == "deepseek": @@ -971,6 +982,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje darkbloom_models.add(key) elif value.get("litellm_provider") == "openinfer": openinfer_models.add(key) + elif value.get("litellm_provider") == "kyma": + kyma_models.add(key) elif value.get("litellm_provider") == "streamlake": streamlake_models.add(key) elif value.get("litellm_provider") == "zerogpu": @@ -1107,6 +1120,7 @@ model_list = list( | text_completion_inception_models | xai_models | zai_models + | bigmodel_models | fal_ai_models | deepseek_models | modelscope_models @@ -1148,6 +1162,7 @@ model_list = list( | publicai_models | darkbloom_models | openinfer_models + | kyma_models | streamlake_models | zerogpu_models | v0_models @@ -1204,7 +1219,16 @@ def _build_models_by_provider() -> dict: | vertex_deepseek_models | vertex_minimax_models | vertex_moonshot_models - | vertex_zai_models, + | vertex_zai_models + | vertex_qwen_models + | vertex_mistral_models + | vertex_llama3_models + | vertex_ai_ai21_models + | vertex_openai_models + | vertex_ai_image_models + | vertex_ai_video_models + | vertex_embedding_models + | vertex_ai_models, "ai21": ai21_models | ai21_chat_models, "bedrock": bedrock_models | bedrock_converse_models, "petals": petals_models, @@ -1221,6 +1245,7 @@ def _build_models_by_provider() -> dict: "text-completion-inception": text_completion_inception_models, "xai": xai_models, "zai": zai_models, + "bigmodel": bigmodel_models, "fal_ai": fal_ai_models, "deepseek": deepseek_models, "tencent": tencent_models, @@ -1269,6 +1294,7 @@ def _build_models_by_provider() -> dict: "publicai": publicai_models, "darkbloom": darkbloom_models, "openinfer": openinfer_models, + "kyma": kyma_models, "streamlake": streamlake_models, "zerogpu": zerogpu_models, "v0": v0_models, diff --git a/litellm/_internal_context.py b/litellm/_internal_context.py index 244efc8047b..c2243b90fbf 100644 --- a/litellm/_internal_context.py +++ b/litellm/_internal_context.py @@ -36,6 +36,8 @@ REDIS_FAMILIES_METADATA_KEY: Final = "families" _service_caller: Final[ContextVar[str | None]] = ContextVar("service_caller", default=None) +_mirrored_call: Final[ContextVar[bool]] = ContextVar("mirrored_call", default=False) + @contextmanager def internal_sub_call() -> Generator[None]: @@ -123,6 +125,30 @@ def current_service_caller() -> str | None: return _service_caller.get() +@contextmanager +def mirrored_call() -> Generator[None]: + """A silent_model mirror of a request: every litellm call inside, its retries and fallbacks included.""" + token: Final = _mirrored_call.set(True) + try: + yield + finally: + _restore(_mirrored_call, token) + + +def in_mirrored_call() -> bool: + return _mirrored_call.get() + + +@contextmanager +def outside_mirrored_call() -> Generator[None]: + """Logging a mirrored call: the calls its callbacks start are not mirrors themselves.""" + token: Final = _mirrored_call.set(False) + try: + yield + finally: + _restore(_mirrored_call, token) + + @contextmanager def pinned_billing_time(moment: datetime) -> Generator[None]: """Price every rate lookup inside this block at ``moment`` rather than at each one's own clock read.""" diff --git a/litellm/_logging.py b/litellm/_logging.py index 8974f85f517..5279cb233b4 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -10,10 +10,12 @@ import sys from collections.abc import Iterator, Mapping from datetime import datetime from logging import Formatter +from types import FrameType from typing import Final, TextIO from urllib.parse import unquote from pydantic import TypeAdapter +from typing_extensions import ReadOnly, TypedDict import litellm from litellm.constants import ( @@ -598,6 +600,7 @@ _LOG_FORMAT_PREFIX: Final = "%(asctime)s - %(name)s:%(levelname)s" _LOG_FORMAT_SUFFIX: Final = ": %(filename)s:%(lineno)s - %(message)s" _PLAIN_LOG_FORMAT: Final = _LOG_FORMAT_PREFIX + _LOG_FORMAT_SUFFIX _COLOR_LOG_FORMAT: Final = f"\033[92m{_LOG_FORMAT_PREFIX}\033[0m{_LOG_FORMAT_SUFFIX}" +_ANSI_ESCAPE: Final = re.compile(r"\x1b\[[0-9;]*m") def _stream_is_tty(stream: TextIO | None) -> bool: @@ -612,15 +615,28 @@ def _stream_is_tty(stream: TextIO | None) -> bool: return False -def _plain_log_format(stdout: TextIO | None, stderr: TextIO | None) -> str: - """The plain-text log format, colorized only when both streams are an interactive terminal. +def _color_enabled(stream: TextIO | None) -> bool: + """True when ANSI colors written to the stream reach an interactive terminal. Honors the NO_COLOR convention from no-color.org: color is disabled when NO_COLOR is present with a non-empty value. """ - if os.environ.get("NO_COLOR"): - return _PLAIN_LOG_FORMAT - return _COLOR_LOG_FORMAT if _stream_is_tty(stdout) and _stream_is_tty(stderr) else _PLAIN_LOG_FORMAT + return not os.environ.get("NO_COLOR") and _stream_is_tty(stream) + + +def _terminal_colors(stdout: TextIO | None, stderr: TextIO | None) -> bool: + """True when both streams are an interactive color terminal, the one rule for log lines and startup prints.""" + return _color_enabled(stdout) and _color_enabled(stderr) + + +def terminal_text(message: str) -> str: + """The message with its ANSI colors kept only on a color terminal, by the rule the plain log format uses.""" + return message if _terminal_colors(sys.stdout, sys.stderr) else _ANSI_ESCAPE.sub("", message) + + +def _plain_log_format(stdout: TextIO | None, stderr: TextIO | None) -> str: + """The plain-text log format, colorized only when both streams are an interactive terminal.""" + return _COLOR_LOG_FORMAT if _terminal_colors(stdout, stderr) else _PLAIN_LOG_FORMAT class LevelRoutingStreamHandler(logging.StreamHandler): @@ -642,8 +658,8 @@ class LevelRoutingStreamHandler(logging.StreamHandler): super().emit(record) -def _parse_json_logs_env(value: str | None) -> bool: - """Strict opt-in parse for the JSON_LOGS env var: only "true" (any case) enables JSON logs. +def _parse_env_flag(value: str | None) -> bool: + """Strict opt-in parse for a boolean env var such as JSON_LOGS: only "true" (any case) enables it. Matches the reader in litellm-proxy-extras/_logging.py. The previous bool(os.getenv(...)) treated any non-empty value, including "false" and "0", @@ -656,7 +672,18 @@ def resolve_log_level(log_level: str) -> int: return getattr(logging, log_level.upper()) -json_logs: Final = _parse_json_logs_env(os.getenv("JSON_LOGS")) +json_logs: Final = _parse_env_flag(os.getenv("JSON_LOGS")) +suppress_startup_output: Final = _parse_env_flag(os.getenv("LITELLM_SUPPRESS_STARTUP_OUTPUT")) + + +class StartupOutputSettings(TypedDict, total=False): + json_logs: ReadOnly[object] + suppress_startup_output: ReadOnly[object] + + +STARTUP_OUTPUT_SETTINGS: Final = TypeAdapter(StartupOutputSettings) + + # Create a handler for the logger (you may need to adapt this based on your needs) log_level: Final = os.getenv("LITELLM_LOG", "DEBUG") numeric_level: Final[int] = resolve_log_level(log_level) @@ -743,7 +770,7 @@ class JsonFormatter(Formatter): return dt.isoformat() def format(self, record): - message_str: Final = record.getMessage() + message_str: Final = _ANSI_ESCAPE.sub("", record.getMessage()) json_record: Final[dict[str, object]] = { "message": message_str, "level": record.levelname, @@ -799,12 +826,14 @@ class CorrelationPlainFormatter(logging.Formatter): """Appends trace_id/session_id to plain-text log lines stamped by CorrelationContextFilter. Mirrors JsonFormatter's handling of these two fields so request_correlation_in_logs - behaves the same whether or not json_logs is enabled. + behaves the same whether or not json_logs is enabled. Unless it was built with the color format, + it also drops the ANSI color codes a message carries, which a piped log would store as text. """ def format(self, record: logging.LogRecord) -> str: rendered: Final = super().format(record) - formatted: Final = rendered if _is_redacted(record) else redact_string(rendered) + uncolored: Final = rendered if self._fmt == _COLOR_LOG_FORMAT else _ANSI_ESCAPE.sub("", rendered) + formatted: Final = uncolored if _is_redacted(record) else redact_string(uncolored) trace_id: Final = getattr(record, "trace_id", None) session_id: Final = getattr(record, "session_id", None) if not trace_id and not session_id: @@ -898,29 +927,46 @@ verbose_proxy_logger.addFilter(_diagnostic_filter) verbose_proxy_stdout_logger.addFilter(_diagnostic_filter) verbose_logger.addFilter(_diagnostic_filter) -_ANSI_ESCAPE: Final = re.compile(r"\x1b\[[0-9;]*m") - def print_startup_message(message: str) -> None: """Write a boot-time message meant for the operator's terminal. - Plain mode prints it unchanged, colors and spacing included. With JSON logs on, every line of - stdout is read by a log pipeline, so the message goes out as one INFO record of the proxy logger - instead, on stdout like the print whatever handlers the logging config gave that logger, with the - ANSI color codes and surrounding blank lines removed, located at the caller and emitted at any log - level, whole and unredacted as the print showed it; a blank spacer produces no record. + LITELLM_SUPPRESS_STARTUP_OUTPUT or litellm_settings.suppress_startup_output turns every one of them + off; a warning goes through print_startup_warning instead. Plain mode prints it with its spacing, + keeping the ANSI colors only on a color terminal by the same check as the plain log format, so a + container log never stores escape codes. + With JSON logs on, every line of stdout is read by a log pipeline, so the message goes out as one + INFO record of the proxy logger instead, on stdout like the print whatever handlers the logging + config gave that logger, with the ANSI color codes and surrounding blank lines removed, located at + the caller and emitted at any log level, whole and unredacted as the print showed it; a blank + spacer produces no record. """ + if litellm.suppress_startup_output is True: + return + frame: Final = inspect.currentframe() + _write_startup_message(message, frame.f_back if frame is not None else None, logging.INFO) + + +def print_startup_warning(message: str) -> None: + """Write a boot-time warning as print_startup_message writes a message, even with suppress_startup_output + on, since it names something the operator has to fix. With JSON logs on it is a WARNING record, on + stdout where the print put it, so a collector that filters on the level field keeps it.""" + frame: Final = inspect.currentframe() + _write_startup_message(message, frame.f_back if frame is not None else None, logging.WARNING) + + +def _write_startup_message(message: str, caller: FrameType | None, level: int) -> None: if not litellm.json_logs: - print(message) # noqa: T201 # plain mode: this is the operator-terminal print it replaces + import click + + click.echo(terminal_text(message)) return text: Final = _ANSI_ESCAPE.sub("", message).strip("\n") if not text: return - frame: Final = inspect.currentframe() - caller: Final = frame.f_back if frame is not None else None record: Final = verbose_proxy_logger.makeRecord( verbose_proxy_logger.name, - logging.INFO, + level, caller.f_code.co_filename if caller is not None else __file__, caller.f_lineno if caller is not None else 0, text, @@ -934,8 +980,8 @@ def print_startup_message(message: str) -> None: def _startup_json_handler() -> logging.Handler: - """The proxy's JSON handler on stdout, filters included, for one startup message.""" - handler: Final = LevelRoutingStreamHandler() + """The proxy's JSON handler on stdout, filters included, for one startup message or warning.""" + handler: Final = logging.StreamHandler(sys.stdout) handler.setFormatter(JsonFormatter()) handler.addFilter(_secret_filter) handler.addFilter(_correlation_filter) diff --git a/litellm/anthropic_beta_gated_fields.py b/litellm/anthropic_beta_gated_fields.py new file mode 100644 index 00000000000..fc36699bc93 --- /dev/null +++ b/litellm/anthropic_beta_gated_fields.py @@ -0,0 +1,325 @@ +"""Body fields that an ``anthropic-beta`` value gates. + +``anthropic_beta_headers_config.json`` says which beta each provider leg +carries. When a leg drops a beta, the request fields that beta unlocks +would make the provider 400. Under ``drop_params`` the same request degrades on +that leg by dropping those fields too; without it the request goes out as sent +and the provider answers its own 400, like any unsupported param. Every helper +here is copy-on-write: the caller's containers are returned untouched when +nothing is stripped, and a shallow copy is returned otherwise, so a fallback +leg still sends the full request. +""" + +from collections.abc import Mapping, Sequence, Set +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final + +from pydantic import TypeAdapter, ValidationError + +from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params +from litellm.types.llms.anthropic import ( + ANTHROPIC_BETA_HEADER_VALUES, + ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER, + ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER, + ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER, + ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, +) + +ANTHROPIC_MID_CONVERSATION_SYSTEM_CLEAR_AT_BETA_HEADER: Final = "mid-conversation-system-clear-at-2026-08-21" +ANTHROPIC_INLINE_TOOLS_BETA_HEADER: Final = "inline-tools-2026-09-15" + +THINKING_BLOCK_BINDING: Final = "thinking.block_binding" +THINKING_DISPLAY_UPDATES: Final = "thinking.display=updates" +SYSTEM_CLEAR_AT: Final = "messages[].system.clear_at" +SYSTEM_OUTPUT_CONFIG: Final = "messages[].system.output_config" +SYSTEM_TOOL_CHANGES: Final = "messages[].system.content[].tool_addition|tool_removal" +SYSTEM_MESSAGE_EMPTIED: Final = "messages[].system (emptied by the strip)" + +_TOOL_CHANGE_BLOCK_TYPES: Final = frozenset({"tool_addition", "tool_removal"}) + +BETA_GATED_FIELDS: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType( + { + ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER: (THINKING_BLOCK_BINDING,), + ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER: (THINKING_DISPLAY_UPDATES,), + ANTHROPIC_MID_CONVERSATION_SYSTEM_CLEAR_AT_BETA_HEADER: (SYSTEM_CLEAR_AT,), + ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER: (SYSTEM_OUTPUT_CONFIG,), + ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value: (SYSTEM_OUTPUT_CONFIG,), + ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER: (SYSTEM_TOOL_CHANGES,), + ANTHROPIC_INLINE_TOOLS_BETA_HEADER: (SYSTEM_TOOL_CHANGES,), + } +) + + +def _betas_unlocking(field: str) -> frozenset[str]: + return frozenset(beta for beta, fields in BETA_GATED_FIELDS.items() if field in fields) + + +_FIELD_BETAS: Final[Mapping[str, frozenset[str]]] = MappingProxyType( + { + field: _betas_unlocking(field) for fields in BETA_GATED_FIELDS.values() for field in fields + } # comprehension-ok: flattening the beta-to-fields map into a field-to-betas index +) + + +def field_is_gated(field: str, active_betas: Set[str]) -> bool: + """True when no active beta unlocks ``field``, so the field must be stripped.""" + return _FIELD_BETAS[field].isdisjoint(active_betas) + + +_MAPPING_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_SEQUENCE_ADAPTER: Final = TypeAdapter(tuple[object, ...]) +_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, str]) + + +def _as_mapping(value: object) -> Mapping[str, object] | None: + try: + return _MAPPING_ADAPTER.validate_python(value, strict=True) + except ValidationError: + return None + + +def _as_sequence(value: object) -> tuple[object, ...] | None: + try: + return _SEQUENCE_ADAPTER.validate_python(value) + except ValidationError: + return None + + +def _is_tool_change_block(block: object) -> bool: + mapping: Final = _as_mapping(block) + block_type: Final = mapping.get("type") if mapping is not None else None + return isinstance(block_type, str) and block_type in _TOOL_CHANGE_BLOCK_TYPES + + +def _header_beta_values(headers: Mapping[str, str]) -> frozenset[str]: + joined: Final = ",".join(value for key, value in headers.items() if key.lower() == "anthropic-beta") + return frozenset(piece.strip() for piece in joined.split(",") if piece.strip()) + + +def _validated_headers(headers: object) -> Mapping[str, str]: + try: + return _HEADERS_ADAPTER.validate_python(headers) + except ValidationError: + return MappingProxyType({}) + + +def active_beta_values(headers: object, body_betas: object = ()) -> frozenset[str]: + """The ``anthropic-beta`` values a request will send: the comma-joined header (key matched + case-insensitively) plus the body's ``anthropic_beta`` list on legs that carry betas in the body.""" + body_values: Final = _as_sequence(body_betas) or () + body_betas_as_strings: Final = frozenset(beta for beta in body_values if isinstance(beta, str)) + return _header_beta_values(_validated_headers(headers)) | body_betas_as_strings + + +@dataclass(frozen=True, slots=True) +class StrippedThinking: + thinking: object + stripped: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class StrippedMessages: + messages: Sequence[object] + stripped: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class StrippedRequest: + request: Mapping[str, object] + stripped: tuple[str, ...] + + +def strip_beta_gated_thinking(thinking: object, active_betas: Set[str], *, drop_params: object) -> StrippedThinking: + """Drop ``thinking`` sub-fields whose beta is absent. Returns the same object when nothing is gated.""" + mapping: Final = _as_mapping(thinking) + if mapping is None or not drops_unsupported_params(drop_params): + return StrippedThinking(thinking=thinking, stripped=()) + binding_gated: Final = "block_binding" in mapping and field_is_gated(THINKING_BLOCK_BINDING, active_betas) + display_gated: Final = mapping.get("display") == "updates" and field_is_gated( + THINKING_DISPLAY_UPDATES, active_betas + ) + if not binding_gated and not display_gated: + return StrippedThinking(thinking=thinking, stripped=()) + gated_keys: Final = frozenset( + {*(("block_binding",) if binding_gated else ()), *(("display",) if display_gated else ())} + ) + stripped: Final = ( + *((THINKING_BLOCK_BINDING,) if binding_gated else ()), + *((THINKING_DISPLAY_UPDATES,) if display_gated else ()), + ) + return StrippedThinking( + thinking={key: value for key, value in mapping.items() if key not in gated_keys}, + stripped=stripped, + ) + + +def _strip_beta_gated_system_message(message: Mapping[str, object], active_betas: Set[str]) -> StrippedMessages: + """One mid-conversation ``role: system`` message. ``messages`` holds the rewritten message, or nothing + when the strip left it with neither content nor ``output_config``, since such a system turn is itself a 400.""" + clear_at_gated: Final = "clear_at" in message and field_is_gated(SYSTEM_CLEAR_AT, active_betas) + output_config_gated: Final = "output_config" in message and field_is_gated(SYSTEM_OUTPUT_CONFIG, active_betas) + content: Final = message.get("content") + blocks: Final = _as_sequence(content) if not isinstance(content, str) else None + tool_changes_gated: Final = ( + blocks is not None + and any(_is_tool_change_block(block) for block in blocks) + and field_is_gated(SYSTEM_TOOL_CHANGES, active_betas) + ) + if not clear_at_gated and not output_config_gated and not tool_changes_gated: + return StrippedMessages(messages=(message,), stripped=()) + kept_content: Final[object] = ( + [block for block in blocks if not _is_tool_change_block(block)] + if tool_changes_gated and blocks is not None + else content + ) + gated_keys: Final = frozenset( + {*(("clear_at",) if clear_at_gated else ()), *(("output_config",) if output_config_gated else ())} + ) + rewritten: Final = { + key: (kept_content if key == "content" else value) for key, value in message.items() if key not in gated_keys + } + stripped: Final = ( + *((SYSTEM_CLEAR_AT,) if clear_at_gated else ()), + *((SYSTEM_OUTPUT_CONFIG,) if output_config_gated else ()), + *((SYSTEM_TOOL_CHANGES,) if tool_changes_gated else ()), + ) + if kept_content in (None, "", []) and "output_config" not in rewritten: + return StrippedMessages(messages=(), stripped=(*stripped, SYSTEM_MESSAGE_EMPTIED)) + return StrippedMessages(messages=(rewritten,), stripped=stripped) + + +def _strip_beta_gated_message(message: object, active_betas: Set[str]) -> StrippedMessages: + mapping: Final = _as_mapping(message) + if mapping is None or mapping.get("role") != "system": + return StrippedMessages(messages=(message,), stripped=()) + return _strip_beta_gated_system_message(mapping, active_betas) + + +def _ends_on_assistant(messages: Sequence[object]) -> bool: + last: Final = _as_mapping(messages[-1]) if messages else None + return last is not None and last.get("role") == "assistant" + + +def strip_beta_gated_system_messages( + messages: Sequence[object], active_betas: Set[str], *, drop_params: object +) -> StrippedMessages: + """Drop the gated fields of every mid-conversation ``role: system`` message. Returns the same + sequence when nothing is gated, or when dropping an emptied system turn would leave the conversation + ending on an assistant turn, which the model would read as a prefill to continue.""" + if not drops_unsupported_params(drop_params): + return StrippedMessages(messages=messages, stripped=()) + per_message: Final = tuple(_strip_beta_gated_message(message, active_betas) for message in messages) + stripped: Final = tuple( + name for result in per_message for name in result.stripped + ) # comprehension-ok: flattening per-message strip reports + if not stripped: + return StrippedMessages(messages=messages, stripped=()) + kept: Final = [ + message for result in per_message for message in result.messages + ] # comprehension-ok: flattening per-message results + if _ends_on_assistant(kept) and not _ends_on_assistant(messages): + return StrippedMessages(messages=messages, stripped=()) + return StrippedMessages(messages=kept, stripped=stripped) + + +def strip_beta_gated_request_fields( + request: Mapping[str, object], active_betas: Set[str], *, drop_params: object +) -> StrippedRequest: + """Apply both strips to an Anthropic ``/v1/messages`` shaped body. Returns the same mapping when + nothing is gated, otherwise a shallow copy with ``thinking`` and ``messages`` replaced.""" + thinking: Final = strip_beta_gated_thinking(request.get("thinking"), active_betas, drop_params=drop_params) + messages: Final = request.get("messages") + message_sequence: Final = _as_sequence(messages) if not isinstance(messages, str) else None + system_messages: Final = ( + strip_beta_gated_system_messages(message_sequence, active_betas, drop_params=drop_params) + if message_sequence is not None + else StrippedMessages(messages=(), stripped=()) + ) + stripped: Final = (*thinking.stripped, *system_messages.stripped) + if not stripped: + return StrippedRequest(request=request, stripped=()) + rewritten: Final = { + **request, + **({"thinking": thinking.thinking} if thinking.stripped else {}), + **({"messages": system_messages.messages} if system_messages.stripped else {}), + } + return StrippedRequest(request=rewritten, stripped=stripped) + + +@dataclass(frozen=True, slots=True) +class StrippedNativeRequest: + thinking: object + thinking_stripped: bool + messages: Sequence[object] + messages_stripped: bool + stripped: tuple[str, ...] + + +def unstripped_native_request(optional_params: object, messages: object) -> StrippedNativeRequest: + """The identity result for legs that strip later, once their final ``anthropic_beta`` list exists.""" + params: Final = _as_mapping(optional_params) or MappingProxyType({}) + message_sequence: Final = _as_sequence(messages) if not isinstance(messages, str) else None + return StrippedNativeRequest( + thinking=params.get("thinking"), + thinking_stripped=False, + messages=message_sequence if message_sequence is not None else (), + messages_stripped=False, + stripped=(), + ) + + +def strip_beta_gated_native_request( + headers: object, optional_params: object, messages: object, *, body_betas: object = (), drop_params: object +) -> StrippedNativeRequest: + """The ``/v1/messages`` seam, fed the final (already filtered) headers, the final body ``anthropic_beta`` + list on legs that carry betas in the body, plus the body parts it may rewrite.""" + if not drops_unsupported_params(drop_params): + return unstripped_native_request(optional_params, messages) + active_betas: Final = active_beta_values(headers, body_betas) + params: Final = _as_mapping(optional_params) or MappingProxyType({}) + thinking: Final = strip_beta_gated_thinking(params.get("thinking"), active_betas, drop_params=drop_params) + message_sequence: Final = _as_sequence(messages) if not isinstance(messages, str) else None + system_messages: Final = ( + strip_beta_gated_system_messages(message_sequence, active_betas, drop_params=drop_params) + if message_sequence is not None + else StrippedMessages(messages=(), stripped=()) + ) + return StrippedNativeRequest( + thinking=thinking.thinking, + thinking_stripped=bool(thinking.stripped), + messages=system_messages.messages, + messages_stripped=bool(system_messages.stripped), + stripped=(*thinking.stripped, *system_messages.stripped), + ) + + +def log_beta_gated_strips(stripped: Sequence[str], model: str | None, provider: str) -> None: + if not stripped: + return + described: Final = ", ".join( + f"{field} (unlocked by anthropic-beta {' or '.join(sorted(_FIELD_BETAS[field]))})" + if field in _FIELD_BETAS + else field + for field in sorted(set(stripped)) + ) + verbose_logger.warning( + "Dropped %s for model=%s provider=%s because drop_params is on and the provider would reject them on " + "this request, either since this provider leg does not send that anthropic-beta (see " + "anthropic_beta_headers_config.json) or since the model turns mid-conversation system messages into user " + "messages.", + described, + model, + provider, + ) + + +def strip_beta_gated_request_body( + request: Mapping[str, object], headers: object, *, drop_params: object, model: str | None, provider: str +) -> Mapping[str, object]: + gated: Final = strip_beta_gated_request_fields( + request, active_beta_values(headers, request.get("anthropic_beta") or ()), drop_params=drop_params + ) + log_beta_gated_strips(gated.stripped, model=model, provider=provider) + return gated.request diff --git a/litellm/anthropic_beta_headers_config.json b/litellm/anthropic_beta_headers_config.json index d68de6c9d4d..cee5ce15469 100644 --- a/litellm/anthropic_beta_headers_config.json +++ b/litellm/anthropic_beta_headers_config.json @@ -248,6 +248,7 @@ "text_editor_20241022": null, "text_editor_20250124": null, "thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01", + "thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18", "token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19", "web-fetch-2025-09-10": null, "web-search-2025-03-05": "web-search-2025-03-05" diff --git a/litellm/anthropic_beta_headers_manager.py b/litellm/anthropic_beta_headers_manager.py index fec1d8e4465..07730a0fbf8 100644 --- a/litellm/anthropic_beta_headers_manager.py +++ b/litellm/anthropic_beta_headers_manager.py @@ -241,6 +241,13 @@ def get_provider_name(provider: str) -> str: return aliases.get(provider, provider) +def has_beta_headers_config(provider: str) -> bool: + config: Final = _bundled_beta_headers_config() + aliases: Final = config.get("provider_aliases", {}) + canonical_provider: Final = (aliases.get(provider) if isinstance(aliases, Mapping) else None) or provider + return canonical_provider in config + + def filter_and_transform_beta_headers( beta_headers: list[str], provider: str, @@ -397,7 +404,7 @@ def update_request_with_filtered_beta( headers: dict, request_data: dict, provider: str, -) -> tuple[dict, dict]: +) -> tuple[dict[str, str], dict[str, object]]: """ Update both headers and request body beta fields based on provider support. Modifies both dicts in place and returns them. diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index 5f5b8210096..1ced13445de 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -455,17 +455,33 @@ class Cache: 3. Else use the `model` passed in kwargs, plus `model_id` when set (e.g. a SageMaker inference component) """ litellm_params: Final[dict] = kwargs.get("litellm_params", {}) or {} - metadata_sources: Final[tuple[dict, ...]] = ( - kwargs.get("metadata") or {}, - kwargs.get("litellm_metadata") or {}, - litellm_params.get("metadata") or {}, - litellm_params.get("litellm_metadata") or {}, + metadata_sources: Final = cast( # cast-ok: each source is the untyped request's metadata dict or empty + "tuple[dict[str, object], dict[str, object], dict[str, object], dict[str, object]]", + ( + kwargs.get("metadata") or {}, + kwargs.get("litellm_metadata") or {}, + litellm_params.get("metadata") or {}, + litellm_params.get("litellm_metadata") or {}, + ), ) - model_group: Final[str | None] = next( - (source["model_group"] for source in metadata_sources if source.get("model_group")), None + metadata, litellm_metadata, params_metadata, params_litellm_metadata = metadata_sources + call_model_info: Final[object] = kwargs.get("model_info") + routed_source: Final = next( + ( + source + for source in (litellm_metadata, metadata, params_litellm_metadata, params_metadata) + if call_model_info and source.get("model_group") and source.get("model_info") == call_model_info + ), + None, ) + model_group: Final = ( + str(routed_source["model_group"]) + if routed_source is not None + else next((str(source["model_group"]) for source in metadata_sources if source.get("model_group")), None) + ) + caching_group_sources: Final = (routed_source,) if routed_source is not None else metadata_sources caching_group: Final = next( - (group for source in metadata_sources if (group := self._get_caching_group(source, model_group))), None + (group for source in caching_group_sources if (group := self._get_caching_group(source, model_group))), None ) model_id_suffix: Final = ( f" model_id: {kwargs['model_id']}" if "model_id" in kwargs and kwargs["model_id"] is not None else "" diff --git a/litellm/caching/qdrant_semantic_cache.py b/litellm/caching/qdrant_semantic_cache.py index 924506dfa45..560a5ba06b8 100644 --- a/litellm/caching/qdrant_semantic_cache.py +++ b/litellm/caching/qdrant_semantic_cache.py @@ -13,7 +13,7 @@ import asyncio import json import os from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Protocol, cast +from typing import TYPE_CHECKING, Any, Final, Protocol from pydantic import TypeAdapter @@ -292,10 +292,7 @@ class QdrantSemanticCache(BaseCache): prompt: Final = get_semantic_cache_prompt_from_messages(messages) # create an embedding for prompt - embedding_response: Final = cast( - EmbeddingResponse, - self._get_embedding(prompt, metadata=kwargs.get("metadata")), - ) + embedding_response: Final = self._get_embedding(prompt, metadata=kwargs.get("metadata")) # get the embedding embedding: Final = _EMBEDDING_VECTOR.validate_python(embedding_response["data"][0]["embedding"]) @@ -331,10 +328,7 @@ class QdrantSemanticCache(BaseCache): prompt: Final = get_semantic_cache_prompt_from_messages(messages) # convert to embedding - embedding_response: Final = cast( - EmbeddingResponse, - self._get_embedding(prompt, metadata=kwargs.get("metadata")), - ) + embedding_response: Final = self._get_embedding(prompt, metadata=kwargs.get("metadata")) # get the embedding embedding: Final = _EMBEDDING_VECTOR.validate_python(embedding_response["data"][0]["embedding"]) diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index b892e4c3a1d..7268511fe7e 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -14,7 +14,9 @@ import asyncio import json import os from collections.abc import Callable, Mapping -from typing import TYPE_CHECKING, Any, Final, cast +from typing import TYPE_CHECKING, Final, cast + +from pydantic import ConfigDict, TypeAdapter import litellm from litellm._logging import print_verbose, verbose_logger @@ -38,6 +40,11 @@ if TYPE_CHECKING: from litellm.router import Router +_CACHE_RESPONSE_ADAPTER: Final[TypeAdapter[str | bytes | None]] = TypeAdapter( + str | bytes | None, config=ConfigDict(strict=True, hide_input_in_errors=True) +) + + class RedisSemanticCache(BaseCache): """ Redis-backed semantic cache for LLM responses. @@ -380,7 +387,7 @@ class RedisSemanticCache(BaseCache): ) return embedding_response["data"][0]["embedding"] - def _get_cache_logic(self, cached_response: Any) -> object: + def _get_cache_logic(self, cached_response: str | bytes | None) -> object: """ Process the cached response to prepare it for use. @@ -391,24 +398,23 @@ class RedisSemanticCache(BaseCache): The processed cache response, or None if input was None """ if cached_response is None: - return cached_response + return None # Convert bytes to string if needed - if isinstance(cached_response, bytes): - cached_response = cached_response.decode("utf-8") + response_text: Final = ( + cached_response.decode("utf-8") if isinstance(cached_response, bytes) else cached_response + ) # Convert string representation to Python object try: - cached_response = json.loads(cached_response) + return json.loads(response_text) except json.JSONDecodeError: try: - cached_response = ast.literal_eval(cached_response) + return ast.literal_eval(response_text) except (ValueError, SyntaxError) as e: print_verbose(f"Error parsing cached response: {e}") return None - return cached_response - def set_cache(self, key: str, value: object, **kwargs) -> None: """ Store a value in the semantic cache. @@ -494,7 +500,7 @@ class RedisSemanticCache(BaseCache): similarity: Final = 1 - vector_distance cached_prompt: Final = cache_hit["prompt"] - cached_response: Final = cache_hit["response"] + cached_response: Final = _CACHE_RESPONSE_ADAPTER.validate_python(cache_hit["response"]) # update kwargs["metadata"] with similarity, don't rewrite the original metadata kwargs.setdefault("metadata", {})["semantic-similarity"] = similarity @@ -645,7 +651,7 @@ class RedisSemanticCache(BaseCache): similarity: Final = 1 - vector_distance cached_prompt: Final = cache_hit["prompt"] - cached_response: Final = cache_hit["response"] + cached_response: Final = _CACHE_RESPONSE_ADAPTER.validate_python(cache_hit["response"]) # update kwargs["metadata"] with similarity, don't rewrite the original metadata kwargs.setdefault("metadata", {})["semantic-similarity"] = similarity diff --git a/litellm/completion_extras/litellm_responses_transformation/handler.py b/litellm/completion_extras/litellm_responses_transformation/handler.py index 3f7017eccc2..76d17ea9693 100644 --- a/litellm/completion_extras/litellm_responses_transformation/handler.py +++ b/litellm/completion_extras/litellm_responses_transformation/handler.py @@ -260,6 +260,7 @@ class ResponsesToCompletionBridgeHandler: structured_output=is_structured_output_request( cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict ), + replays_openai_history=custom_llm_provider == "openai", ) streamwrapper: Final = CustomStreamWrapper( completion_stream=completion_stream, @@ -363,6 +364,7 @@ class ResponsesToCompletionBridgeHandler: structured_output=is_structured_output_request( cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict ), + replays_openai_history=custom_llm_provider == "openai", ) streamwrapper: Final = CustomStreamWrapper( completion_stream=completion_stream, diff --git a/litellm/completion_extras/litellm_responses_transformation/history.py b/litellm/completion_extras/litellm_responses_transformation/history.py new file mode 100644 index 00000000000..4435de64cde --- /dev/null +++ b/litellm/completion_extras/litellm_responses_transformation/history.py @@ -0,0 +1,432 @@ +import hashlib +import json +from collections.abc import Iterator, Mapping, Sequence +from dataclasses import dataclass +from itertools import accumulate, chain, pairwise +from types import MappingProxyType +from typing import Annotated, Final, Literal, TypeAlias + +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError + +REPLAY_FIELD: Final = "openai_responses_replay" +PHASE_FIELD: Final = "phase" + +Phase: TypeAlias = Literal["commentary", "final_answer"] + + +class _Frozen(BaseModel): + model_config = ConfigDict(frozen=True) + + +class _OutputText(_Frozen): + type: Literal["output_text"] + text: str + + +class _OutputMessage(_Frozen): + type: Literal["message"] + role: Literal["assistant"] + content: tuple[_OutputText, ...] + phase: Phase | None = None + + +class _OutputReasoning(_Frozen): + type: Literal["reasoning"] + id: str + + +class _OutputToolCall(_Frozen): + type: Literal["function_call", "custom_tool_call"] + call_id: str + id: str | None = None + + +class _OutputToolSearch(_Frozen): + type: Literal["tool_search_call", "tool_search_output"] + + +_OutputItem: TypeAlias = Annotated[ + _OutputMessage | _OutputReasoning | _OutputToolCall | _OutputToolSearch, Field(discriminator="type") +] +_OUTPUT_ITEM: Final[TypeAdapter[_OutputItem]] = TypeAdapter(_OutputItem) + + +class _AnyMessage(_Frozen): + type: Literal["message"] + + +class _MessageEntry(_Frozen): + type: Literal["message"] + parts: tuple[Annotated[int, Field(ge=0)], ...] + phase: Phase | None = None + + +class _IndexedEntry(_Frozen): + type: Literal["reasoning", "function_call", "custom_tool_call", "tool_search_call", "tool_search_output"] + index: Annotated[int, Field(ge=0)] + + +_Entry: TypeAlias = Annotated[_MessageEntry | _IndexedEntry, Field(discriminator="type")] + + +class _Replay(_Frozen): + version: Literal[1] + items: tuple[_Entry, ...] + checksum: str + + +class _Namespaced(_Frozen): + namespace: str | None = None + + +class _ToolFunction(_Frozen): + name: str = "" + arguments: str = "" + provider_specific_fields: _Namespaced | None = None + + +class _ToolCall(_Frozen): + id: str + function: _ToolFunction + provider_specific_fields: _Namespaced | None = None + + +class _SummaryText(_Frozen): + type: str = "summary_text" + text: str = "" + + +class _Reasoning(_Frozen): + id: str | None = None + encrypted_content: str | None = None + summary: tuple[_SummaryText, ...] | None = None + + +class _ProviderFields(_Frozen): + phase: Phase | None = None + tool_search_items: tuple[Mapping[str, object], ...] | None = None + openai_responses_replay: object = None + + +class _ChatMessage(_Frozen): + role: str | None = None + content: str | None = None + tool_calls: tuple[_ToolCall, ...] | None = None + reasoning_items: tuple[_Reasoning, ...] | None = None + phase: Phase | None = None + provider_specific_fields: _ProviderFields | None = None + + +class _PhaseFields(_Frozen): + phase: Phase | None = None + + +class _RequestedPhaseFields(_Frozen): + phase: object = None + + +class _RequestedPhase(_Frozen): + phase: object = None + provider_specific_fields: _RequestedPhaseFields | None = None + + +@dataclass(frozen=True, slots=True) +class UnknownPhase: + value: object + + +_PHASE: Final[TypeAdapter[Phase]] = TypeAdapter(Phase) + + +class _RetainedFields(_PhaseFields): + openai_responses_replay: object = None + + +@dataclass(frozen=True, slots=True) +class ReplayedTurn: + items: tuple[Mapping[str, object], ...] + custom_call_ids: frozenset[str] + + +def _as_json(value: object) -> object: + return value.model_dump() if isinstance(value, BaseModel) else value + + +def _parsed_output_item(item: object) -> _OutputItem | None: + try: + return _OUTPUT_ITEM.validate_python(_as_json(item)) + except ValidationError: + return None + + +def _is_message(item: object) -> bool: + try: + _AnyMessage.model_validate(_as_json(item)) + except ValidationError: + return False + return True + + +def _chat_message(message: object) -> _ChatMessage | None: + try: + return _ChatMessage.model_validate(_as_json(message)) + except ValidationError: + return None + + +def requested_phase(message: object) -> Phase | UnknownPhase | None: + try: + requested: Final = _RequestedPhase.model_validate(_as_json(message)) + except ValidationError: + return None + fields: Final = requested.provider_specific_fields + value: Final = requested.phase or (fields.phase if fields else None) + if value is None: + return None + try: + return _PHASE.validate_python(value) + except ValidationError: + return UnknownPhase(value) + + +def reasoning_input_item( + item_id: str | None, summary: Sequence[object] | None, encrypted_content: str | None +) -> Mapping[str, object]: + return { + "type": "reasoning", + **({"id": item_id} if item_id else {}), + "summary": list(summary or ()), + **({"encrypted_content": encrypted_content} if encrypted_content else {}), + } + + +def retained_replay_fields(provider_specific_fields: object) -> Mapping[str, object] | None: + try: + retained: Final = _RetainedFields.model_validate(provider_specific_fields) + except ValidationError: + return None + return retained.model_dump(exclude_none=True) or None + + +def output_message_text(output_items: Sequence[object], *, first_message_only: bool) -> str: + parsed: Final = map(_parsed_output_item, output_items) + messages: Final = tuple(item for item in parsed if isinstance(item, _OutputMessage)) + return "".join(_texts(messages[:1] if first_message_only else messages)) + + +def _tool_search_entries(message: _ChatMessage) -> tuple[Mapping[str, object], ...]: + fields: Final = message.provider_specific_fields + items: Final = (fields.tool_search_items if fields else None) or () + return tuple(item for item in items if item.get("type") in ("tool_search_call", "tool_search_output")) + + +def _checksum(message: _ChatMessage, entries: Sequence[_MessageEntry | _IndexedEntry]) -> str: + fields: Final = message.provider_specific_fields + bound: Final = { + "content": message.content or "", + "tool_calls": [[call.id, call.function.name, call.function.arguments] for call in message.tool_calls or ()], + "reasoning": [ + [item.id, item.encrypted_content, [part.text for part in item.summary or ()]] + for item in message.reasoning_items or () + ], + "tool_search_items": list((fields.tool_search_items if fields else None) or ()), + "phase": message.phase or (fields.phase if fields else None), + "items": [entry.model_dump(mode="json", exclude_none=True) for entry in entries], + } + encoded: Final = json.dumps(bound, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + return hashlib.sha256(encoded.encode("utf-8", "surrogatepass")).hexdigest() + + +def _kind(item: _OutputItem) -> str: + match item: + case _OutputToolCall(): + return "tool_call" + case _OutputToolSearch(): + return "tool_search" + case _OutputMessage() | _OutputReasoning(): + return item.type + + +def _entry(items: Sequence[_OutputItem], position: int) -> _MessageEntry | _IndexedEntry: + item: Final = items[position] + if isinstance(item, _OutputMessage): + return _MessageEntry(type="message", parts=tuple(len(part.text) for part in item.content), phase=item.phase) + ordinal: Final = sum(1 for earlier in items[:position] if _kind(earlier) == _kind(item)) + return _IndexedEntry(type=item.type, index=ordinal) + + +def _texts(items: Sequence[_OutputItem]) -> Iterator[str]: + for item in items: + if isinstance(item, _OutputMessage): + yield from (part.text for part in item.content) + + +def _matches(items: Sequence[_OutputItem], message: _ChatMessage) -> bool: + reasoning_ids: Final = tuple(item.id for item in items if isinstance(item, _OutputReasoning)) + calls: Final = tuple(item for item in items if isinstance(item, _OutputToolCall)) + tool_calls: Final = message.tool_calls or () + searches: Final = sum(1 for item in items if isinstance(item, _OutputToolSearch)) + return ( + "".join(_texts(items)) == (message.content or "") + and reasoning_ids == tuple(item.id for item in message.reasoning_items or ()) + and len(calls) == len(tool_calls) + and all(chat.id in (call.call_id, call.id) for call, chat in zip(calls, tool_calls)) + and searches == len(_tool_search_entries(message)) + ) + + +def _content_phase( + output_items: Sequence[object], parsed: Sequence[_OutputItem | None], *, first_message_only: bool +) -> Phase | None: + messages: Final = tuple(item for raw, item in zip(output_items, parsed) if _is_message(raw)) + if not messages: + return None + content_message: Final = messages[0] if first_message_only else messages[-1] + return content_message.phase if isinstance(content_message, _OutputMessage) else None + + +def _with_phase(message: _ChatMessage, phase: Phase | None) -> _ChatMessage: + if phase is None: + return message + fields: Final = (message.provider_specific_fields or _ProviderFields()).model_copy(update={PHASE_FIELD: phase}) + return message.model_copy(update={"provider_specific_fields": fields}) + + +def replay_fields( + output_items: Sequence[object], message: object, *, completed: bool, first_message_only: bool +) -> Mapping[str, object]: + parsed: Final = tuple(map(_parsed_output_item, output_items)) + phase: Final = _content_phase(output_items, parsed, first_message_only=first_message_only) + phase_fields: Final[Mapping[str, object]] = MappingProxyType({PHASE_FIELD: phase} if phase else {}) + items: Final = tuple(item for item in parsed if item is not None) + parsed_message: Final = _chat_message(message) + chat: Final = _with_phase(parsed_message, phase) if parsed_message is not None else None + if not completed or not items or len(items) != len(parsed) or chat is None or not _matches(items, chat): + return phase_fields + entries: Final = tuple(_entry(items, position) for position in range(len(items))) + replay: Final = _Replay(version=1, items=entries, checksum=_checksum(chat, entries)) + return MappingProxyType({**phase_fields, REPLAY_FIELD: replay.model_dump(mode="json", exclude_none=True)}) + + +def _stored_replay(message: _ChatMessage) -> _Replay | None: + fields: Final = message.provider_specific_fields + if fields is None or fields.openai_responses_replay is None: + return None + try: + return _Replay.model_validate(fields.openai_responses_replay) + except ValidationError: + return None + + +def _message_texts(entries: Sequence[_MessageEntry], content: str) -> tuple[tuple[str, ...], ...]: + lengths: Final = tuple(chain.from_iterable(entry.parts for entry in entries)) + bounds: Final = tuple(accumulate(lengths, initial=0)) + parts: Final = tuple(content[start:end] for start, end in pairwise(bounds)) + groups: Final = tuple(accumulate((len(entry.parts) for entry in entries), initial=0)) + return tuple(parts[start:end] for start, end in pairwise(groups)) + + +def _message_item(texts: Sequence[str], phase: Phase | None) -> Mapping[str, object]: + return { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": text} for text in texts], + **({PHASE_FIELD: phase} if phase else {}), + } + + +def _function_call_item(call: _ToolCall) -> Mapping[str, object]: + namespaced: Final = call.provider_specific_fields or call.function.provider_specific_fields + namespace: Final = namespaced.namespace if namespaced else None + return { + "type": "function_call", + "call_id": call.id, + "name": call.function.name, + "arguments": call.function.arguments, + **({"namespace": namespace} if namespace else {}), + } + + +def _custom_tool_call_item(call: _ToolCall) -> Mapping[str, object]: + return { + "type": "custom_tool_call", + "call_id": call.id, + "name": call.function.name, + "input": call.function.arguments, + } + + +def _reasoning_item(item: _Reasoning) -> Mapping[str, object]: + summary: Final = tuple(part.model_dump() for part in item.summary or ()) + return reasoning_input_item(item.id, summary, item.encrypted_content) + + +@dataclass(frozen=True, slots=True) +class _Resolved: + message: _ChatMessage + texts: Mapping[int, tuple[str, ...]] + tool_searches: tuple[Mapping[str, object], ...] + + +def _input_item(resolved: _Resolved, position: int, entry: _MessageEntry | _IndexedEntry) -> Mapping[str, object]: + if isinstance(entry, _MessageEntry): + return _message_item(resolved.texts[position], entry.phase) + tool_calls: Final = resolved.message.tool_calls or () + match entry.type: + case "reasoning": + return _reasoning_item((resolved.message.reasoning_items or ())[entry.index]) + case "function_call": + return _function_call_item(tool_calls[entry.index]) + case "custom_tool_call": + return _custom_tool_call_item(tool_calls[entry.index]) + case "tool_search_call" | "tool_search_output": + return dict(resolved.tool_searches[entry.index]) + + +def _fits(message: _ChatMessage, entries: Sequence[_MessageEntry | _IndexedEntry]) -> bool: + tool_calls: Final = len(message.tool_calls or ()) + tool_searches: Final = len(_tool_search_entries(message)) + sizes: Final = MappingProxyType( + { + "reasoning": len(message.reasoning_items or ()), + "function_call": tool_calls, + "custom_tool_call": tool_calls, + "tool_search_call": tool_searches, + "tool_search_output": tool_searches, + } + ) + parts: Final = chain.from_iterable(entry.parts for entry in entries if isinstance(entry, _MessageEntry)) + indexed: Final = (entry for entry in entries if isinstance(entry, _IndexedEntry)) + return sum(parts) == len(message.content or "") and all(entry.index < sizes[entry.type] for entry in indexed) + + +def _resolve( + message: _ChatMessage, entries: Sequence[_MessageEntry | _IndexedEntry], replay_tool_search: bool +) -> ReplayedTurn | None: + indexed: Final = tuple(entry for entry in entries if isinstance(entry, _IndexedEntry)) + if not replay_tool_search and any(entry.type in ("tool_search_call", "tool_search_output") for entry in indexed): + return None + messages: Final = tuple(entry for entry in entries if isinstance(entry, _MessageEntry)) + calls: Final = tuple(entry for entry in indexed if entry.type in ("function_call", "custom_tool_call")) + texts: Final = _message_texts(messages, message.content or "") + positions: Final = tuple(position for position, entry in enumerate(entries) if isinstance(entry, _MessageEntry)) + resolved: Final = _Resolved( + message=message, + texts=MappingProxyType(dict(zip(positions, texts, strict=True))), + tool_searches=_tool_search_entries(message), + ) + tool_calls: Final = message.tool_calls or () + return ReplayedTurn( + items=tuple(_input_item(resolved, position, entry) for position, entry in enumerate(entries)), + custom_call_ids=frozenset(tool_calls[entry.index].id for entry in calls if entry.type == "custom_tool_call"), + ) + + +def replay_items(message: object, *, replay_tool_search: bool) -> ReplayedTurn | None: + chat: Final = _chat_message(message) + stored: Final = _stored_replay(chat) if chat is not None and chat.role == "assistant" else None + if chat is None or stored is None or stored.checksum != _checksum(chat, stored.items): + return None + if not _fits(chat, stored.items): + return None + return _resolve(chat, stored.items, replay_tool_search) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 62f390089dc..bb59b4617af 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -24,9 +24,19 @@ from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError import litellm from litellm import ModelResponse from litellm._logging import verbose_logger +from litellm.completion_extras.litellm_responses_transformation.history import ( + PHASE_FIELD, + UnknownPhase, + output_message_text, + reasoning_input_item, + replay_fields, + replay_items, + requested_phase, +) from litellm.integrations.anthropic_cache_control_hook import supports_openai_prompt_cache_breakpoint from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params from litellm.litellm_core_utils.prompt_templates.common_utils import ( + prompt_cache_breakpoint_for_wire, responses_reasoning_items_from_thinking_blocks, with_prompt_cache_breakpoint, ) @@ -52,7 +62,6 @@ from litellm.types.llms.openai import ( ChatCompletionToolCallChunk, ChatCompletionToolCallFunctionChunk, ChatCompletionToolParamFunctionChunk, - PromptCacheBreakpoint, Reasoning, ResponsesAPIOptionalRequestParams, ResponsesAPIResponse, @@ -490,7 +499,7 @@ def _fold_choices_into_one( choices: Sequence["Choices"], tool_calls: Sequence[Mapping[str, object]], trailing_reasoning_content: str | None, - trailing_reasoning_item: _BuiltReasoningItem | None, + trailing_reasoning_items: Sequence[_BuiltReasoningItem], tool_search_items: Sequence[Mapping[str, object]], structured_output: bool = False, ) -> "Choices": @@ -509,7 +518,7 @@ def _fold_choices_into_one( reasoning_contents: Final = (*(_message_reasoning_content(m) for m in messages), trailing_reasoning_content) reasoning_items: Final = ( *chain.from_iterable(_message_reasoning_items(m) for m in messages), - *(() if trailing_reasoning_item is None else (trailing_reasoning_item,)), + *trailing_reasoning_items, ) return Choices( message=Message( @@ -526,6 +535,61 @@ def _fold_choices_into_one( ) +def _with_replay_fields( + choice: "Choices", output_items: Sequence[object], *, completed: bool, structured_output: bool +) -> "Choices": + fields: Final = { + **(choice.message.provider_specific_fields or {}), + **replay_fields(output_items, choice.message, completed=completed, first_message_only=structured_output), + } + message: Final = choice.message.model_copy(update={"provider_specific_fields": fields or None}) + return choice.model_copy(update={"message": message}) + + +_STREAMED_OUTPUT_ITEMS: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter( + tuple[Mapping[str, object], ...] +) + + +def completed_stream_history( + output_payload: object, + tool_search_fields: Mapping[str, object] | None, + *, + completed: bool, + structured_output: bool, +) -> dict[str, object] | None: # mutable-ok: Delta.provider_specific_fields is typed as a dict + items: Final = _STREAMED_OUTPUT_ITEMS.validate_python(output_payload) + calls: Final = tuple(item for item in items if item.get("type") in ("function_call", "custom_tool_call")) + streamed_message: Final = { + "role": "assistant", + "content": output_message_text(items, first_message_only=structured_output), + "tool_calls": [tool_call_dict_from_output_item(call, index) for index, call in enumerate(calls)], + "reasoning_items": list(_reasoning_items_from_output_items(items)), + "provider_specific_fields": tool_search_fields, + } + fields: Final = { + **(tool_search_fields or {}), + **replay_fields(items, streamed_message, completed=completed, first_message_only=structured_output), + } + return fields or None + + +def _assistant_phase_field(msg: object, replay_openai_history: bool, drop_params: bool) -> Mapping[str, object]: + match requested_phase(msg) if replay_openai_history else None: + case None: + return {} + case UnknownPhase(value=value) if drop_params: + verbose_logger.debug("Chat provider: dropping assistant phase=%r under drop_params", value) + return {} + case UnknownPhase(value=value): + raise litellm.UnsupportedParamsError( + message=f"phase must be 'commentary' or 'final_answer', got {value!r}. " + "To drop it, set `litellm.drop_params=True` or pass `drop_params=True`." + ) + case phase: + return {PHASE_FIELD: phase} + + def _responses_event_type(chunk: Mapping[str, object]) -> object: event_type: Final = chunk.get("type") return event_type.value if isinstance(event_type, ResponsesAPIStreamEvents) else event_type @@ -567,19 +631,6 @@ def _map_incomplete_reason_to_finish_reason(incomplete_reason: str | None) -> Li return "length" -_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(PromptCacheBreakpoint) - - -def _prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object: - if marker is None or not drop_params: - return marker - try: - return _PROMPT_CACHE_BREAKPOINT.validate_python(marker) - except ValidationError: - verbose_logger.debug("Chat provider: dropping malformed prompt_cache_breakpoint %r under drop_params", marker) - return None - - def _input_file_from_file_value(file_value: object) -> dict[str, object]: if not isinstance(file_value, dict): return {"type": "input_file"} @@ -667,15 +718,7 @@ def _reasoning_item_to_response_input( An item without an id is sent without one: the Responses API accepts that and verifies the encrypted content on its own, while it rejects any id it did not mint. """ - item_id: Final = r_item.get("id") - encrypted_content: Final = r_item.get("encrypted_content") - return { - "type": "reasoning", - **({"id": item_id} if item_id else {}), - # summary is always required by the Responses API, even when empty - "summary": r_item.get("summary") or [], - **({"encrypted_content": encrypted_content} if encrypted_content else {}), - } + return dict(reasoning_input_item(r_item.get("id"), r_item.get("summary"), r_item.get("encrypted_content"))) def _requested_defer_loading(tool: Mapping[str, object], function_tool: Mapping[str, object]) -> object: @@ -804,12 +847,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): keep_prompt_cache_breakpoints: bool = False, replay_tool_search: bool = False, include_namespace: bool = False, + replay_openai_history: bool = False, ) -> tuple[list[object], str | None]: converted_input_items, instructions = self._convert_chat_completion_messages_to_responses_input( messages, drop_params=drop_params, replay_tool_search=replay_tool_search, include_namespace=include_namespace, + replay_openai_history=replay_openai_history, ) return ( converted_input_items @@ -825,10 +870,18 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): drop_params: bool = False, replay_tool_search: bool = False, include_namespace: bool = False, + replay_openai_history: bool = False, ) -> tuple[list[object], str | None]: input_items: Final[list[object]] = [] instructions: str | None = None - custom_tool_call_ids: Final = frozenset( + replayed_turns: Final = tuple( + replay_items(msg, replay_tool_search=replay_tool_search) if replay_openai_history else None + for msg in messages + ) + replayed_custom_call_ids: Final = frozenset( + chain.from_iterable(turn.custom_call_ids for turn in replayed_turns if turn is not None) + ) + custom_tool_call_ids: Final = replayed_custom_call_ids | frozenset( tool_call["id"] for msg in messages if msg.get("role") == "assistant" and isinstance(msg.get("tool_calls"), list) @@ -849,7 +902,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): tool_calls = msg.get("tool_calls") tool_call_id = msg.get("tool_call_id") - if role == "system": + if (replayed_turn := replayed_turns[index]) is not None: + input_items.extend(replayed_turn.items) + elif role == "system": # Extract system message as instructions if isinstance(content, str) and index < leading_system_count: if instructions: @@ -915,6 +970,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): "content": self._convert_content_to_responses_format( content, "assistant", drop_params=drop_params ), + **_assistant_phase_field(msg, replay_openai_history, drop_params), } ) for tool_call in tool_calls: @@ -955,6 +1011,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): "content": self._convert_content_to_responses_format( content, cast(str, role), drop_params=drop_params ), + **_assistant_phase_field(msg, replay_openai_history and role == "assistant", drop_params), } ) elif role == "assistant": @@ -1081,6 +1138,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): keep_prompt_cache_breakpoints=supports_prompt_cache_breakpoint, replay_tool_search=_requests_tool_search(cast(object, optional_params)), # cast-ok: upcast include_namespace=True, + replay_openai_history=provider == "openai", ) # OpenAI's Responses API rejects an empty input. For a system-only # request, carry the system message as a system-role input item instead @@ -1165,6 +1223,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): output_items: Sequence[object], handle_raw_dict_callback: Callable[..., tuple["Choices | None", int]] | None = None, structured_output: bool = False, + replay_openai_history: bool = False, + completed: bool = False, ) -> list["Choices"]: """ Convert Responses API output items to chat completion choices. @@ -1174,6 +1234,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): handle_raw_dict_callback: Optional callback for handling raw dict items structured_output: The chat request carried a json response_format, so only the first message item becomes content + replay_openai_history: Record the output's item sequence for the next turn + completed: The response finished, so its item sequence is whole Returns: List of Choices objects @@ -1196,7 +1258,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): choices: Final[list[Choices]] = [] index = 0 reasoning_content: str | None = None - pending_reasoning_item: _BuiltReasoningItem | None = None + pending_reasoning_items: tuple[_BuiltReasoningItem, ...] = () # Collect all tool calls to put them in a single choice # (Chat Completions API expects all tool calls in one message) @@ -1205,12 +1267,19 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): for item in output_items: if isinstance(item, ResponseReasoningItem): - pending_reasoning_item = _build_reasoning_item( - item_id=item.id, - encrypted_content=getattr(item, "encrypted_content", None), - summary_raw=item.summary, + pending_reasoning_items = ( + *(pending_reasoning_items if replay_openai_history else ()), + _build_reasoning_item( + item_id=item.id, + encrypted_content=getattr(item, "encrypted_content", None), + summary_raw=item.summary, + ), + ) + reasoning_content = " ".join( + s["text"] + for s in chain.from_iterable(built["summary"] for built in pending_reasoning_items) + if s.get("text") ) - reasoning_content = " ".join(s["text"] for s in pending_reasoning_item["summary"] if s.get("text")) elif isinstance(item, ResponseOutputMessage): for content in item.content: @@ -1227,7 +1296,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): annotations=annotations, reasoning_items=cast( list[ChatCompletionReasoningItem] | None, - ([pending_reasoning_item] if pending_reasoning_item is not None else None), + (list(pending_reasoning_items) or None), ), ) @@ -1240,7 +1309,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ) reasoning_content = None # flush - pending_reasoning_item = None # flush + pending_reasoning_items = () # flush index += 1 elif isinstance(item, ResponseFunctionToolCall): @@ -1300,16 +1369,17 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): output_items, "stop", tool_search_items=tool_search_items ) ] - return [ - _fold_choices_into_one( - choices=choices, - tool_calls=accumulated_tool_calls, - trailing_reasoning_content=reasoning_content, - trailing_reasoning_item=pending_reasoning_item, - tool_search_items=tool_search_items, - structured_output=structured_output, - ) - ] + folded: Final = _fold_choices_into_one( + choices=choices, + tool_calls=accumulated_tool_calls, + trailing_reasoning_content=reasoning_content, + trailing_reasoning_items=pending_reasoning_items, + tool_search_items=tool_search_items, + structured_output=structured_output, + ) + if not replay_openai_history: + return [folded] + return [_with_replay_fields(folded, output_items, completed=completed, structured_output=structured_output)] @staticmethod def _build_empty_choice( @@ -1443,6 +1513,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): structured_output=is_structured_output_request( cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict ), + replay_openai_history=_served_provider( + cast("Mapping[str, object]", litellm_params) # cast-ok: litellm_params is an untyped dict + ) + == "openai", + completed=raw_response.status == "completed", ) response_is_incomplete: Final = raw_response.status == "incomplete" or ( @@ -1506,9 +1581,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): sync_stream: bool, json_mode: bool | None = False, structured_output: bool = False, + replays_openai_history: bool = False, ) -> BaseModelResponseIterator: return OpenAiResponsesToChatCompletionStreamIterator( - streaming_response, sync_stream, json_mode, structured_output=structured_output + streaming_response, + sync_stream, + json_mode, + structured_output=structured_output, + replays_openai_history=replays_openai_history, ) def _convert_content_str_to_input_text(self, content: str, role: str) -> dict[str, object]: @@ -1586,7 +1666,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): if original_type == "text": converted = with_prompt_cache_breakpoint( self._convert_content_str_to_input_text(item.get("text", ""), role), - _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), + prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), ) result.append(converted) verbose_logger.debug("Chat provider: text -> %s", converted) @@ -1599,14 +1679,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): cast(ChatCompletionImageObject, item), role ), ), - _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), + prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), ) result.append(converted) verbose_logger.debug("Chat provider: image_url -> %s", converted) elif original_type == "input_audio": converted = with_prompt_cache_breakpoint( {"type": "input_audio", "input_audio": item.get("input_audio")}, - _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), + prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), ) result.append(converted) verbose_logger.debug("Chat provider: input_audio -> %s", converted) @@ -1622,7 +1702,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): _input_file_from_file_value( cast("ChatCompletionFileObject", item).get("file"), # cast-ok: type tag checked ), - _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), + prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), ) result.append(converted) verbose_logger.debug("Chat provider: file -> %s", converted) @@ -1646,7 +1726,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): # Default to input_text for unknown types converted = with_prompt_cache_breakpoint( self._convert_content_str_to_input_text(str(item.get("text", item)), role), - _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), + prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), ) result.append(converted) verbose_logger.debug("Chat provider: unknown(%s) -> %s", original_type, converted) @@ -1890,8 +1970,10 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): sync_stream: bool, json_mode: bool | None = False, structured_output: bool = False, + replays_openai_history: bool = False, ): super().__init__(streaming_response, sync_stream, json_mode) + self._replays_openai_history: Final = replays_openai_history self._chat_completion_id: str | None = None self._served_service_tier: str | None = None self._tool_call_index_map: dict[int, int] = {} # mutable-ok: per-stream accumulator state @@ -1938,6 +2020,8 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): def translate_responses_chunk_to_openai_stream( parsed_chunk: dict | BaseModel, tool_call_index_map: dict[int, int] | None = None, # mutable-ok: per-stream state, remapped in place + replays_openai_history: bool = False, + structured_output: bool = False, reasoning_summary_streamed: bool = False, ) -> "ModelResponseStream": """ @@ -1946,6 +2030,8 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): Args: parsed_chunk: Dict containing the Responses API event chunk tool_call_index_map: Per-stream output_index -> sequential tool_call index map + replays_openai_history: Record the completed output's item sequence for the next turn + structured_output: Only the first message item became content reasoning_summary_streamed: Whether earlier chunks already streamed reasoning summary text Returns: @@ -2159,6 +2245,10 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): terminal_reasoning_items_typed: Final = _as_chat_reasoning_items( _reasoning_items_from_output_items(output_items) ) + output_payload: Final = cast(object, output_items) # cast-ok: untyped + tool_search_fields: Final = _tool_search_provider_fields( + _tool_search_items_from_output_items(output_payload) + ) usage = None if response_data.get("usage"): @@ -2172,10 +2262,17 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): StreamingChoices( index=0, delta=Delta( - content="", + content=None if replays_openai_history else "", reasoning_items=terminal_reasoning_items_typed, - provider_specific_fields=_tool_search_provider_fields( - _tool_search_items_from_output_items(cast(object, output_items)) # cast-ok: untyped + provider_specific_fields=( + completed_stream_history( + output_payload, + tool_search_fields, + completed=event_type == "response.completed", + structured_output=structured_output, + ) + if replays_openai_history + else tool_search_fields ), ), finish_reason=finish_reason, @@ -2223,6 +2320,8 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): else OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream( chunk, tool_call_index_map=self._tool_call_index_map, + replays_openai_history=self._replays_openai_history, + structured_output=self._structured_output, reasoning_summary_streamed=self._reasoning_summary_streamed, ) ) diff --git a/litellm/constants.py b/litellm/constants.py index 42ad624026c..d02114f56b1 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -158,7 +158,7 @@ DEFAULT_REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_int("DEFAULT_REPLICATE_ DEFAULT_IMAGE_TOKEN_COUNT: Final = get_env_int("DEFAULT_IMAGE_TOKEN_COUNT", 250) HF_CONFIG_FETCH_TIMEOUT_SECONDS: Final = 10.0 HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS: Final = get_env_int_in_range( - "HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS", 300, 1, 86400 + "HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS", 60, 1, 86400 ) # Maximum wall-clock seconds a streaming response is allowed to run. @@ -627,6 +627,7 @@ STS_CREDENTIAL_EXPIRY_SAFETY_MARGIN_SECONDS: Final = 60 BEDROCK_MIN_THINKING_BUDGET_TOKENS: Final = get_env_int("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024) # Anthropic's Messages API rejects thinking.budget_tokens < 1024. ANTHROPIC_MIN_THINKING_BUDGET_TOKENS: Final = 1024 +ANTHROPIC_BILLING_METADATA_PREFIX: Final = "x-anthropic-billing-header:" REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_float("REPLICATE_POLLING_DELAY_SECONDS", 0.5) DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS: Final = get_env_int("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 128000) DEFAULT_OCI_CHAT_MAX_TOKENS: Final = 4096 diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index b5dc0ce13de..a4a9a5bee39 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -22,7 +22,7 @@ from litellm.constants import ( OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES, REALTIME_METERED_USAGE_KEY, ) -from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.core_helpers import as_str_mapping, billed_usage_cost from litellm.litellm_core_utils.cost_discount import resolve_cost_discount from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, is_decisions_call_type from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR @@ -50,6 +50,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( parse_prompt_tokens_details, select_cost_metric_for_model, ) +from litellm.litellm_core_utils.llm_response_utils.response_metadata import response_ms_since_logging_start from litellm.llms.anthropic.cost_calculation import ( cost_per_token as anthropic_cost_per_token, ) @@ -1396,9 +1397,9 @@ def _apply_cost_margin( def _store_cost_breakdown_in_logging_obj( litellm_logging_obj: LitellmLoggingObject | None, - prompt_tokens_cost_usd_dollar: float, - completion_tokens_cost_usd_dollar: float, - cost_for_built_in_tools_cost_usd_dollar: float, + prompt_tokens_cost_usd_dollar: float | None, + completion_tokens_cost_usd_dollar: float | None, + cost_for_built_in_tools_cost_usd_dollar: float | None, total_cost_usd_dollar: float, additional_costs: dict | None = None, original_cost: float | None = None, @@ -1500,9 +1501,45 @@ def _response_time_ms_for_cost( return float(stamped) if total_time: return total_time - if litellm_logging_obj is not None: - return litellm_logging_obj.get_response_ms() - return 0.0 + if litellm_logging_obj is None: + return 0.0 + reported_ms: Final = response_ms_since_logging_start(litellm_logging_obj) + if reported_ms is not None and litellm_logging_obj.stream is not True: + return reported_ms + return litellm_logging_obj.get_response_ms() + + +def _adjusted_provider_billed_cost( + reported_cost: float, + custom_llm_provider: str | None, + model: str | None, + region_name: str | None, + litellm_logging_obj: LitellmLoggingObject | None, + service_tier: str | None, + data_residency: str | None, +) -> float: + discounted_cost, discount_percent, discount_amount = _apply_cost_discount( + base_cost=reported_cost, custom_llm_provider=custom_llm_provider, model=model, region_name=region_name + ) + adjusted_cost, margin_percent, margin_fixed_amount, margin_total_amount = _apply_cost_margin( + base_cost=discounted_cost, custom_llm_provider=custom_llm_provider + ) + _store_cost_breakdown_in_logging_obj( + litellm_logging_obj=litellm_logging_obj, + prompt_tokens_cost_usd_dollar=None, + completion_tokens_cost_usd_dollar=None, + cost_for_built_in_tools_cost_usd_dollar=None, + total_cost_usd_dollar=adjusted_cost, + original_cost=reported_cost, + discount_percent=discount_percent, + discount_amount=discount_amount, + margin_percent=margin_percent, + margin_fixed_amount=margin_fixed_amount, + margin_total_amount=margin_total_amount, + service_tier=service_tier, + data_residency=data_residency, + ) + return adjusted_cost def completion_cost( @@ -1616,6 +1653,9 @@ def completion_cost( ) explicit_pricing: Final = custom_pricing is True or base_model is not None + caller_set_prices: Final = ( + explicit_pricing or custom_cost_per_token is not None or custom_cost_per_second is not None + ) custom_priced_request_model: Final = model if custom_pricing is True else None configured_cost_per_query: Final = ( _configured_search_cost_per_query(litellm_logging_obj) if custom_pricing else None @@ -1666,6 +1706,7 @@ def completion_cost( raw_candidates if custom_pricing is True else _priced_candidates_before_unpriced_rules(raw_candidates) ) + priced_response: Final[object] = completion_response for idx, (candidate_model, candidate_pricing_provider) in enumerate(potential_model_names): model = candidate_model try: @@ -1984,6 +2025,19 @@ def completion_cost( total_time=total_time, ) model_for_discount = custom_priced_request_model or model + if ( + not caller_set_prices + and (reported_cost := billed_usage_cost(priced_response, custom_llm_provider)) is not None + ): + return _adjusted_provider_billed_cost( + reported_cost=reported_cost, + custom_llm_provider=custom_llm_provider, + model=model_for_discount, + region_name=region_name if isinstance(region_name, str) else None, + litellm_logging_obj=litellm_logging_obj, + service_tier=service_tier, + data_residency=data_residency, + ) # Calculate cost based on prompt_tokens, completion_tokens if ( "togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai" diff --git a/litellm/exceptions.py b/litellm/exceptions.py index cd5139f5f79..61e3a8fb5dc 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -139,6 +139,7 @@ class AuthenticationError(openai.AuthenticationError): litellm_debug_info: str | None = None, max_retries: int | None = None, num_retries: int | None = None, + body: Mapping[str, object] | None = None, ): self.status_code = 401 self.message = f"litellm.AuthenticationError: {message}" @@ -152,7 +153,7 @@ class AuthenticationError(openai.AuthenticationError): request=httpx.Request(method="GET", url="https://litellm.ai"), # mock request object ) super().__init__( - self.message, response=self.response, body=None + self.message, response=self.response, body=body ) # Call the base class constructor with the parameters it needs def __str__(self): @@ -189,6 +190,7 @@ class NotFoundError(openai.NotFoundError): litellm_debug_info: str | None = None, max_retries: int | None = None, num_retries: int | None = None, + body: Mapping[str, object] | None = None, ): self.status_code = 404 self.message = f"litellm.NotFoundError: {message}" @@ -202,7 +204,7 @@ class NotFoundError(openai.NotFoundError): request=httpx.Request(method="GET", url="https://litellm.ai"), # mock request object ) super().__init__( - self.message, response=self.response, body=None + self.message, response=self.response, body=body ) # Call the base class constructor with the parameters it needs def __str__(self): diff --git a/litellm/images/main.py b/litellm/images/main.py index 6e77cbb7046..b3247ff938b 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -641,6 +641,8 @@ def image_generation( timeout=timeout, client=custom_client, ) + elif custom_llm_provider == LlmProviders.KYMA.value: + raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider) return model_response except Exception as e: diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index b0deec24c1a..fc1f1830129 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -72,6 +72,8 @@ MAX_CACHE_CONTROL_BLOCKS: Final = 4 CACHE_BREAKPOINT_KEYS: Final = ("cache_control", "prompt_cache_breakpoint") OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION: Final = (5, 6) _GPT_VERSION_PATTERN: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?") +_FINE_TUNED_BASE_PATTERN: Final = re.compile(r"^ft:([^:]+)") +_O_SERIES_PATTERN: Final = re.compile(r"^o\d+(?:-|$)") OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES: Final = frozenset( {"text", "image", "image_url", "file", "input_audio", "input_text", "input_image", "input_file"} ) @@ -125,6 +127,10 @@ def _served_provider_for_model(model: str) -> str | None: def supports_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None = None) -> bool: + return openai_prompt_cache_breakpoint_support(model, custom_llm_provider) is True + + +def openai_prompt_cache_breakpoint_support(model: str, custom_llm_provider: str | None = None) -> bool | None: hosted_flag: Final = ( None if custom_llm_provider is None @@ -135,9 +141,19 @@ def supports_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str model_map_flag: Final = _model_map_prompt_cache_breakpoint_flag(model) if model_map_flag is not None: return model_map_flag - version_match: Final = _GPT_VERSION_PATTERN.match(model.rsplit("/", 1)[-1].lower()) - if version_match is None: + fine_tuned_base: Final = _FINE_TUNED_BASE_PATTERN.match(model.rsplit("/", 1)[-1]) + if fine_tuned_base is not None: + return False if _named_model_breakpoint_support(fine_tuned_base.group(1)) is False else None + return _named_model_breakpoint_support(model) + + +def _named_model_breakpoint_support(model: str) -> bool | None: + bare_model: Final = model.rsplit("/", 1)[-1].lower() + if _O_SERIES_PATTERN.match(bare_model) is not None: return False + version_match: Final = _GPT_VERSION_PATTERN.match(bare_model) + if version_match is None: + return None version: Final = (int(version_match.group(1)), int(version_match.group(2) or 0)) return version >= OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index a4de1be1842..0579eb15baf 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -513,7 +513,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac data: dict, user_api_key_dict: UserAPIKeyAuth, call_type: CallTypesLiteral, - ) -> Any: + ) -> object: pass async def async_post_call_streaming_hook( diff --git a/litellm/integrations/datadog/datadog_cost_management.py b/litellm/integrations/datadog/datadog_cost_management.py index 1a81fa019dc..d4b75fc0bce 100644 --- a/litellm/integrations/datadog/datadog_cost_management.py +++ b/litellm/integrations/datadog/datadog_cost_management.py @@ -205,11 +205,12 @@ class DatadogCostManagementLogger(CustomBatchLogger): # see _RESERVED_TAG_KEYS for the rationale. if self.cost_tag_keys: allow: Final = set(self.cost_tag_keys) + client_allow: Final = allow - {"is_silent_experiment"} for rt in log.get("request_tags") or []: if not isinstance(rt, str) or ":" not in rt: continue k, _, v = rt.partition(":") - if k in allow and v: + if k in client_allow and v: self._set_custom_tag(tags, k, v) for k, v in metadata.items(): if k in allow and v is not None and not isinstance(v, (dict, list)): @@ -218,7 +219,7 @@ class DatadogCostManagementLogger(CustomBatchLogger): nested = metadata.get(nested_key) if isinstance(nested, dict): for k, v in nested.items(): - if k in allow and v is not None and not isinstance(v, (dict, list)): + if k in client_allow and v is not None and not isinstance(v, (dict, list)): self._set_custom_tag(tags, k, str(v)) return tags diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index b8c6e91d0d7..bd68ffe8cd6 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -833,6 +833,12 @@ class LangFuseLogger: ("vertex_location", vertex_location, bool(vertex_location)), ("aws_region_name", aws_region_name, bool(aws_region_name)), ("cache_hit", kwargs.get("cache_hit") or False, "cache_hit" in kwargs), + ( + "is_silent_experiment", + "is_silent_experiment" in allowlisted_metadata + and allowlisted_metadata["is_silent_experiment"] is True, + "is_silent_experiment" in allowlisted_metadata, + ), ) enrichments: Final[Mapping[str, object]] = { key: value for key, value, include in candidate_enrichments if include diff --git a/litellm/integrations/opentelemetry_utils/base_otel_llm_obs_attributes.py b/litellm/integrations/opentelemetry_utils/base_otel_llm_obs_attributes.py index 32cf93b7937..97a970c8966 100644 --- a/litellm/integrations/opentelemetry_utils/base_otel_llm_obs_attributes.py +++ b/litellm/integrations/opentelemetry_utils/base_otel_llm_obs_attributes.py @@ -15,7 +15,7 @@ class BaseLLMObsOTELAttributes(ABC): pass -def cast_as_primitive_value_type(value) -> str | bool | int | float: +def cast_as_primitive_value_type(value: object) -> str | bool | int | float: """ Converts a value to an OTEL-supported primitive for Arize/Phoenix observability. """ diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index bc8344fa686..e62b99f8b0f 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -1173,8 +1173,8 @@ class PrometheusLogger(CustomLogger): # Build label filters from valid configurations label_filters: Final = self._build_label_filters(parsed_configs) - # Pretty print the processed configuration - self._pretty_print_prometheus_config(label_filters) + if litellm.suppress_startup_output is not True: + self._pretty_print_prometheus_config(label_filters) return label_filters def _parse_exclude_config(self) -> tuple[frozenset[str], frozenset[str]]: @@ -5148,6 +5148,12 @@ def _get_combined_custom_metadata_from_standard_logging_payload( **(requester_metadata if isinstance(requester_metadata, dict) else {}), **(user_api_key_auth_metadata if isinstance(user_api_key_auth_metadata, dict) else {}), **(spend_logs_metadata if isinstance(spend_logs_metadata, dict) else {}), + "is_silent_experiment": ( + "true" + if "is_silent_experiment" in standard_logging_metadata + and standard_logging_metadata["is_silent_experiment"] is True + else None + ), } diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 6e1ba3eea08..c25fcfe3e70 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -37,6 +37,7 @@ from litellm.integrations.websearch_interception.transformation import ( from litellm.litellm_core_utils.agentic_loop_settings import ( validated_max_agentic_loops, ) +from litellm.litellm_core_utils.core_helpers import qualify_provider_stripped_model from litellm.llms.base_llm.search.transformation import SearchResponse from litellm.types.integrations.custom_logger import ( CHAT_COMPLETION_AGENTIC_SURFACE, @@ -114,6 +115,18 @@ class _WebSearchSettingsView(TypedDict): websearch_interception_params: WebSearchInterceptionConfig +def _caller_model(model: str, logging_obj: "LiteLLMLoggingObj | None", kwargs: Mapping[str, object]) -> str: + agentic_view: Final[_AgenticLoopParamsView] = { + "agentic_loop_params": logging_obj.model_call_details.get("agentic_loop_params", {}) if logging_obj else {} + } + recorded_model: Final = agentic_view["agentic_loop_params"].get("model") + custom_llm_provider: Final = kwargs.get("custom_llm_provider") + return qualify_provider_stripped_model( + recorded_model if isinstance(recorded_model, str) and recorded_model else model, + custom_llm_provider if isinstance(custom_llm_provider, str) else "", + ) + + class _SearchToolLitellmParams(TypedDict, total=False): search_provider: ReadOnly[str | None] @@ -1155,6 +1168,7 @@ class WebSearchInterceptionLogger(CustomLogger): tool_calls=tool_calls, optional_params=optional_params, kwargs=kwargs, + logging_obj=logging_obj, response_format=response_format, ) return AgenticLoopPlan( @@ -1181,6 +1195,7 @@ class WebSearchInterceptionLogger(CustomLogger): tool_calls=tool_calls, optional_params=optional_params, kwargs=kwargs, + logging_obj=logging_obj, ) return AgenticLoopPlan( run_agentic_loop=True, @@ -1195,6 +1210,7 @@ class WebSearchInterceptionLogger(CustomLogger): tool_calls: list[dict], optional_params: dict, kwargs: dict, + logging_obj: "LiteLLMLoggingObj | None", ) -> AgenticLoopRequestPatch: """Execute litellm.asearch() and build a Responses API rerun patch.""" search_tasks: Final = [ @@ -1254,9 +1270,7 @@ class WebSearchInterceptionLogger(CustomLogger): } } - full_model_name = model - if "/" not in model and isinstance(kwargs.get("custom_llm_provider"), str): - full_model_name = f"{kwargs['custom_llm_provider']}/{model}" + full_model_name: Final = _caller_model(model, logging_obj, kwargs) verbose_logger.debug( "WebSearchInterception: Built responses request patch model=%s input_items=%d searches=%d", @@ -1814,6 +1828,7 @@ class WebSearchInterceptionLogger(CustomLogger): tool_calls=tool_calls, optional_params=dict[str, object](optional_params), kwargs=dict[str, object](kwargs), + logging_obj=logging_obj, response_format=response_format, ) if request_patch.messages is None: @@ -1836,7 +1851,8 @@ class WebSearchInterceptionLogger(CustomLogger): messages: list[dict], tool_calls: list[dict], optional_params: dict, - kwargs: dict, + kwargs: dict[str, object], + logging_obj: "LiteLLMLoggingObj | None", response_format: str = "openai", ) -> AgenticLoopRequestPatch: """Execute litellm.search() and build chat-completion rerun patch.""" @@ -1922,11 +1938,7 @@ class WebSearchInterceptionLogger(CustomLogger): k: v for k, v in kwargs.items() if not k.startswith("_websearch_interception") and k not in internal_params } - full_model_name = model - if "custom_llm_provider" in kwargs: - custom_llm_provider: Final = kwargs["custom_llm_provider"] - if not model.startswith(custom_llm_provider) and "/" not in model: - full_model_name = f"{custom_llm_provider}/{model}" + full_model_name: Final = _caller_model(model, logging_obj, kwargs) verbose_logger.debug( "WebSearchInterception: Built chat completion request patch model=%s messages=%d", diff --git a/litellm/interactions/litellm_responses_transformation/handler.py b/litellm/interactions/litellm_responses_transformation/handler.py index d84f0b5ffbd..f95b5611831 100644 --- a/litellm/interactions/litellm_responses_transformation/handler.py +++ b/litellm/interactions/litellm_responses_transformation/handler.py @@ -103,6 +103,7 @@ class LiteLLMResponsesInteractionsHandler: return LiteLLMResponsesInteractionsConfig.transform_responses_response_to_interactions_response( responses_response=responses_api_response, model=model, + custom_llm_provider=custom_llm_provider, ) async def async_interactions_api_handler( @@ -138,4 +139,5 @@ class LiteLLMResponsesInteractionsHandler: return LiteLLMResponsesInteractionsConfig.transform_responses_response_to_interactions_response( responses_response=responses_api_response, model=model, + custom_llm_provider=responses_request.get("custom_llm_provider"), ) diff --git a/litellm/interactions/litellm_responses_transformation/transformation.py b/litellm/interactions/litellm_responses_transformation/transformation.py index ade33831cdc..2e9b62dcfe2 100644 --- a/litellm/interactions/litellm_responses_transformation/transformation.py +++ b/litellm/interactions/litellm_responses_transformation/transformation.py @@ -12,6 +12,7 @@ from typing import Final, cast from pydantic import BaseModel +from litellm.litellm_core_utils.core_helpers import billed_usage_cost from litellm.types.interactions import ( InteractionInput, InteractionsAPIOptionalRequestParams, @@ -190,6 +191,7 @@ class LiteLLMResponsesInteractionsConfig: def transform_responses_response_to_interactions_response( responses_response: ResponsesAPIResponse, model: str | None = None, + custom_llm_provider: str | None = None, ) -> InteractionsAPIResponse: """ Transform a Responses API response to an Interactions API response. @@ -266,9 +268,11 @@ class LiteLLMResponsesInteractionsConfig: # (total_input_tokens, total_output_tokens) usage: Final[object] = getattr(responses_response, "usage", None) if usage: + reported_cost: Final = billed_usage_cost(responses_response, custom_llm_provider) interactions_response_dict["usage"] = { "total_input_tokens": getattr(usage, "input_tokens", 0), "total_output_tokens": getattr(usage, "output_tokens", 0), + **({} if reported_cost is None else {"cost": reported_cost}), } # Add updated (same as created for now) diff --git a/litellm/litellm_core_utils/chat_completion_agentic_loop.py b/litellm/litellm_core_utils/chat_completion_agentic_loop.py index 628486f377f..2ba47cbb400 100644 --- a/litellm/litellm_core_utils/chat_completion_agentic_loop.py +++ b/litellm/litellm_core_utils/chat_completion_agentic_loop.py @@ -15,6 +15,7 @@ from litellm.litellm_core_utils.agentic_loop_settings import ( DEFAULT_MAX_AGENTIC_LOOPS, validated_max_agentic_loops, ) +from litellm.litellm_core_utils.core_helpers import qualify_agentic_followup_model from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject from litellm.llms.base_llm.base_model_iterator import MockResponseIterator from litellm.types.integrations.custom_logger import ( @@ -182,9 +183,7 @@ async def _execute_chat_completion_agentic_plan( if patch.messages is None: raise ValueError("Agentic loop plan missing patched messages") - full_model_name = patch.model or model - if "/" not in full_model_name: - full_model_name = f"{custom_llm_provider}/{full_model_name}" + full_model_name: Final = qualify_agentic_followup_model(patch.model, model, custom_llm_provider) optional_params_for_followup: Final = {**optional_params, **patch.optional_params} if patch.tools is not None: diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 55025508e40..d940baf8b56 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -2,6 +2,7 @@ ## Helper utilities import copy import logging +import math import re from collections.abc import Collection, Iterable, Iterator, Mapping from types import MappingProxyType @@ -34,6 +35,49 @@ def is_codex_user_agent(user_agent: str) -> bool: return bool(_CODEX_CLIENT_PREFIX_RE.match(user_agent)) +def qualify_provider_stripped_model(model: str, custom_llm_provider: str) -> str: + """Put the provider prefix back on a provider-stripped model.""" + if not custom_llm_provider or model.startswith(f"{custom_llm_provider}/"): + return model + if _already_routes_to(model, custom_llm_provider): + return model + return f"{custom_llm_provider}/{model}" + + +def _already_routes_to(model: str, custom_llm_provider: str) -> bool: + from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider + + return _leads_with_provider(model) and inferred_provider(model) == custom_llm_provider + + +def _leads_with_provider(model: str) -> bool: + leading_segment, separator, _ = model.partition("/") + return bool(separator) and names_a_provider(leading_segment) + + +def names_a_provider(candidate: str) -> bool: + """Whether a model name's leading segment is a provider rather than part of the model.""" + import litellm + from litellm.litellm_core_utils.get_llm_provider_logic import is_registered_custom_provider + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + provider_names: Final = {getattr(provider, "value", provider) for provider in litellm.provider_list} + return ( + candidate in provider_names + or JSONProviderRegistry.exists(candidate) + or is_registered_custom_provider(candidate) + ) + + +def qualify_agentic_followup_model(patch_model: str | None, model: str, custom_llm_provider: str) -> str: + """Resolve the model an agentic follow-up re-dispatches, keeping a hook's own provider.""" + if not patch_model or patch_model == model: + return qualify_provider_stripped_model(model, custom_llm_provider) + if _leads_with_provider(patch_model): + return patch_model + return qualify_provider_stripped_model(patch_model, custom_llm_provider) + + def safe_divide_seconds(seconds: float, denominator: float, default: float | None = None) -> float | None: """ Safely divide seconds by denominator, handling zero division. @@ -973,6 +1017,16 @@ def chunk_usage(chunk: object) -> object: return getattr(chunk, "usage", None) +USAGE_COST_BILLED_PROVIDERS: Final = frozenset({"kyma"}) + + +def billed_usage_cost(response: object, custom_llm_provider: str | None) -> float | None: + if custom_llm_provider not in USAGE_COST_BILLED_PROVIDERS: + return None + cost: Final = provider_reported_usage_cost(_usage_field(response, "usage"), custom_llm_provider) + return cost if cost is not None and math.isfinite(cost) and cost >= 0 else None + + _MESSAGES_REPORTED_COST_PROVIDERS: Final = frozenset({"opencode_go"}) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 47ec8331c83..64a95eb5f28 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -18,6 +18,7 @@ from litellm.litellm_core_utils.bug_report import ( ) from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none from litellm.litellm_core_utils.secret_redaction import redact_string as redact_secret_string +from litellm.llms.bigmodel.common_utils import raise_bigmodel_business_error from litellm.types.utils import LlmProviders from ..exceptions import ( @@ -412,6 +413,25 @@ def _map_openai_exception( body=getattr(original_exception, "body", None), ) elif "invalid_request_error" in error_str and "Incorrect API key provided" not in error_str: + invalid_request_status: Final = getattr(original_exception, "status_code", None) + if invalid_request_status == 401: + raise AuthenticationError( + message=f"{exception_provider} - {message}", + llm_provider=custom_llm_provider, + model=model, + response=response, + litellm_debug_info=extra_information, + body=getattr(original_exception, "body", None), + ) + if invalid_request_status == 404: + raise NotFoundError( + message=f"{exception_provider} - {message}", + model=model, + llm_provider=custom_llm_provider, + response=response, + litellm_debug_info=extra_information, + body=getattr(original_exception, "body", None), + ) raise BadRequestError( message=f"{exception_provider} - {message}", llm_provider=custom_llm_provider, @@ -2762,6 +2782,8 @@ def exception_type( exception_provider=exception_provider, extra_information=extra_information, ) + if custom_llm_provider == "bigmodel": + raise_bigmodel_business_error(model, f"{exception_provider} - {error_str}", original_exception) # pyright: ignore[reportUnknownArgumentType] # exception_type takes untyped parameters if custom_llm_provider == "openrouter": _map_openrouter_exception( model=model, diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index d9933087334..f8c8ff71f8e 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -794,6 +794,13 @@ def _get_openai_compatible_provider_info( api_base, dynamic_api_key, ) = litellm.XAIChatConfig().get_openai_compatible_provider_info(api_base, api_key) + elif custom_llm_provider == "bigmodel": + from litellm.llms.bigmodel.common_utils import get_bigmodel_api_key, get_bigmodel_url + + ( + api_base, # rebind-ok: existing provider-resolution pipeline resolves the API base in place + dynamic_api_key, # rebind-ok: existing provider-resolution pipeline resolves the API key in place + ) = get_bigmodel_url(api_base, "chat"), get_bigmodel_api_key(api_key) elif custom_llm_provider == "zai": ( api_base, diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index b19bcd941fa..940730eeb9f 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -31,7 +31,13 @@ EXTRACTION_HEALTH_CHECK_FIELDS: Final[Mapping[str, Mapping[str, str]]] = Mapping ) -def native_health_check_mode(model: str, custom_llm_provider: str | None) -> Literal["anthropic_messages"] | None: +def native_health_check_mode( + model: str, custom_llm_provider: str | None +) -> Literal["anthropic_messages", "realtime"] | None: + if custom_llm_provider == "soniox": + from litellm.llms.soniox.common_utils import soniox_health_check_mode + + return soniox_health_check_mode(model) if custom_llm_provider != "bedrock_mantle": return None from litellm.llms.bedrock_mantle.common_utils import mantle_health_check_mode diff --git a/litellm/litellm_core_utils/internal_call_metadata.py b/litellm/litellm_core_utils/internal_call_metadata.py index 1200d799018..f3dbd612bee 100644 --- a/litellm/litellm_core_utils/internal_call_metadata.py +++ b/litellm/litellm_core_utils/internal_call_metadata.py @@ -120,7 +120,7 @@ def is_unbilled_non_inference_call( origin: Final = metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) if metadata is not None else None if origin == BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN: return False - if not is_background_response(response): + if not is_background_response(_finished_response(response)): return True return origin == BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN or _is_managed_background_call(metadata) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 7f4b069ded6..c0e988412a1 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -23,7 +23,7 @@ from httpx import Response, Timeout from pydantic import BaseModel, JsonValue, TypeAdapter import litellm -from litellm._internal_context import post_response_phase +from litellm._internal_context import in_mirrored_call, outside_mirrored_call, post_response_phase from litellm._logging import ( is_debugging_on, redact_string, @@ -162,9 +162,10 @@ from litellm.types.llms.openai import ( from litellm.types.mcp import MCPPostCallResponseObject from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.rerank import RerankResponse -from litellm.types.router import RouterRateLimitError +from litellm.types.router import RouterRateLimitError, SilentExperimentFlag from litellm.types.utils import ( DEPLOYMENT_SCOPED_PRICING_FIELDS, + AgenticLoopParams, CachingDetails, CallTypes, CostBreakdown, @@ -314,7 +315,7 @@ else: _in_memory_loggers: Final[list[CustomLogger]] = [] _STANDARD_LOGGING_METADATA_RESOLVED_KEYS: Final[frozenset[str]] = frozenset( - ("used_client_oauth_token", "usage_object", KUBERNETES_POD_ROUTING_KEY) + ("used_client_oauth_token", "usage_object", KUBERNETES_POD_ROUTING_KEY, "is_silent_experiment") ) _STANDARD_LOGGING_METADATA_KEYS: Final[frozenset[str]] = ( frozenset(StandardLoggingMetadata.__annotations__.keys()) - _STANDARD_LOGGING_METADATA_RESOLVED_KEYS @@ -808,6 +809,7 @@ class Logging(LiteLLMLoggingBaseClass): litellm_params = scrub_sensitive_keys_in_metadata(litellm_params) self.litellm_params = litellm_params + self.is_silent_experiment: Final = in_mirrored_call() # Initialize cost breakdown field self.cost_breakdown: CostBreakdown | None = None @@ -1158,6 +1160,9 @@ class Logging(LiteLLMLoggingBaseClass): if "custom_llm_provider" in self.model_call_details: self.custom_llm_provider = self.model_call_details["custom_llm_provider"] + def record_agentic_loop_params(self, params: AgenticLoopParams) -> None: + self.model_call_details["agentic_loop_params"] = params + def record_applied_timeout(self, timeout: float | Timeout) -> None: self.applied_timeout = timeout self.litellm_params["timeout"] = timeout @@ -1937,10 +1942,10 @@ class Logging(LiteLLMLoggingBaseClass): def set_cost_breakdown( self, - input_cost: float, - output_cost: float, + input_cost: float | None, + output_cost: float | None, total_cost: float, - cost_for_built_in_tools_cost_usd_dollar: float, + cost_for_built_in_tools_cost_usd_dollar: float | None, additional_costs: dict | None = None, original_cost: float | None = None, discount_percent: float | None = None, @@ -1979,14 +1984,17 @@ class Logging(LiteLLMLoggingBaseClass): self.billed_token_rates = billed_token_rates self.cost_breakdown = CostBreakdown( - input_cost=input_cost, - output_cost=output_cost, total_cost=total_cost, - tool_usage_cost=cost_for_built_in_tools_cost_usd_dollar, service_tier=service_tier, data_residency=data_residency, vertex_location=vertex_location, ) + if input_cost is not None: + self.cost_breakdown["input_cost"] = input_cost + if output_cost is not None: + self.cost_breakdown["output_cost"] = output_cost + if cost_for_built_in_tools_cost_usd_dollar is not None: + self.cost_breakdown["tool_usage_cost"] = cost_for_built_in_tools_cost_usd_dollar if cache_read_cost is not None and cache_read_cost > 0: self.cost_breakdown["cache_read_cost"] = cache_read_cost if cache_creation_cost is not None and cache_creation_cost > 0: @@ -3249,7 +3257,7 @@ class Logging(LiteLLMLoggingBaseClass): """Restores trace_id/session_id contextvars once this attempt's own success logging (including any nested calls its callbacks trigger) is fully done.""" try: - with post_response_phase(): + with post_response_phase(), outside_mirrored_call(): return self._success_handler_body( result=result, start_time=start_time, @@ -3749,7 +3757,7 @@ class Logging(LiteLLMLoggingBaseClass): """Restores trace_id/session_id contextvars once this attempt's own success logging (including any nested calls its callbacks trigger) is fully done.""" try: - with post_response_phase(): + with post_response_phase(), outside_mirrored_call(): return await self._async_success_handler_body( result=result, start_time=start_time, end_time=end_time, cache_hit=cache_hit, **kwargs ) @@ -4266,13 +4274,14 @@ class Logging(LiteLLMLoggingBaseClass): ``for_async_request`` marks the sync run of a failure whose async run also logs it, so a CustomLogger that gets ``async_log_failure_event`` from that async run skips its sync hook.""" try: - return self._failure_handler_body( - exception=exception, - traceback_exception=traceback_exception, - start_time=start_time, - end_time=end_time, - for_async_request=for_async_request, - ) + with outside_mirrored_call(): + return self._failure_handler_body( + exception=exception, + traceback_exception=traceback_exception, + start_time=start_time, + end_time=end_time, + for_async_request=for_async_request, + ) finally: self._restore_correlation_context() @@ -4465,12 +4474,13 @@ class Logging(LiteLLMLoggingBaseClass): """Restores trace_id/session_id contextvars once this attempt's own failure logging (including any nested calls its callbacks trigger) is fully done.""" try: - return await self._async_failure_handler_body( - exception=exception, - traceback_exception=traceback_exception, - start_time=start_time, - end_time=end_time, - ) + with outside_mirrored_call(): + return await self._async_failure_handler_body( + exception=exception, + traceback_exception=traceback_exception, + start_time=start_time, + end_time=end_time, + ) finally: self._restore_correlation_context() @@ -6559,6 +6569,7 @@ class StandardLoggingPayloadSetup: start_time: dt_object | None = None, response_id: str | None = None, custom_llm_provider: str | None = None, + is_silent_experiment: bool = False, ) -> StandardLoggingMetadata: """ Clean and filter the metadata dictionary to include only the specified keys in StandardLoggingMetadata. @@ -6632,6 +6643,7 @@ class StandardLoggingPayloadSetup: proxy_stamped_used_client_oauth_token(metadata, litellm_params), custom_llm_provider, ), + **(SilentExperimentFlag(is_silent_experiment=True) if is_silent_experiment else SilentExperimentFlag()), ) if isinstance(metadata, dict): for key in metadata.keys() & _STANDARD_LOGGING_METADATA_KEYS: @@ -7432,6 +7444,7 @@ def get_standard_logging_object_payload( start_time=start_time, response_id=id, custom_llm_provider=selected_provider if isinstance(selected_provider, str) else None, + is_silent_experiment=logging_obj.is_silent_experiment is True, ) _request_body: Final = proxy_server_request.get("body", {}) end_user_id: Final = clean_metadata["user_api_key_end_user_id"] or _request_body.get( @@ -7716,9 +7729,8 @@ def get_standard_logging_metadata( ) if isinstance(metadata, dict): # Update the clean_metadata with values from input metadata that match StandardLoggingMetadata fields - for key in StandardLoggingMetadata.__annotations__: - if key in metadata: - clean_metadata[key] = metadata[key] + for key in (metadata.keys() & StandardLoggingMetadata.__annotations__.keys()) - {"is_silent_experiment"}: + clean_metadata[key] = metadata[key] if metadata.get("user_api_key") is not None: if is_valid_sha256_hash(str(metadata.get("user_api_key"))): diff --git a/litellm/litellm_core_utils/llm_request_utils.py b/litellm/litellm_core_utils/llm_request_utils.py index 0bf59a6d1f2..db881aa16c6 100644 --- a/litellm/litellm_core_utils/llm_request_utils.py +++ b/litellm/litellm_core_utils/llm_request_utils.py @@ -1,8 +1,9 @@ -from collections.abc import Mapping +from collections.abc import Iterable, Iterator, Mapping from typing import Final, cast import litellm from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH +from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider def _form_field_value(value: object) -> str: @@ -153,24 +154,33 @@ def pick_cheapest_chat_models_from_llm_provider(custom_llm_provider: str, n=1): if custom_llm_provider not in litellm.models_by_provider: return [] - known_models: Final = litellm.models_by_provider.get(custom_llm_provider, []) - model_costs: Final = [] + known_models: Final[Iterable[str]] = litellm.models_by_provider.get(custom_llm_provider, ()) + ranked_rows: Final = ( + (route_rank, cost, model) + for model, cost in _chat_row_costs(known_models, custom_llm_provider) + if (route_rank := _route_rank(model, custom_llm_provider)) is not None + ) + return [model for _, _, model in sorted(ranked_rows, key=lambda row: row[:2])[:n]] - for model in known_models: + +def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float]]: + for model in models: try: model_info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) except Exception: continue if model_info.get("mode") != "chat": continue - _cost = (model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0) - model_costs.append((model, _cost)) + yield model, (model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0) - # Sort by cost (ascending) - model_costs.sort(key=lambda x: x[1]) - # Return the top n cheapest models - return [model for model, _ in model_costs[:n]] +def _route_rank(model: str, custom_llm_provider: str) -> int | None: + routed_provider: Final = inferred_provider(model) + if routed_provider is None: + return 1 + if routed_provider == inferred_provider(f"{custom_llm_provider}/{model}"): + return 0 + return None def get_proxy_server_request_headers(litellm_params: dict | None) -> dict: diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 8b94c34942c..de911beabdd 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -19,6 +19,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.llms.ollama.common_utils import OllamaResponseIdNormalizer from litellm.types.llms.databricks import DatabricksTool from litellm.types.llms.openai import ( + ChatCompletionAnnotation, ChatCompletionReasoningItem, ChatCompletionThinkingBlock, ImageURLListItem, @@ -59,6 +60,31 @@ _CHOICES_FIELDS: Final[frozenset] = frozenset(Choices.model_fields.keys()) _MODEL_RESPONSE_FIELDS: Final[frozenset] = frozenset(ModelResponse.model_fields.keys()) | {"usage"} +_ANNOTATION_ENTRIES: Final = TypeAdapter(tuple[object, ...]) + + +def _is_provider_specific_annotation(annotation: object) -> bool: + match annotation: + case {"type": "url_citation"}: + return False + case {"type": _}: + return True + case _: + return False + + +def _annotation_entries(annotations: object) -> tuple[object, ...]: + return _ANNOTATION_ENTRIES.validate_python(annotations) if isinstance(annotations, list) else () + + +def partition_annotations(annotations: object) -> tuple[object, tuple[object, ...]]: + entries: Final = _annotation_entries(annotations) + others: Final = tuple(annotation for annotation in entries if _is_provider_specific_annotation(annotation)) + if not others: + return annotations, () + return [annotation for annotation in entries if not _is_provider_specific_annotation(annotation)] or None, others + + def _normalize_images_for_message( images: list[dict] | None, ) -> list[ImageURLListItem] | None: @@ -787,6 +813,9 @@ def convert_to_model_response_object( provider_specific_fields = dict(choice["message"].get("provider_specific_fields", None) or {}) for f in choice["message"].keys() - _MESSAGE_FIELDS: provider_specific_fields[f] = choice["message"][f] + annotations, other_annotations = partition_annotations(choice["message"].get("annotations", None)) + if other_annotations: + provider_specific_fields["annotations"] = list(other_annotations) # Handle reasoning models that display `reasoning_content` within `content` reasoning_content, content = extract_reasoning_content(choice["message"]) @@ -809,7 +838,9 @@ def convert_to_model_response_object( reasoning_content=reasoning_content, thinking_blocks=thinking_blocks, reasoning_items=choice["message"].get("reasoning_items", None), - annotations=choice["message"].get("annotations", None), + annotations=cast( # cast-ok: provider annotation dicts pass through whole, keeping keys such as url_citation.content + list[ChatCompletionAnnotation] | None, annotations + ), images=_normalize_images_for_message(choice["message"].get("images", None)), ) finish_reason = choice.get("finish_reason", None) diff --git a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py index ace57a64aae..051a8fdf256 100644 --- a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py +++ b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py @@ -27,6 +27,15 @@ def _timing_window_start( return start_time, False +def response_ms_since_logging_start(logging_obj: LiteLLMLoggingObject) -> float | None: + reported_ms: Final = logging_obj.response_timing_metrics.get("_response_ms") + if reported_ms is None: + return None + logging_start: Final = logging_obj.start_time + window_start: Final = _timing_window_start(logging_start, logging_obj)[0] + return reported_ms - max(0.0, logging_start.timestamp() - window_start.timestamp()) * 1000 + + def _union_duration_ms(windows: object, lower: float, upper: float) -> float | None: if not isinstance(windows, (list, tuple)): return None diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 101f069c0f8..b3f89b33d1d 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -17,7 +17,7 @@ from pathlib import Path from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError import litellm from litellm import verbose_logger @@ -424,6 +424,109 @@ def anthropic_image_source_to_openai_url(image_source: Mapping[str, object]) -> return None +@dataclass(frozen=True, slots=True) +class InlineDocument: + filename: str + file_data: str + + +def _document_media_type(source: object) -> str: + match source: + case {"media_type": str() as media_type} if media_type: + return media_type + case _: + return "application/pdf" + + +def anthropic_inline_document(document: Mapping[str, object]) -> InlineDocument | None: + source: Final = document.get("source") + match source: + case {"type": "base64", "data": str() as data} if data: + title: Final = document.get("title") + return InlineDocument( + filename=title if isinstance(title, str) and title else "document.pdf", + file_data=f"data:{_document_media_type(source)};base64,{data}", + ) + case _: + return None + + +def is_pdf_file_part(part: object) -> bool: + match part: + case {"type": "file", "file": {"file_data": str() as file_data}} if file_data.startswith("data:"): + return file_data.startswith("data:application/pdf") + case {"type": "file", "file": {"format": str() as file_format}}: + return file_format == "application/pdf" + case {"type": "file", "file": {"filename": str() as filename}}: + return filename.lower().endswith(".pdf") + case _: + return False + + +_CONTENT_PARTS: Final = TypeAdapter(tuple[object, ...]) + + +def content_parts(message: Mapping[str, object]) -> Sequence[object]: + content: Final = message.get("content") + return _CONTENT_PARTS.validate_python(content) if isinstance(content, list) else () + + +def _has_pdf_file_part(message: Mapping[str, object]) -> bool: + return any(is_pdf_file_part(part) for part in content_parts(message)) + + +def messages_have_pdf_file_part(messages: Sequence[Mapping[str, object]]) -> bool: + return any(_has_pdf_file_part(message) for message in messages) + + +def _message_without_pdf_file_parts(message: AllMessageValues) -> AllMessageValues: + if not _has_pdf_file_part(message): + return message + kept: Final = [part for part in content_parts(message) if not is_pdf_file_part(part)] + return cast(AllMessageValues, {**message, "content": kept}) # cast-ok: same message with fewer content parts + + +def _is_pdf_only_user_turn(message: AllMessageValues) -> bool: + parts: Final = content_parts(message) + return message["role"] == "user" and bool(parts) and all(is_pdf_file_part(part) for part in parts) + + +def messages_without_pdf_file_parts(messages: Sequence[AllMessageValues]) -> tuple[AllMessageValues, ...]: + return tuple( + _message_without_pdf_file_parts(message) for message in messages if not _is_pdf_only_user_turn(message) + ) + + +_CONTENT_PART: Final = TypeAdapter(Mapping[str, object]) + + +def _file_part_as_image_url(part: object) -> object: + match part: + case {"type": "file", "file": {"file_data": str() as file_data}} if file_data: + fields: Final = _CONTENT_PART.validate_python(part) + return {key: value for key, value in fields.items() if key != "file"} | { + "type": "image_url", + "image_url": {"url": file_data}, + } + case _: + return part + + +def message_with_file_parts_as_image_urls(message: Mapping[str, object]) -> Mapping[str, object]: + parts: Final = content_parts(message) + if not parts: + return message + return {**message, "content": [_file_part_as_image_url(part) for part in parts]} + + +def _typed_message_with_file_parts_as_image_urls(message: AllMessageValues) -> AllMessageValues: + return cast(AllMessageValues, message_with_file_parts_as_image_urls(message)) # cast-ok: same message, image parts + + +def messages_with_file_parts_as_image_urls(messages: Sequence[AllMessageValues]) -> tuple[AllMessageValues, ...]: + return tuple(_typed_message_with_file_parts_as_image_urls(message) for message in messages) + + def _image_part_url(part: Mapping[str, object]) -> str | None: """The image URL carried by one content part, whichever of the three dialects wrote it.""" part_type: Final = part.get("type") @@ -2073,6 +2176,19 @@ def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT: return cast(_MarkedT, marked) # cast-ok: same block shape as the input plus the marker key +_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(PromptCacheBreakpoint) + + +def prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object: + if marker is None or not drop_params: + return marker + try: + return _PROMPT_CACHE_BREAKPOINT.validate_python(marker) + except ValidationError: + verbose_logger.debug("Dropping malformed prompt_cache_breakpoint %r under drop_params", marker) + return None + + def cache_control_as_prompt_cache_breakpoint(block: _MarkedT) -> _MarkedT: """Anthropic's ``cache_control`` mark rewritten as OpenAI's ``prompt_cache_breakpoint`` on the same block.""" if block.get("cache_control") is None: diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 927caec3350..484bacdc72c 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -20,6 +20,7 @@ from litellm.constants import ( REALTIME_SESSION_SUCCESS_LOGGED_KEY, ) from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.llms.base_llm.realtime.transcription_protocol import RealtimeTranscriptionProtocolError from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeBackend from litellm.types.llms.openai import ( OpenAIRealtimeEvents, @@ -1281,24 +1282,29 @@ class RealTimeStreaming: verbose_logger.exception("Connection closed in backend to client send messages - %s", e) close: Final = backend_close_from(e) self._flush_unbilled_transcription_usage() - if self._backend_refused_session(close): - await self.log_backend_refusal(e) - else: - await self.log_messages() + await self._log_session_end(e if self._backend_refused_session(close) else None) return close except asyncio.CancelledError: self._flush_unbilled_transcription_usage() - await self.log_messages() + await self._log_session_end(None) raise except Exception as e: verbose_logger.exception("Error in backend to client send messages: %s", e) self._flush_unbilled_transcription_usage() - await self.log_messages() + await self._log_session_end(None) return BackendClose(code=1011, reason="proxy failed while relaying the upstream websocket") def _backend_refused_session(self, close: BackendClose) -> bool: return close.code != 1000 and not self._backend_sent_frames + async def _log_session_end(self, transport_refusal: Exception | None) -> None: + provider_refusal: Final = None if self.provider_config is None else self.provider_config.session_refusal() + refusal: Final = provider_refusal or transport_refusal + if refusal is None: + await self.log_messages() + return + await self.log_backend_refusal(refusal) + async def log_backend_refusal(self, error: Exception) -> None: if not self.logging_obj: return @@ -1691,7 +1697,13 @@ class RealTimeStreaming: # ``_maybe_send_guardrail_turn_detection_update`` would retry. if msg_type == "session.update" and self.provider_config is None: await self._send_session_gate_once() - sent = await self._send_to_backend(message) + try: + sent = await self._send_to_backend(message) + except RealtimeTranscriptionProtocolError as e: + await self.websocket.send_text(realtime_error_event(str(e), error_type="invalid_request_error")) + if self._uses_deferred_backend_setup(): + raise + continue if guardrail_turn_detection_injected and sent: self._guardrail_turn_detection_update_sent = True diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 27e59cc913d..05415cf8718 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -12,9 +12,14 @@ from typing_extensions import ReadOnly, Required import litellm from litellm._logging import verbose_logger -from litellm.litellm_core_utils.core_helpers import chunk_usage, provider_reported_usage_cost +from litellm.litellm_core_utils.core_helpers import ( + USAGE_COST_BILLED_PROVIDERS, + chunk_usage, + provider_reported_usage_cost, +) from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import BEDROCK_TRACE_FIELDS, object_items, object_mapping +from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import partition_annotations from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_content_from_model_response, # pyright: ignore[reportUnknownVariableType] # response param accepts an untyped dict ) @@ -1375,7 +1380,7 @@ def stream_chunk_builder_text_completion( return TextCompletionResponse(**response) -_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value}) +_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value}) | USAGE_COST_BILLED_PROVIDERS def _logged_provider(logging_obj: Optional["Logging"]) -> object: @@ -1645,14 +1650,15 @@ def _assembled_message_fields(processor: ChunkProcessor, chunks: Sequence) -> It and chunk["choices"][0]["delta"]["annotations"] is not None ] - if annotation_chunks: - # Merge annotations from ALL chunks — providers may spread - # them across multiple streaming chunks or send them only in - # the final chunk. - all_annotations: Final[list[object]] = [] - for ac in annotation_chunks: - all_annotations.extend(ac["choices"][0]["delta"]["annotations"]) - yield "annotations", all_annotations + message_annotations, provider_annotations = ( + partition_annotations( + list(chain.from_iterable(chunk["choices"][0]["delta"]["annotations"] for chunk in annotation_chunks)) + ) + if annotation_chunks + else (None, ()) + ) + if message_annotations is not None: + yield "annotations", message_annotations audio_chunks: Final = [ chunk @@ -1692,28 +1698,26 @@ def _assembled_message_fields(processor: ChunkProcessor, chunks: Sequence) -> It and chunk["choices"][0]["delta"]["provider_specific_fields"] is not None ] - if provider_specific_chunks: - provider_field_dicts: Final = tuple( - fields - for chunk in provider_specific_chunks - for fields in (chunk["choices"][0]["delta"]["provider_specific_fields"],) - if isinstance(fields, dict) - ) - streamed_citations: Final = tuple( - fields["citation"] for fields in provider_field_dicts if fields.get("citation") is not None - ) - citation_fields: Final = ( - {"citations": _joined_streamed_citations(streamed_citations)} if streamed_citations else {} - ) - combined_provider_fields: Final = { - key: value - for fields in (citation_fields, *provider_field_dicts) - for key, value in fields.items() - if key != "citation" - } + provider_field_dicts: Final = tuple( + fields + for chunk in provider_specific_chunks + for fields in (chunk["choices"][0]["delta"]["provider_specific_fields"],) + if isinstance(fields, dict) + ) + streamed_citations: Final = tuple( + fields["citation"] for fields in provider_field_dicts if fields.get("citation") is not None + ) + citation_fields: Final = {"citations": _joined_streamed_citations(streamed_citations)} if streamed_citations else {} + provider_annotation_fields: Final = {"annotations": list(provider_annotations)} if provider_annotations else {} + combined_provider_fields: Final = { + key: value + for fields in (provider_annotation_fields, citation_fields, *provider_field_dicts) + for key, value in fields.items() + if key != "citation" + } - if combined_provider_fields: - yield "provider_specific_fields", combined_provider_fields + if combined_provider_fields: + yield "provider_specific_fields", combined_provider_fields def stream_chunk_builder( diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index f2f06d801a8..5542249d2c7 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -1938,6 +1938,16 @@ class CustomStreamWrapper: model_response: Final = self.model_response_creator( hidden_params=None if self.received_model_id is None else {"received_model_id": self.received_model_id} ) + served_tier: Final = next( + ( + tier + for chunk in reversed(self.chunks) + if isinstance(tier := getattr(chunk, "service_tier", None), str) and tier + ), + None, + ) + if served_tier is not None: + setattr(model_response, "service_tier", served_tier) # noqa: B010 # pydantic extra, not a declared field _finish_reason: Final = self.received_finish_reason or self.intermittent_finish_reason if _finish_reason is not None: model_response.choices[0].finish_reason = _finish_reason diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 2fe840dc863..591d77bdb16 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -611,7 +611,7 @@ T_ParamSpec: Final = ParamSpec("T_ParamSpec") T_Retval = TypeVar("T_Retval") _COUNT_OFFLOAD_LIMITER: Final = anyio.lowlevel.RunVar[anyio.CapacityLimiter]("litellm_count_offload_limiter") _ANTHROPIC_TOOL_KEYS_MAPPED_BY_BRIDGE: Final = frozenset( - ("name", "input_schema", "description", "cache_control", "strict", "type", "eager_input_streaming") + ("name", "input_schema", "description", "cache_control", "strict", "type", "eager_input_streaming", "custom") ) diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index d371aab4581..31b0f06251d 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -432,27 +432,30 @@ class AnthropicMessagesHandler(BaseTranslation): message, _ = serialize_http_exception_detail(exc.detail) return tuple(anthropic_sse_error_frames(message)) - def _standalone_block_chunks(self, exc: "ModifyResponseException") -> list[bytes]: + def build_block_response(self, exc: "ModifyResponseException") -> "AnthropicMessagesResponse": import uuid - from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( - FakeAnthropicMessagesStreamIterator, - ) from litellm.llms.base_llm.guardrail_translation.utils import ( blocked_response_usage, ) from litellm.types.utils import AnthropicMessagesResponse - block_response: Final = AnthropicMessagesResponse( + return AnthropicMessagesResponse( id=f"msg_{uuid.uuid4()}", type="message", role="assistant", content=[{"type": "text", "text": exc.message}], model=exc.model, stop_reason="end_turn", - usage=blocked_response_usage(getattr(exc, "original_response", None)), + usage=blocked_response_usage(exc.original_response), ) - return list(FakeAnthropicMessagesStreamIterator(response=block_response)) + + def _standalone_block_chunks(self, exc: "ModifyResponseException") -> list[bytes]: + from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( + FakeAnthropicMessagesStreamIterator, + ) + + return list(FakeAnthropicMessagesStreamIterator(response=self.build_block_response(exc))) def _block_continuation_chunks( self, exc: "ModifyResponseException", responses_so_far: Sequence[object] diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 8a82ed98991..e1436473426 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -17,6 +17,7 @@ import litellm import litellm.litellm_core_utils import litellm.types import litellm.types.utils +from litellm.anthropic_beta_gated_fields import strip_beta_gated_request_body from litellm.anthropic_beta_headers_manager import ( update_request_with_filtered_beta, ) @@ -506,11 +507,20 @@ class AnthropicChatCompletion(BaseLLM): The pair stays mutable because the streaming path rewrites it in place (`data["stream"] = True`) before sending. """ - request_headers, data = update_request_with_filtered_beta( + request_headers, filtered_data = update_request_with_filtered_beta( headers=headers, request_data=request_data, provider=custom_llm_provider, ) + data: Final = dict( + strip_beta_gated_request_body( + filtered_data, + request_headers, + drop_params=litellm_params.get("drop_params"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # litellm_params is the handler's bare dict + model=model, + provider=custom_llm_provider, + ) + ) logging_obj.pre_call( input=messages, api_key=api_key, diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 92e7c84031c..865247afc63 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -12,6 +12,7 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm.constants import ( + ANTHROPIC_BILLING_METADATA_PREFIX, ANTHROPIC_MIN_THINKING_BUDGET_TOKENS, ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES, DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS, @@ -2066,7 +2067,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if not system_message_block["content"]: continue if self.should_strip_billing_metadata() and system_message_block["content"].startswith( - "x-anthropic-billing-header:" + ANTHROPIC_BILLING_METADATA_PREFIX ): continue anthropic_system_message_content = AnthropicSystemMessageContent( @@ -2086,7 +2087,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): self.should_strip_billing_metadata() and _content.get("type") == "text" and text_value - and text_value.startswith("x-anthropic-billing-header:") + and text_value.startswith(ANTHROPIC_BILLING_METADATA_PREFIX) ): continue anthropic_system_message_content = AnthropicSystemMessageContent( diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 1f5e4e7de39..3b7fcdea0e4 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -51,6 +51,7 @@ from litellm.types.llms.anthropic import ( ANTHROPIC_OAUTH_BETA_HEADER, ANTHROPIC_OAUTH_TOKEN_PREFIX, ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX, + ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER, ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, ANTHROPIC_TOOL_SEARCH_TOOL_TYPES, AllAnthropicToolsValues, @@ -121,6 +122,54 @@ _DECODED_JSON: Final = TypeAdapter(object) _CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/") +def is_claude_tool_target(custom_llm_provider: str | None, model: str) -> bool: + if custom_llm_provider == "anthropic": + return True + model_lower: Final = model.lower() + if custom_llm_provider == "bedrock": + return "claude" in model_lower or ("arn:" in model_lower and ":bedrock:" in model_lower) + if custom_llm_provider == "azure_ai" or custom_llm_provider == "vertex_ai": + return "claude" in model_lower + return False + + +def is_claude_model_target(custom_llm_provider: str | None, model: str) -> bool: + return "claude" in model.lower() and is_claude_tool_target(custom_llm_provider, model) + + +def _hoistable_defer_loading(tool_dict: Mapping[object, object], custom: object) -> bool | None: + if not isinstance(custom, dict) or "defer_loading" in tool_dict: + return None + defer_loading: Final = OBJECT_MAPPING_ADAPTER.validate_python(custom).get("defer_loading") + return defer_loading if isinstance(defer_loading, bool) else None + + +def _tool_without_custom_field(tool: object, hoist_defer_loading: bool, drop_custom: bool) -> object: + if not isinstance(tool, dict): + return tool + + if "custom" not in tool: + return tool + + tool_dict: Final = OBJECT_MAPPING_ADAPTER.validate_python(tool) + + custom: Final = tool_dict["custom"] + without_custom: Final = {key: value for key, value in tool_dict.items() if key != "custom"} + defer_loading: Final = _hoistable_defer_loading(tool_dict, custom) if hoist_defer_loading else None + if defer_loading is None: + return without_custom if drop_custom else tool_dict + + unhoisted: Final = { + key: value for key, value in OBJECT_MAPPING_ADAPTER.validate_python(custom).items() if key != "defer_loading" + } + kept_custom: Final = {"custom": unhoisted} if unhoisted and not drop_custom else {} + return {**without_custom, "defer_loading": defer_loading, **kept_custom} + + +def tools_without_custom_field(tools: Sequence[object], hoist_defer_loading: bool, drop_custom: bool) -> list[object]: + return [_tool_without_custom_field(tool, hoist_defer_loading, drop_custom) for tool in tools] + + def is_anthropic_messages_url(url: str) -> bool: """Check whether a URL addresses Anthropic's Messages API.""" parsed_url: Final = urlparse(url) @@ -552,6 +601,20 @@ class AnthropicModelInfo(BaseLLMModelInfo): return False return thinking.get("type") in ("adaptive", "enabled") and thinking.get("display") == "updates" + def thinking_betas(self, thinking: AnthropicThinkingParam | None) -> tuple[str, ...]: + return ( + *( + (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,) + if self.is_thinking_display_updates_used(thinking) + else () + ), + *( + (ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER,) + if isinstance(thinking, dict) and "block_binding" in thinking + else () + ), + ) + def is_mid_conversation_tool_change_used(self, messages: Sequence[object]) -> bool: for message in messages: if message_field(message, "role") != "system": @@ -1132,7 +1195,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): *, custom_llm_provider: str, is_mid_conversation_output_config_used: bool = False, - is_thinking_display_updates_used: bool = False, + thinking: AnthropicThinkingParam | None = None, is_mid_conversation_tool_change_used: bool = False, ) -> list[str]: """ @@ -1169,13 +1232,10 @@ class AnthropicModelInfo(BaseLLMModelInfo): if is_mid_conversation_output_config_used: betas.append(ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER) - thinking_display_betas: Final = ( - (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,) if is_thinking_display_updates_used else () - ) tool_change_betas: Final = ( (ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,) if is_mid_conversation_tool_change_used else () ) - return list(set(betas).union(thinking_display_betas, tool_change_betas)) + return list(set(betas).union(self.thinking_betas(thinking), tool_change_betas)) @staticmethod def _make_api_key_auth_header( @@ -1237,7 +1297,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): api_base: str | None = None, use_bearer_for_custom_base: bool = False, is_mid_conversation_output_config_used: bool = False, - is_thinking_display_updates_used: bool = False, + thinking: AnthropicThinkingParam | None = None, is_mid_conversation_tool_change_used: bool = False, wif_minted: bool = False, ) -> dict: @@ -1298,7 +1358,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): betas.update(user_anthropic_beta_headers) all_betas: Final = betas.union( - (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,) if is_thinking_display_updates_used else (), + self.thinking_betas(thinking), (ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,) if is_mid_conversation_tool_change_used else (), ) @@ -1385,7 +1445,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): auth_token=auth_token, file_id_used=file_id_used, is_mid_conversation_output_config_used=is_mid_conversation_output_config_used, - is_thinking_display_updates_used=self.is_thinking_display_updates_used(optional_params.get("thinking")), + thinking=optional_params.get("thinking"), is_mid_conversation_tool_change_used=self.is_mid_conversation_tool_change_used(messages), web_search_tool_used=web_search_tool_used, is_vertex_request=optional_params.get("is_vertex_request", False), diff --git a/litellm/llms/anthropic/count_tokens/handler.py b/litellm/llms/anthropic/count_tokens/handler.py index a230294ad3f..f8bf7232d9c 100644 --- a/litellm/llms/anthropic/count_tokens/handler.py +++ b/litellm/llms/anthropic/count_tokens/handler.py @@ -39,6 +39,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): tools: list[dict[str, JsonValue]] | None = None, system: JsonValue = None, optional_params: Mapping[str, JsonValue] | None = None, + drop_params: bool = False, ) -> dict[str, JsonValue]: """ Handle a CountTokens request using httpx. @@ -69,6 +70,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): tools=tools, system=system, optional_params=optional_params, + drop_params=drop_params, ) verbose_logger.debug("Transformed request: %s", request_body) diff --git a/litellm/llms/anthropic/count_tokens/token_counter.py b/litellm/llms/anthropic/count_tokens/token_counter.py index 368bb929eeb..91fc77773d1 100644 --- a/litellm/llms/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/anthropic/count_tokens/token_counter.py @@ -11,6 +11,7 @@ from pydantic import JsonValue import litellm from litellm._logging import verbose_logger from litellm.exceptions import AuthenticationError +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler from litellm.llms.anthropic.count_tokens.transformation import ( AnthropicCountFields, @@ -131,6 +132,7 @@ class AnthropicTokenCounter(BaseTokenCounter): tools=tools, system=system, optional_params=count_options, + drop_params=drops_unsupported_params(litellm_params.get("drop_params")), ) if result is not None: diff --git a/litellm/llms/anthropic/count_tokens/transformation.py b/litellm/llms/anthropic/count_tokens/transformation.py index 0967c22f93a..e60ffe67c8d 100644 --- a/litellm/llms/anthropic/count_tokens/transformation.py +++ b/litellm/llms/anthropic/count_tokens/transformation.py @@ -26,7 +26,11 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import split_leading_system_run, ) from litellm.litellm_core_utils.url_utils import join_api_base -from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers +from litellm.llms.anthropic.common_utils import ( + is_claude_model_target, + merge_anthropic_beta_headers, + tools_without_custom_field, +) from litellm.llms.anthropic.wif import resolve_anthropic_base from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse @@ -372,6 +376,7 @@ class AnthropicCountTokensConfig: tools: list[dict[str, JsonValue]] | None = None, system: JsonValue = None, optional_params: Mapping[str, JsonValue] | None = None, + drop_params: bool = False, ) -> dict[str, JsonValue]: # mutable-ok: provider transport requires JSON dictionaries """ Transform request to Anthropic CountTokens format. @@ -391,7 +396,19 @@ class AnthropicCountTokensConfig: **MappingProxyType( { key: value - for key, value in (("system", counted_system), ("tools", tools)) + for key, value in ( + ("system", counted_system), + ( + "tools", + None + if tools is None + else tools_without_custom_field( + tools, + hoist_defer_loading=is_claude_model_target("anthropic", model), + drop_custom=drop_params, + ), + ), + ) if value is not None } ), diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index 1ed552d46e3..ddda759384f 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -123,6 +123,7 @@ from openai.types.chat.chat_completion_chunk import Choice as OpenAIStreamingCho from litellm.litellm_core_utils.prompt_templates.common_utils import ( anthropic_image_source_to_openai_url, + anthropic_inline_document, parse_tool_call_arguments, reasoning_content_from_thinking_blocks, with_prompt_cache_breakpoint, @@ -185,6 +186,8 @@ from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionAssistantMessage, ChatCompletionAssistantToolCall, + ChatCompletionFileObject, + ChatCompletionFileObjectFile, ChatCompletionImageObject, ChatCompletionImageUrlObject, ChatCompletionRedactedThinkingBlock, @@ -203,7 +206,7 @@ from litellm.types.llms.openai import ( ChatCompletionUserMessage, ToolMessageContentPart, ) -from litellm.types.utils import Choices, ModelResponse, StreamingChoices, Usage +from litellm.types.utils import Choices, LlmProviders, ModelResponse, StreamingChoices, Usage from litellm.utils import supports_mid_conversation_system from .streaming_iterator import AnthropicSSEStream, AnthropicStreamWrapper @@ -362,6 +365,9 @@ class AnthropicAdapter: _BlockT: Final = TypeVar("_BlockT", bound=Mapping[str, object]) +_CLAUDE_OVER_REASONING_EFFORT_PROVIDERS: Final = frozenset({LlmProviders.KYMA.value}) + + class LiteLLMAnthropicMessagesAdapter: def __init__(self): pass @@ -495,7 +501,9 @@ class LiteLLMAnthropicMessagesAdapter: for m in ordered_messages: user_message: ChatCompletionUserMessage | None = None tool_message_list: list[ChatCompletionToolMessage] = [] - new_user_content_list: list[ChatCompletionTextObject | ChatCompletionImageObject] = [] + new_user_content_list: list[ + ChatCompletionTextObject | ChatCompletionImageObject | ChatCompletionFileObject + ] = [] if m["role"] == "system": system_message = self._translate_midturn_system_message_to_openai(m, model) if system_message is not None: @@ -528,13 +536,8 @@ class LiteLLMAnthropicMessagesAdapter: self._add_prompt_cache_breakpoint_if_present(content, image_obj) ) elif content.get("type") == "document": - # Convert Anthropic document format (PDF, etc.) to OpenAI format - source = content.get("source", {}) - openai_image_url = self._translate_anthropic_image_to_openai(cast(dict, source)) - - if openai_image_url: - image_url_obj = ChatCompletionImageUrlObject(url=openai_image_url) - doc_obj = ChatCompletionImageObject(type="image_url", image_url=image_url_obj) + doc_obj = self._translate_anthropic_document_to_openai(content) + if doc_obj is not None: self._add_cache_control_if_applicable(content, doc_obj, model) new_user_content_list.append(doc_obj) elif content.get("type") == "tool_result": @@ -817,6 +820,7 @@ class LiteLLMAnthropicMessagesAdapter: "name", "input_schema", "description", + "custom", "cache_control", "strict", "type", @@ -1127,7 +1131,8 @@ class LiteLLMAnthropicMessagesAdapter: """Translate Anthropic thinking to either thinking or reasoning_effort. A Claude-family target keeps ``thinking`` verbatim, since every bridged provider serving one - speaks that param. Carrying its adaptive effort tier alongside takes two different params, + speaks that param, except Kyma, which serves Claude over the OpenAI dialect and takes only + ``reasoning_effort``, so its Claude models get the non-Claude mapping. Carrying its adaptive effort tier alongside takes two different params, because the two are not interchangeable at the provider mapping below. Bedrock takes ``output_config`` directly, which attaches the tier and leaves ``thinking`` @@ -1158,7 +1163,9 @@ class LiteLLMAnthropicMessagesAdapter: is_bedrock_target: Final = model.startswith(("bedrock/", "converse/", "invoke/")) or self.is_bedrock_arn_model( model ) - is_claude_target: Final = self.is_anthropic_claude_model(model) or self.is_bedrock_arn_model(model) + is_claude_target: Final = ( + self.is_anthropic_claude_model(model) or self.is_bedrock_arn_model(model) + ) and custom_llm_provider not in _CLAUDE_OVER_REASONING_EFFORT_PROVIDERS output_config: Final = anthropic_message_request.get("output_config") if is_claude_target: @@ -1350,7 +1357,7 @@ class LiteLLMAnthropicMessagesAdapter: text: Final = tool_result_text_part(block) return None if text is None else ChatCompletionTextObject(type="text", text=text) case "image" | "document": - return self._tool_result_image_part(block.get("source")) + return self._source_image_url_part(block.get("source")) case "tool_reference": return ChatCompletionToolReferenceObject( type="tool_reference", tool_name=str(block.get("tool_name") or "") @@ -1358,7 +1365,18 @@ class LiteLLMAnthropicMessagesAdapter: case _: return None - def _tool_result_image_part(self, image_source: object) -> ChatCompletionImageObject | None: + def _translate_anthropic_document_to_openai( + self, document: Mapping[str, object] + ) -> ChatCompletionFileObject | ChatCompletionImageObject | None: + inline_document: Final = anthropic_inline_document(document) + if inline_document is None: + return self._source_image_url_part(document.get("source")) + return ChatCompletionFileObject( + type="file", + file=ChatCompletionFileObjectFile(filename=inline_document.filename, file_data=inline_document.file_data), + ) + + def _source_image_url_part(self, image_source: object) -> ChatCompletionImageObject | None: if not isinstance(image_source, dict): return None openai_image_url = self._translate_anthropic_image_to_openai(image_source) diff --git a/litellm/llms/anthropic/messages/native/transformation.py b/litellm/llms/anthropic/messages/native/transformation.py index cc53858f842..c086618e829 100644 --- a/litellm/llms/anthropic/messages/native/transformation.py +++ b/litellm/llms/anthropic/messages/native/transformation.py @@ -2,10 +2,22 @@ from collections.abc import AsyncIterator, Mapping, Sequence from typing import ClassVar, Final import httpx +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_logger -from litellm.anthropic_beta_headers_manager import get_unsupported_headers, is_beta_header_supported_by_bundled_config +from litellm.anthropic_beta_gated_fields import ( + log_beta_gated_strips, + strip_beta_gated_native_request, + strip_beta_gated_system_messages, + unstripped_native_request, +) +from litellm.anthropic_beta_headers_manager import ( + get_unsupported_headers, + has_beta_headers_config, + is_beta_header_supported_by_bundled_config, +) +from litellm.constants import ANTHROPIC_BILLING_METADATA_PREFIX from litellm.exceptions import AuthenticationError from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -19,7 +31,6 @@ from litellm.types.llms.anthropic import ( ANTHROPIC_ADVISOR_TOOL_TYPE, ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER, - ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, AnthropicMessagesRequest, ) from litellm.types.llms.anthropic_messages.anthropic_response import ( @@ -29,13 +40,16 @@ from litellm.types.llms.anthropic_tool_search import get_tool_search_beta_header from litellm.types.router import GenericLiteLLMParams from ...common_utils import ( + OBJECT_LIST_ADAPTER, AnthropicError, AnthropicModelInfo, + is_claude_model_target, merge_anthropic_beta_headers, optionally_handle_anthropic_oauth, requires_native_compaction_beta, strip_advisor_blocks_from_messages, strip_encrypted_reasoning_blocks_from_anthropic_messages, + tools_without_custom_field, ) from .mid_conversation_system import ( as_system_content_blocks, @@ -47,6 +61,8 @@ DEFAULT_ANTHROPIC_API_VERSION: Final = "2023-06-01" _CALLER_CREDENTIAL_HEADERS: Final = frozenset({"x-api-key", "authorization"}) +_MESSAGE_MAPPINGS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) + def _carries_caller_credential(headers: Mapping[str, str]) -> bool: """Whether the caller sent their own Anthropic credential, in which case this passthrough @@ -81,6 +97,9 @@ def _ensure_stop_sequence(response: AnthropicMessagesResponse) -> AnthropicMessa return response +_MESSAGES_ADAPTER: Final = TypeAdapter(list[dict[str, object]]) + + class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): _workload_identity_eligible: ClassVar[bool] = True @@ -181,6 +200,16 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): """ return False + def hoists_wrapped_defer_loading(self, model: str) -> bool: + return is_claude_model_target(self._resolved_provider, model) + + def tools_for_upstream(self, tools: Sequence[object], model: str, drop_params: object) -> Sequence[object]: + return tools_without_custom_field( + tools, + hoist_defer_loading=self.hoists_wrapped_defer_loading(model), + drop_custom=drops_unsupported_params(drop_params), + ) + @staticmethod def _filter_billing_headers_from_system(system_param): """ @@ -194,7 +223,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): """ if isinstance(system_param, str): # If it's a string and starts with billing header, filter it out - if system_param.startswith("x-anthropic-billing-header:"): + if system_param.startswith(ANTHROPIC_BILLING_METADATA_PREFIX): return None return system_param elif isinstance(system_param, list): @@ -205,7 +234,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): text = content_block.get("text", "") content_type = content_block.get("type", "") # Skip text blocks that start with billing header - if content_type == "text" and text.startswith("x-anthropic-billing-header:"): + if content_type == "text" and text.startswith(ANTHROPIC_BILLING_METADATA_PREFIX): continue filtered_list.append(content_block) else: @@ -215,7 +244,18 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): else: return system_param - def _normalize_system_role_messages(self, anthropic_messages_request: dict, model: str) -> None: + def _strip_beta_gated_system_turns( + self, messages: Sequence[Mapping[str, object]], model: str, drop_params: object + ) -> Sequence[Mapping[str, object]]: + if not has_beta_headers_config(self._resolved_provider): + return messages + gated: Final = strip_beta_gated_system_messages(messages, frozenset(), drop_params=drop_params) + log_beta_gated_strips(gated.stripped, model=model, provider=self._resolved_provider) + return _MESSAGE_MAPPINGS.validate_python(gated.messages) if gated.stripped else messages + + def _normalize_system_role_messages( + self, anthropic_messages_request: dict, model: str, drop_params: object + ) -> None: """Normalize ``role: "system"`` entries in ``messages`` per the Anthropic ``/v1/messages`` contract, which the first-party API, Bedrock Invoke, Vertex, and Azure Foundry all enforce identically. @@ -263,7 +303,15 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): custom_llm_provider=self.custom_llm_provider, key="supports_mid_conversation_system", ) - else list(convert_mid_conversation_system_turns(messages[leading_count:])) + else list( + convert_mid_conversation_system_turns( + self._strip_beta_gated_system_turns( + messages[leading_count:], # pyright: ignore[reportUnknownArgumentType] # messages comes from the bare request dict + model=model, + drop_params=drop_params, + ) + ) + ) ) if hoisted or remaining != messages: anthropic_messages_request["messages"] = remaining @@ -322,7 +370,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): model=model, ), ) - return self._finalize_messages_headers(headers, optional_params, messages), api_base + return self.finalize_messages_headers(headers, optional_params, messages), api_base async def avalidate_anthropic_messages_environment( self, @@ -362,7 +410,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): model=model, ), ) - return self._finalize_messages_headers(oauth_headers, optional_params, messages), api_base + return self.finalize_messages_headers(oauth_headers, optional_params, messages), api_base def _require_auth_header(self, auth_header: Mapping[str, str] | None, model: str) -> Mapping[str, str]: if auth_header is None: @@ -396,7 +444,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): return config_allows_workload_identity(self) - def _finalize_messages_headers( + def finalize_messages_headers( self, headers: dict, # mutable-ok: out-param optional_params: dict, # mutable-ok: out-param @@ -605,6 +653,12 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): if thinking_enabled or effort_enabled: optional_params.pop("temperature", None) + def strips_beta_gated_fields_from_final_headers(self) -> bool: + """False on legs whose final ``anthropic_beta`` list is only known after this transform (Bedrock Invoke), + which strip beta-gated fields themselves once that list exists, and on providers with no row in + ``anthropic_beta_headers_config.json``, whose filter drops every beta and so cannot say which fields fail.""" + return has_beta_headers_config(self._resolved_provider) + def transform_anthropic_messages_request( self, model: str, @@ -700,20 +754,47 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): # Auto-strip advisor blocks from history if advisor tool is absent. # Prevents Anthropic 400: advisor_tool_result in history requires advisor tool. - _tools: Final = anthropic_messages_optional_request_params.get("tools") or [] + tools: Final = anthropic_messages_optional_request_params.get("tools") + _tools: Final = tools or [] _has_advisor: Final = any(isinstance(t, dict) and t.get("type") == ANTHROPIC_ADVISOR_TOOL_TYPE for t in _tools) if not _has_advisor: messages = strip_advisor_blocks_from_messages(messages) + gated: Final = ( + strip_beta_gated_native_request( + headers, # pyright: ignore[reportUnknownArgumentType] # the base signature types this as a bare dict + anthropic_messages_optional_request_params, # pyright: ignore[reportUnknownArgumentType] # the base signature types this as a bare dict + messages, # pyright: ignore[reportUnknownArgumentType] # the base signature types this as a bare list + drop_params=getattr(litellm_params, "drop_params", None), + ) + if self.strips_beta_gated_fields_from_final_headers() + else unstripped_native_request(anthropic_messages_optional_request_params, messages) # pyright: ignore[reportUnknownArgumentType] # the base signature types these as bare dicts + ) + log_beta_gated_strips(gated.stripped, model=model, provider=self._resolved_provider) anthropic_messages_request: Final[AnthropicMessagesRequest] = AnthropicMessagesRequest( messages=strip_encrypted_reasoning_blocks_from_anthropic_messages( - drop_non_string_tool_result_texts(messages, getattr(litellm_params, "drop_params", None)) + drop_non_string_tool_result_texts( + _MESSAGES_ADAPTER.validate_python(gated.messages) if gated.messages_stripped else messages, + getattr(litellm_params, "drop_params", None), + ) ), max_tokens=max_tokens, model=model, **anthropic_messages_optional_request_params, ) - return dict(anthropic_messages_request) + request: Final = ( + {**dict(anthropic_messages_request), "thinking": gated.thinking} + if gated.thinking_stripped + else dict(anthropic_messages_request) # pyright: ignore[reportUnknownVariableType] # built from the base signature's bare dict + ) + if not isinstance(tools, list): + return request + return { + **request, + "tools": self.tools_for_upstream( + OBJECT_LIST_ADAPTER.validate_python(tools), model, litellm_params.drop_params + ), + } def transform_anthropic_messages_response( self, @@ -844,17 +925,13 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): if AnthropicModelInfo().is_tool_search_used(tools): beta_values.add(get_tool_search_beta_header(custom_llm_provider)) - thinking_display_betas: Final = ( - (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,) - if AnthropicModelInfo().is_thinking_display_updates_used(optional_params.get("thinking")) - else () - ) + thinking_betas: Final = AnthropicModelInfo().thinking_betas(optional_params.get("thinking")) tool_change_betas: Final = ( (ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,) if AnthropicModelInfo().is_mid_conversation_tool_change_used(messages) else () ) - all_beta_values: Final = beta_values.union(thinking_display_betas, tool_change_betas) + all_beta_values: Final = beta_values.union(thinking_betas, tool_change_betas) dangerous_tool_use: Final = ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value forwarded_beta_values: Final = ( # pyright: ignore[reportUnknownVariableType] # derived from the bare beta set all_beta_values - {dangerous_tool_use} diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index 99a07a1e9bf..9b22170c9db 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -15,7 +15,11 @@ from litellm.integrations.anthropic_cache_control_hook import ( supports_openai_prompt_cache_breakpoint, ) from litellm.litellm_core_utils.core_helpers import drops_unsupported_params -from litellm.litellm_core_utils.prompt_templates.common_utils import cache_control_as_prompt_cache_breakpoint +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + cache_control_as_prompt_cache_breakpoint, + prompt_cache_breakpoint_for_wire, + with_prompt_cache_breakpoint, +) from litellm.responses.utils import auto_reasoning_summary from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS from litellm.types.llms.anthropic import ( @@ -90,14 +94,28 @@ def _without_prompt_cache_breakpoint(block: Mapping[str, object]) -> Mapping[str return {key: value for key, value in block.items() if key != "prompt_cache_breakpoint"} +def _cache_control_as_wire_safe_prompt_cache_breakpoint(block: Mapping[str, object]) -> Mapping[str, object]: + if "prompt_cache_breakpoint" not in block: + return cache_control_as_prompt_cache_breakpoint(block) + wire_marker: Final = prompt_cache_breakpoint_for_wire(block["prompt_cache_breakpoint"], drop_params=True) + return cache_control_as_prompt_cache_breakpoint( + with_prompt_cache_breakpoint(_without_prompt_cache_breakpoint(block), wire_marker) + ) + + def _prompt_cache_block_rewrite(model: str, forwarded_kwargs: Mapping[str, object]) -> BlockRewrite | None: """Client cache marks become breakpoints for a deployment that reads them, and a model that takes no breakpoints drops raw ones under drop_params.""" + drops_params: Final = drops_unsupported_params(forwarded_kwargs.get("drop_params")) if _reads_prompt_cache_breakpoint(model, forwarded_kwargs): - return cache_control_as_prompt_cache_breakpoint + return ( + _cache_control_as_wire_safe_prompt_cache_breakpoint + if drops_params + else cache_control_as_prompt_cache_breakpoint + ) if _model_takes_prompt_cache_breakpoint(model, forwarded_kwargs): return None - if drops_unsupported_params(forwarded_kwargs.get("drop_params")): + if drops_params: return _without_prompt_cache_breakpoint return None diff --git a/litellm/llms/anthropic/messages/responses_adapters/transformation.py b/litellm/llms/anthropic/messages/responses_adapters/transformation.py index 1dd07c1739b..94c608f90e6 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/messages/responses_adapters/transformation.py @@ -13,6 +13,7 @@ from typing import Any, Final, cast from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_BOUNDARY, TOOL_RESULT_IMAGE_PLACEHOLDER, + anthropic_inline_document, encrypted_reasoning_signature, responses_reasoning_items_from_thinking_blocks, with_prompt_cache_breakpoint, @@ -106,27 +107,18 @@ class LiteLLMAnthropicToResponsesAPIAdapter: block: Mapping[str, object], ) -> dict[str, str] | None: # mutable-ok: API message payload """Convert an Anthropic document block to a Responses input_file part.""" + inline_document: Final = anthropic_inline_document(block) + if inline_document is not None: + return { + "type": "input_file", + "filename": inline_document.filename, + "file_data": inline_document.file_data, + } raw_source: Final = block.get("source") if not isinstance(raw_source, Mapping): return None source: Final = cast(Mapping[str, object], raw_source) # cast-ok: untrusted client payload - source_type: Final = source.get("type") - if source_type == "base64": - data: Final = source.get("data") - if not isinstance(data, str) or not data: - return None - raw_media_type: Final = source.get("media_type") - media_type: Final = ( - raw_media_type if isinstance(raw_media_type, str) and raw_media_type else "application/pdf" - ) - raw_title: Final = block.get("title") - filename: Final = raw_title if isinstance(raw_title, str) and raw_title else "document.pdf" - return { - "type": "input_file", - "filename": filename, - "file_data": f"data:{media_type};base64,{data}", - } - if source_type == "url": + if source.get("type") == "url": url: Final = source.get("url") if not isinstance(url, str) or not url: return None diff --git a/litellm/llms/anthropic/prompt_cache_prediction.py b/litellm/llms/anthropic/prompt_cache_prediction.py index 17c178d169e..27e003ce8b6 100644 --- a/litellm/llms/anthropic/prompt_cache_prediction.py +++ b/litellm/llms/anthropic/prompt_cache_prediction.py @@ -661,12 +661,13 @@ def prepare_native_baseline_body(request: Mapping[str, object], model: str) -> M TypeAdapter(tuple[str, ...]).validate_python(owned.get("additional_drop_params") or ()), dict(options), ) - body: Final = AnthropicMessagesConfig().transform_anthropic_messages_request( + config: Final = AnthropicMessagesConfig() + body: Final = config.transform_anthropic_messages_request( model=resolved_model, messages=strip_provider_specific_fields_from_anthropic_messages(messages), anthropic_messages_optional_request_params=filtered, litellm_params=GenericLiteLLMParams(), - headers={}, + headers=config.finalize_messages_headers({}, dict(filtered), list(messages)), # pyright: ignore[reportUnknownMemberType] # the shared helper takes and returns bare dicts ) return MappingProxyType(_JSON_OBJECT.validate_python(body)) except Exception: # noqa: BLE001 # an unsupported hypothetical request is unavailable, never an inference failure diff --git a/litellm/llms/azure/chat/gpt_reasoning_transformation.py b/litellm/llms/azure/chat/gpt_reasoning_transformation.py index 5bd03f511c5..72a0500e0af 100644 --- a/litellm/llms/azure/chat/gpt_reasoning_transformation.py +++ b/litellm/llms/azure/chat/gpt_reasoning_transformation.py @@ -24,6 +24,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): GPT5_SERIES_ROUTE = "gpt5_series/" OPT_IN_REASONING_EFFORTS: ClassVar[frozenset[str]] = frozenset(("xhigh", "none")) + LOOKS_UP_BASE_MODEL: ClassVar[bool] = True @classmethod def _model_map_lookup_name(cls, model: str) -> str: @@ -88,7 +89,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): drop_params: bool, api_version: str = "", *, - declared_reasoning_efforts: Collection[str] = (), + declared_reasoning_efforts: Collection[str] | None = None, ) -> dict: return OpenAIGPT5Config.map_openai_params( self, diff --git a/litellm/llms/azure/chat/gpt_transformation.py b/litellm/llms/azure/chat/gpt_transformation.py index 26ae30635d4..990c8869de3 100644 --- a/litellm/llms/azure/chat/gpt_transformation.py +++ b/litellm/llms/azure/chat/gpt_transformation.py @@ -1,4 +1,4 @@ -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final @@ -15,6 +15,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.prompt_templates.factory import ( convert_to_azure_openai_messages, ) +from litellm.llms.base_llm.base_utils import map_developer_role_to_system_role from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.openai.chat.gpt_reasoning_transformation import GPT_REASONING_SERIES_MARKERS from litellm.types.llms.azure import ( @@ -98,6 +99,22 @@ class AzureOpenAIConfig(BaseConfig): if key != "self" and value is not None: setattr(self.__class__, key, value) + def translate_developer_role_to_system_role( + self, + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: + """ + Azure OpenAI accepts `system` messages at any position, so a developer message + only changes role. Declared here so `AzureOpenAIGPT5Config` resolves to it + ahead of the OpenAI-compatible hoist it also inherits. + """ + return map_developer_role_to_system_role(messages=messages) + @classmethod def get_config(cls): return super().get_config() diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index eb6d103efb9..09777da8669 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -335,7 +335,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, - ) -> tuple[str, dict]: + ) -> tuple[str, dict[str, str]]: """ Transform the get response API request into a URL and data @@ -343,7 +343,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): - GET /v1/responses/{response_id} """ get_url: Final = self._construct_url_for_response_id_in_path(api_base=api_base, response_id=response_id) - data: Final[dict] = {} + data: Final[dict[str, str]] = {} verbose_logger.debug("get response url=%s", get_url) return get_url, data diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py index 0f8afaae681..4b3f263201a 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py @@ -13,6 +13,7 @@ from pydantic import JsonValue import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import asyncify +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.llms.anthropic.common_utils import AnthropicError from litellm.llms.azure_ai.anthropic.count_tokens.transformation import ( AzureAIAnthropicCountTokensConfig, @@ -69,6 +70,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig): tools=tools, system=system, optional_params=optional_params, + drop_params=drops_unsupported_params((litellm_params or {}).get("drop_params")), ) verbose_logger.debug("Transformed request: %s", request_body) diff --git a/litellm/llms/azure_ai/anthropic/handler.py b/litellm/llms/azure_ai/anthropic/handler.py index ca4dad5ee3e..1a2003e6262 100644 --- a/litellm/llms/azure_ai/anthropic/handler.py +++ b/litellm/llms/azure_ai/anthropic/handler.py @@ -4,11 +4,12 @@ Azure Anthropic handler - reuses AnthropicChatCompletion logic with Azure authen import copy import json -from collections.abc import Callable +from collections.abc import Callable, Mapping from typing import Final import httpx +from litellm.anthropic_beta_gated_fields import strip_beta_gated_request_body from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.anthropic.chat.handler import AnthropicChatCompletion from litellm.llms.anthropic.chat.transformation import JsonModeRequest @@ -75,13 +76,22 @@ class AzureAnthropicChatCompletion(AnthropicChatCompletion): litellm_params=litellm_params, ) - data: Final = config.transform_request( + request_body: Final[Mapping[str, object]] = config.transform_request( model=model, messages=messages, optional_params=optional_params, litellm_params=litellm_params, headers=headers, ) + data: Final = dict( + strip_beta_gated_request_body( + request_body, + headers, # pyright: ignore[reportUnknownArgumentType] # validate_environment returns a bare dict + drop_params=litellm_params.get("drop_params"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # litellm_params is the handler's bare dict + model=model, + provider=custom_llm_provider, + ) + ) ## LOGGING logging_obj.pre_call( diff --git a/litellm/llms/azure_ai/anthropic/messages_transformation.py b/litellm/llms/azure_ai/anthropic/messages_transformation.py index 6b2da96de23..1f9ded827b6 100644 --- a/litellm/llms/azure_ai/anthropic/messages_transformation.py +++ b/litellm/llms/azure_ai/anthropic/messages_transformation.py @@ -145,6 +145,8 @@ class AzureAnthropicMessagesConfig(AnthropicMessagesConfig): litellm_params=litellm_params, headers=headers, ) - self._normalize_system_role_messages(anthropic_messages_request, model=model) + self._normalize_system_role_messages( + anthropic_messages_request, model=model, drop_params=getattr(litellm_params, "drop_params", None) + ) self._remove_scope_from_cache_control(anthropic_messages_request) return anthropic_messages_request diff --git a/litellm/llms/azure_ai/anthropic/transformation.py b/litellm/llms/azure_ai/anthropic/transformation.py index c75213c3e2c..6516827359d 100644 --- a/litellm/llms/azure_ai/anthropic/transformation.py +++ b/litellm/llms/azure_ai/anthropic/transformation.py @@ -107,9 +107,11 @@ class AzureAnthropicConfig(AnthropicConfig): is_vertex_request=optional_params.get("is_vertex_request", False), user_anthropic_beta_headers=user_anthropic_beta_headers, mcp_server_used=mcp_server_used, + thinking=optional_params.get("thinking"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # optional_params is the base config's bare dict ) # Merge headers - Azure auth (api-key or Authorization) takes precedence - headers = {**anthropic_headers, **headers} + merged_beta_header: Final = {k: v for k, v in anthropic_headers.items() if k == "anthropic-beta"} + headers = {**anthropic_headers, **headers, **merged_beta_header} # Ensure anthropic-version header is set if "anthropic-version" not in headers: diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 6aed7eb3c54..d382d653a53 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -1,7 +1,7 @@ import copy import enum import re -from collections.abc import AsyncIterator, Collection, Iterator +from collections.abc import AsyncIterator, Collection, Iterator, Mapping from typing import TYPE_CHECKING, Final, cast import httpx @@ -10,10 +10,11 @@ from httpx import Response import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.prompt_templates.common_utils import ( - audio_or_image_in_message_content, + content_parts, convert_content_list_to_str, drop_tool_reference_parts_from_tool_messages, filter_value_from_dict, + is_pdf_file_part, ) from litellm.llms.azure.common_utils import BaseAzureLLM from litellm.llms.azure_ai.common_utils import ( @@ -46,6 +47,20 @@ NON_OPENAI_SPEC_MESSAGE_FIELDS: Final = ( "cache_control", ) +LIST_ONLY_CONTENT_PART_TYPES: Final = frozenset({"image_url", "input_audio"}) + + +def _is_list_only_content_part(part: object) -> bool: + match part: + case {"type": str() as part_type} if part_type in LIST_ONLY_CONTENT_PART_TYPES: + return True + case _: + return is_pdf_file_part(part) + + +def _has_list_only_content_part(message: Mapping[str, object]) -> bool: + return any(_is_list_only_content_part(part) for part in content_parts(message)) + class AzureAIChatCompletionResponseIterator(OpenAIChatCompletionResponseIterator): def chunk_parser(self, chunk: dict[str, object]) -> ModelResponseStream: # mutable-ok: mirrors override contract @@ -106,7 +121,7 @@ class AzureAIStudioConfig(OpenAIConfig): model: str, drop_params: bool, *, - declared_reasoning_efforts: Collection[str] = (), + declared_reasoning_efforts: Collection[str] | None = None, ) -> dict[str, object]: # mutable-ok: OpenAIConfig.map_openai_params signature if not azureAIGPT5Config.is_model_gpt_5_model(model): return super().map_openai_params( @@ -246,7 +261,7 @@ class AzureAIStudioConfig(OpenAIConfig): these with "Extra inputs are not permitted", which breaks multi-turn Anthropic-format clients that echo thinking blocks back as history. 2. Transforms list content to a string. - 3. If message contains an image or audio, send as is (user-intended) + 3. If message contains an image, audio, or PDF file part, send as is (user-intended) Operates on a deep copy so the caller's messages keep their thinking blocks and provider metadata, which a fallback to another provider still needs. @@ -257,8 +272,7 @@ class AzureAIStudioConfig(OpenAIConfig): for field in NON_OPENAI_SPEC_MESSAGE_FIELDS: filter_value_from_dict(message_dict, field) - # Do nothing if the message contains an image or audio - if audio_or_image_in_message_content(message): + if _has_list_only_content_part(message): continue texts = convert_content_list_to_str(message=message) diff --git a/litellm/llms/azure_ai/passthrough/transformation.py b/litellm/llms/azure_ai/passthrough/transformation.py index 35b7642d7de..2e5514b3975 100644 --- a/litellm/llms/azure_ai/passthrough/transformation.py +++ b/litellm/llms/azure_ai/passthrough/transformation.py @@ -5,8 +5,9 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Final import httpx -from pydantic import TypeAdapter, ValidationError +from pydantic import ConfigDict, TypeAdapter, ValidationError +from litellm._lazy_imports import get_default_encoding from litellm._logging import verbose_logger from litellm.llms.azure_ai.common_utils import ( AzureFoundryModelInfo, @@ -22,10 +23,15 @@ from litellm.llms.base_llm.passthrough.transformation import ( relayed_body, strip_leading_model_segment, ) +from litellm.llms.cohere.embed.transformation import ( + CohereEmbeddingRowShape, + CohereImageTokenBilling, + cohere_embedding_response, +) from litellm.rust_bridge.ocr.entrypoints import NATIVE_OCR_PASSTHROUGH_RESPONSE, NativeOcrPassthroughResponse from litellm.types.llms.openai import AllMessageValues from litellm.types.rerank import RerankResponse -from litellm.types.utils import CallTypes, ImageResponse, StandardPassThroughResponseObject +from litellm.types.utils import CallTypes, EmbeddingResponse, ImageResponse, StandardPassThroughResponseObject if TYPE_CHECKING: from httpx import URL, Response @@ -35,6 +41,7 @@ if TYPE_CHECKING: EMPTY_QUERY: Final[Mapping[str, object]] = MappingProxyType({}) +ECHOED_TEXTS: Final = TypeAdapter(tuple[str, ...], config=ConfigDict(hide_input_in_errors=True)) def api_version_from(litellm_params: Mapping[str, object]) -> str | None: @@ -85,9 +92,29 @@ def relay_query_params( return MappingProxyType({**(request_query_params or EMPTY_QUERY), "api-version": api_version}) +def echoed_texts(body: Mapping[str, object]) -> tuple[str, ...]: + try: + return ECHOED_TEXTS.validate_python(body.get("texts")) + except ValidationError: + return () + + +def cohere_embed_relay_response(body: Mapping[str, object]) -> EmbeddingResponse: + return cohere_embedding_response( + body, + None, + get_default_encoding(), + echoed_texts(body), + CohereImageTokenBilling.FOLDED_INTO_INPUT_TOKENS, + CohereEmbeddingRowShape.TYPED, + ) + + FOUNDRY_RELAY_SHAPES: Final = ( RelayShape("/rerank", CallTypes.arerank, RerankResponse.model_validate), RelayShape("/providers/blackforestlabs/v1/flux-2-pro", CallTypes.aimage_generation, ImageResponse.model_validate), + RelayShape("/providers/cohere/v1/embed", CallTypes.aembedding, cohere_embed_relay_response), + RelayShape("/providers/cohere/v2/embed", CallTypes.aembedding, cohere_embed_relay_response), ) diff --git a/litellm/llms/azure_ai/responses/transformation.py b/litellm/llms/azure_ai/responses/transformation.py index da734cf8189..b19915648ab 100644 --- a/litellm/llms/azure_ai/responses/transformation.py +++ b/litellm/llms/azure_ai/responses/transformation.py @@ -232,7 +232,7 @@ class AzureAIAgentResponsesAPIConfig(AzureAIResponsesAPIConfig): model: str, drop_params: bool, *, - declared_reasoning_efforts: Collection[str], + declared_reasoning_efforts: Collection[str] | None, ) -> Mapping[str, object]: mapped: Final = super().map_openai_params_for_deployment( response_api_optional_params, model, drop_params, declared_reasoning_efforts=declared_reasoning_efforts diff --git a/litellm/llms/base_llm/base_utils.py b/litellm/llms/base_llm/base_utils.py index a9ec1f97d20..ef8d080027f 100644 --- a/litellm/llms/base_llm/base_utils.py +++ b/litellm/llms/base_llm/base_utils.py @@ -10,14 +10,23 @@ from abc import ABC, abstractmethod from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType -from typing import Any, Final +from typing import Any, Final, TypeAlias from openai.lib import _parsing, _pydantic # pyright: ignore[reportPrivateUsage] # SDK parser internals from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError +from typing_extensions import TypeIs # noqa: TID251 # narrows a message to its system member by the role literal from litellm._logging import verbose_logger +from litellm.constants import ANTHROPIC_BILLING_METADATA_PREFIX from litellm.litellm_core_utils.core_helpers import as_str_mapping -from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionCachedContent, + ChatCompletionSystemMessage, + ChatCompletionTextObject, + ChatCompletionToolCallChunk, + ChatCompletionUserMessage, +) from litellm.types.utils import ( CountContextManagement, Message, @@ -458,7 +467,7 @@ def type_to_response_format_param( def map_developer_role_to_system_role( - messages: list[AllMessageValues], + messages: Sequence[AllMessageValues], ) -> list[AllMessageValues]: """ Translate `developer` role to `system` role for non-OpenAI providers. @@ -473,3 +482,226 @@ def map_developer_role_to_system_role( else: new_messages.append(m) return new_messages + + +SystemMessageContent: TypeAlias = str | list[object] + +_system_content_adapter: Final = TypeAdapter[SystemMessageContent | None](SystemMessageContent | None) + + +def _system_content(message: ChatCompletionSystemMessage) -> SystemMessageContent | None: + if "content" not in message: + return "" + try: + content: Final = _system_content_adapter.validate_python(message["content"]) + except ValidationError: + return None + return "" if content is None else content + + +def _message_cache_control(message: ChatCompletionSystemMessage) -> ChatCompletionCachedContent | None: + if "cache_control" not in message: + return None + return message["cache_control"] + + +_SystemPart: TypeAlias = tuple[SystemMessageContent, ChatCompletionCachedContent | None] + + +def _with_message_cache_control( + blocks: Sequence[object], cache_control: ChatCompletionCachedContent | None +) -> tuple[object, ...]: + last: Final = as_str_mapping(blocks[-1]) if blocks else None + if last is None or cache_control is None or "cache_control" in last: + return tuple(blocks) + return (*blocks[:-1], {**last, "cache_control": cache_control}) + + +def _as_system_message(message: AllMessageValues) -> AllMessageValues: + if message["role"] != "developer": + return message + content_only: Final[ChatCompletionSystemMessage] = {"role": "system", "content": message["content"]} + named: Final[ChatCompletionSystemMessage] = ( + content_only if "name" not in message else {**content_only, "name": message["name"]} + ) + if "cache_control" not in message: + return named + cached: Final[ChatCompletionSystemMessage] = {**named, "cache_control": message["cache_control"]} + return cached + + +def _text_blocks( + content: SystemMessageContent, cache_control: ChatCompletionCachedContent | None +) -> tuple[object, ...]: + if not isinstance(content, str): + return _with_message_cache_control(content, cache_control) + if not content: + return () + if cache_control: + cached_block: Final[ChatCompletionTextObject] = { + "type": "text", + "text": content, + "cache_control": cache_control, + } + return (cached_block,) + text_block: Final[ChatCompletionTextObject] = {"type": "text", "text": content} + return (text_block,) + + +def _lone_text_block_text(blocks: Sequence[object]) -> str | None: + match blocks: + case [{"type": "text", "text": str() as text} as block] if block == {"type": "text", "text": text}: + return text + case _: + return None + + +def _plain_text(content: SystemMessageContent, cache_control: ChatCompletionCachedContent | None) -> str | None: + if cache_control: + return None + text: Final = content if isinstance(content, str) else _lone_text_block_text(content) + if text is None or text.startswith(ANTHROPIC_BILLING_METADATA_PREFIX): + return None + return text + + +def _is_system_message( + message: AllMessageValues, +) -> TypeIs[ChatCompletionSystemMessage]: # guard-ok: the role literal picks the TypedDict member + return message["role"] == "system" + + +def _with_leading_separator(blocks: tuple[object, ...]) -> tuple[object, ...]: + first: Final = as_str_mapping(blocks[0]) + text: Final = None if first is None else first.get("text") + if first is None or not isinstance(text, str) or text.startswith(ANTHROPIC_BILLING_METADATA_PREFIX): + return blocks + return ({**first, "text": f"\n\n{text}"}, *blocks[1:]) + + +def _merged_system_content(parts: Sequence[_SystemPart]) -> SystemMessageContent: + texts: Final = tuple(itertools.starmap(_plain_text, parts)) + if all(text is not None for text in texts): + return "\n\n".join(text for text in texts if text) + block_runs: Final = tuple(blocks for blocks in itertools.starmap(_text_blocks, parts) if blocks) + separated: Final = (*block_runs[:1], *map(_with_leading_separator, block_runs[1:])) + return list(itertools.chain.from_iterable(separated)) + + +def _merged_system_message( + run: Sequence[ChatCompletionSystemMessage], parts: Sequence[_SystemPart] +) -> ChatCompletionSystemMessage: + content: Final = _merged_system_content(parts) + named: Final = next((message for message in reversed(run) if "name" in message), None) + if named is None: + merged: Final[ChatCompletionSystemMessage] = {"role": "system", "content": content} + return merged + with_name: Final[ChatCompletionSystemMessage] = {"role": "system", "content": content, "name": named["name"]} + return with_name + + +def _merged_run(run: tuple[AllMessageValues, ...]) -> tuple[AllMessageValues, ...]: + system_run: Final = tuple(message for message in run if _is_system_message(message)) + if len(system_run) < 2: + return run + contents: Final = tuple((_system_content(message), _message_cache_control(message)) for message in system_run) + parts: Final = tuple((content, cache_control) for content, cache_control in contents if content is not None) + if len(parts) != len(contents): + return run + return (_merged_system_message(system_run, parts),) + + +def _merged_system_runs(messages: Sequence[AllMessageValues]) -> Iterator[AllMessageValues]: + runs: Final = (tuple(run) for _, run in itertools.groupby(messages, key=_is_system_message)) + return itertools.chain.from_iterable(map(_merged_run, runs)) + + +_INSTRUCTION_ROLES: Final = frozenset(("system", "developer")) + + +def _leading_instruction_block_length(roles: Sequence[str | None]) -> int: + return next((index for index, role in enumerate(roles) if role not in _INSTRUCTION_ROLES), len(roles)) + + +def _closing_instruction_block_start(roles: Sequence[str | None], leading_length: int) -> int: + last_conversation_index: Final = next( + (index for index in range(len(roles) - 1, leading_length - 1, -1) if roles[index] not in _INSTRUCTION_ROLES), + None, + ) + if last_conversation_index is None or roles[last_conversation_index] != "assistant": + return len(roles) + return last_conversation_index + 1 + + +def hoisted_instruction_indices(roles: Sequence[str | None]) -> tuple[int, ...]: + """ + Indices of the leading instruction block plus every developer item after it, except + a developer block that closes the conversation right after an assistant turn, which + stays in place so the request does not end on the assistant's turn. Items without a + role (tool calls and their outputs) count as conversation. + """ + leading_length: Final = _leading_instruction_block_length(roles) + closing_start: Final = _closing_instruction_block_start(roles, leading_length) + moved: Final = tuple(index for index in range(leading_length, closing_start) if roles[index] == "developer") + return (*range(leading_length), *moved) + + +def closing_instruction_indices(roles: Sequence[str | None]) -> frozenset[int]: + closing_start: Final = _closing_instruction_block_start(roles, _leading_instruction_block_length(roles)) + return frozenset(range(closing_start, len(roles))) + + +def _role_name(role: object) -> str | None: + return role if isinstance(role, str) else None + + +def _message_roles(messages: Sequence[AllMessageValues]) -> tuple[str | None, ...]: + return tuple(_role_name(message["role"]) for message in messages) + + +def _hoisted_message_order(messages: Sequence[AllMessageValues]) -> tuple[int, ...]: + hoisted: Final = hoisted_instruction_indices(_message_roles(messages)) + hoisted_set: Final = frozenset(hoisted) + return (*hoisted, *(index for index in range(len(messages)) if index not in hoisted_set)) + + +def _developer_message_as_user(message: AllMessageValues) -> AllMessageValues: + if message["role"] != "developer": + return message + copied: Final = copy.deepcopy(message) + if "cache_control" not in copied: + as_user: Final[ChatCompletionUserMessage] = {"role": "user", "content": copied["content"]} + return as_user + cached_as_user: Final[ChatCompletionUserMessage] = { + "role": "user", + "content": copied["content"], + "cache_control": copied["cache_control"], + } + return cached_as_user + + +def merge_system_message_runs(messages: Sequence[AllMessageValues]) -> Sequence[AllMessageValues]: + """ + Translate `developer` role to `system` role in place and fold each run of + consecutive system messages into one, for chat templates that render a single + system message per run. + """ + return tuple(_merged_system_runs(tuple(map(_as_system_message, messages)))) + + +def hoist_developer_messages_into_leading_system_message( + messages: Sequence[AllMessageValues], +) -> Sequence[AllMessageValues]: + """ + Place instructions for an OpenAI-compatible model whose chat template allows a single + system message and only at the start: developer messages after the first user turn move + into the leading system block, a developer message that closes the conversation right + after an assistant turn becomes a user turn so the request does not end on the + assistant's turn, and each run of consecutive system messages folds into one. + """ + order: Final = _hoisted_message_order(messages) + closing: Final = closing_instruction_indices(_message_roles(messages)) + placed: Final = tuple( + _developer_message_as_user(messages[index]) if index in closing else messages[index] for index in order + ) + return merge_system_message_runs(placed) diff --git a/litellm/llms/base_llm/chat/transformation.py b/litellm/llms/base_llm/chat/transformation.py index 207a4705c92..b87a783e244 100644 --- a/litellm/llms/base_llm/chat/transformation.py +++ b/litellm/llms/base_llm/chat/transformation.py @@ -4,7 +4,7 @@ Common base config for all LLM providers import types from abc import ABC, abstractmethod -from collections.abc import AsyncIterator, Iterator, Mapping +from collections.abc import AsyncIterator, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, Final, Union import httpx @@ -199,12 +199,19 @@ class BaseConfig(ABC): def translate_developer_role_to_system_role( self, - messages: list[AllMessageValues], - ) -> list[AllMessageValues]: + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: """ Translate `developer` role to `system` role for non-OpenAI providers. - Overriden by OpenAI/Azure + Overridden by OpenAI, which also places the instructions in one leading system + message for an OpenAI-compatible model declared to reject a later system + message, and by the o-series and Azure configs. """ return map_developer_role_to_system_role(messages=messages) diff --git a/litellm/llms/base_llm/realtime/transformation.py b/litellm/llms/base_llm/realtime/transformation.py index 1f4ad29fa74..f1539d16b30 100644 --- a/litellm/llms/base_llm/realtime/transformation.py +++ b/litellm/llms/base_llm/realtime/transformation.py @@ -4,7 +4,6 @@ from types import TracebackType from typing import TYPE_CHECKING, Any, Protocol import httpx -from typing_extensions import Self from litellm.types.llms.openai import OpenAIRealtimeStreamSessionEvents from litellm.types.realtime import ( @@ -24,7 +23,7 @@ else: class RealtimeBackend(Protocol): - async def __aenter__(self) -> Self: ... + async def __aenter__(self) -> object: ... async def __aexit__( self, @@ -97,9 +96,15 @@ class BaseRealtimeConfig(ABC): def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None: return None + def session_refusal(self) -> Exception | None: + return None + async def open_backend(self, url: str, headers: Mapping[str, str]) -> RealtimeBackend | None: return None + def wrap_backend(self, backend: RealtimeBackend) -> RealtimeBackend: + return backend + def transform_session_created_event( self, model: str, diff --git a/litellm/llms/base_llm/responses/transformation.py b/litellm/llms/base_llm/responses/transformation.py index 01baeed9bef..792b14090b8 100644 --- a/litellm/llms/base_llm/responses/transformation.py +++ b/litellm/llms/base_llm/responses/transformation.py @@ -69,6 +69,9 @@ class BaseResponsesAPIConfig(ABC): def supports_encrypted_agent_messages(self) -> bool: return False + def supports_streamed_retrieval(self) -> bool: + return self.custom_llm_provider in (LlmProviders.OPENAI, LlmProviders.AZURE, LlmProviders.AZURE_AI) + def sign_request( self, headers: dict, @@ -209,7 +212,7 @@ class BaseResponsesAPIConfig(ABC): api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, - ) -> tuple[str, dict]: + ) -> tuple[str, dict[str, str]]: pass @abstractmethod @@ -268,6 +271,9 @@ class BaseResponsesAPIConfig(ABC): """Returns True if litellm should fake a stream for the given model and stream value""" return False + def may_answer_stream_with_json(self) -> bool: + return False + def supports_native_websocket(self) -> bool: """ Returns True if the provider has a native WebSocket endpoint for Responses API. diff --git a/litellm/llms/base_llm/vector_store/transformation.py b/litellm/llms/base_llm/vector_store/transformation.py index 6885183aeba..4924c5a5acc 100644 --- a/litellm/llms/base_llm/vector_store/transformation.py +++ b/litellm/llms/base_llm/vector_store/transformation.py @@ -226,6 +226,9 @@ class BaseVectorStoreConfig: ``get_complete_url`` adds a query string, like Azure, override this.""" return f"{api_base}/{encoded_vector_store_id}" + def get_vector_store_list_url(self, api_base: str) -> str: + return api_base + def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException: from ..chat.transformation import BaseLLMException diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index f54c352fd6e..443e722168d 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -17,6 +17,8 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger +from litellm.anthropic_beta_gated_fields import active_beta_values, log_beta_gated_strips, strip_beta_gated_thinking +from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers from litellm.constants import ( BEDROCK_MIN_THINKING_BUDGET_TOKENS, RESPONSE_FORMAT_TOOL_NAME, @@ -117,6 +119,7 @@ from litellm.utils import ( ) from ..common_utils import ( + DROP_DISABLED_REASONING_EFFORT_WARNING, BedrockError, BedrockModelInfo, bedrock_cache_point_ttl_supported, @@ -1192,11 +1195,7 @@ class AmazonConverseConfig(BaseConfig): ), status_code=400, ) - verbose_logger.debug( - "Dropping unsupported `reasoning_effort=%s` for Bedrock model=%s.", - value, - model, - ) + verbose_logger.warning(DROP_DISABLED_REASONING_EFFORT_WARNING, "reasoning_effort", value, model) elif param == "reasoning_effort" and isinstance(value, str): self._handle_reasoning_effort_parameter( model=model, reasoning_effort=value, optional_params=optional_params @@ -1828,7 +1827,16 @@ class AmazonConverseConfig(BaseConfig): filtered_tools.append(tool) base_model: Final = BedrockModelInfo.get_base_model(model) - client_beta_list: Final = get_anthropic_beta_from_headers(headers or {}) + header_beta_list: Final = get_anthropic_beta_from_headers(headers or {}) + client_beta_list: Final = header_beta_list or filter_and_transform_beta_headers( + sorted( + active_beta_values( + {}, + additional_request_params.get("anthropic_beta"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # additional_request_params is a bare dict + ) + ), + provider="bedrock_converse", + ) eager_beta: Final = ( (ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER,) if base_model.startswith("anthropic") @@ -1836,7 +1844,15 @@ class AmazonConverseConfig(BaseConfig): and ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER not in client_beta_list else () ) - anthropic_beta_list: Final = [*client_beta_list, *eager_beta] + thinking_betas: Final = AnthropicModelInfo().thinking_betas( + additional_request_params.get("thinking"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # additional_request_params is a bare dict + ) + missing_thinking_betas: Final = ( + tuple(beta for beta in thinking_betas if beta not in client_beta_list) + if base_model.startswith("anthropic") + else () + ) + anthropic_beta_list: Final = [*client_beta_list, *eager_beta, *missing_thinking_betas] # Only separate tools if computer use tools are actually present if filtered_tools and self.is_computer_use_tool_used(filtered_tools, model): @@ -1906,7 +1922,8 @@ class AmazonConverseConfig(BaseConfig): else: computer_use_header = "computer-use-2024-10-22" - anthropic_beta_list.append(computer_use_header) + if computer_use_header not in anthropic_beta_list: + anthropic_beta_list.append(computer_use_header) # Transform computer use tools to proper Bedrock format transformed_computer_tools: Final = self._transform_computer_use_tools(computer_use_tools) additional_request_params["tools"] = transformed_computer_tools @@ -2031,6 +2048,16 @@ class AmazonConverseConfig(BaseConfig): bedrock_tools, anthropic_beta_list = self._process_tools_and_beta( original_tools, model, headers, additional_request_params, litellm_params, deployment_model ) + requested_thinking: Final = additional_request_params.get("thinking") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # _prepare_request_params hands back a bare dict + body_betas: Final = additional_request_params.get("anthropic_beta") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # same bare dict + gated_thinking: Final = strip_beta_gated_thinking( + requested_thinking, # pyright: ignore[reportUnknownArgumentType] # same bare dict + active_beta_values({}, body_betas) | frozenset(anthropic_beta_list), # pyright: ignore[reportUnknownArgumentType] # _process_tools_and_beta hands back a bare list + drop_params=drop_params, + ) + if gated_thinking.stripped: + log_beta_gated_strips(gated_thinking.stripped, model=model, provider="bedrock_converse") + additional_request_params["thinking"] = gated_thinking.thinking # Append cachePoint to tools if cache_control_injection_points has tool_config cache_injection_points: Final = additional_request_params.pop("cache_control_injection_points", None) diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py index eb6ed5c75e3..e32df0bbdbb 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx from pydantic import TypeAdapter +from litellm.anthropic_beta_gated_fields import strip_beta_gated_request_body from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import object_mapping from litellm.litellm_core_utils.prompt_templates.factory import ( @@ -210,7 +211,15 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): if beta_list: _anthropic_request["anthropic_beta"] = beta_list - return _anthropic_request + return dict( + strip_beta_gated_request_body( + _anthropic_request, # pyright: ignore[reportUnknownArgumentType] # built by the base signature's bare dict transform + {}, + drop_params=litellm_params.get("drop_params"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # litellm_params is the base signature's bare dict + model=model, + provider="bedrock_invoke", + ) + ) @property def uses_async_transform_request(self) -> bool: @@ -294,7 +303,6 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): programmatic_tool_calling_used: Final = self.is_programmatic_tool_calling_used(tools) input_examples_used: Final = self.is_input_examples_used(tools) is_mid_conversation_output_config_used: Final = self.is_mid_conversation_output_config_used(messages) - is_thinking_display_updates_used: Final = self.is_thinking_display_updates_used(thinking) client_beta_set: Final = set(get_anthropic_beta_from_headers(headers)) user_beta_set: Final = ( @@ -312,7 +320,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): mcp_server_used=self.is_mcp_server_used(optional_params.get("mcp_servers")), custom_llm_provider="bedrock", is_mid_conversation_output_config_used=is_mid_conversation_output_config_used, - is_thinking_display_updates_used=is_thinking_display_updates_used, + thinking=thinking, ) beta_set.update(auto_betas) diff --git a/litellm/llms/bedrock/claude_platform/transformation.py b/litellm/llms/bedrock/claude_platform/transformation.py index 0194f1c7e45..a5ecc37d394 100644 --- a/litellm/llms/bedrock/claude_platform/transformation.py +++ b/litellm/llms/bedrock/claude_platform/transformation.py @@ -72,6 +72,7 @@ class BedrockClaudePlatformConfig(BedrockClaudePlatformMixin, AnthropicConfig): ), code_execution_tool_used=self.is_code_execution_tool_used(tools=optional_params.get("tools")), container_with_skills_used=self.is_container_with_skills_used(optional_params=optional_params), + thinking=optional_params.get("thinking"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # optional_params is the base config's bare dict ) anthropic_headers["anthropic-workspace-id"] = workspace_id return {**headers, **anthropic_headers} diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index fde995c7907..4175479f068 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -15,7 +15,11 @@ from dataclasses import dataclass from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias -from typing_extensions import ReadOnly, TypedDict +from typing_extensions import ( + ReadOnly, + TypedDict, + TypeIs, # noqa: TID251 # narrows the caller's tools without copying them +) if TYPE_CHECKING: from botocore.eventstream import EventStreamMessage @@ -573,6 +577,26 @@ def strip_unsupported_bedrock_invoke_output_config_keys( request_body["output_config"] = {"format": preserved_format} # rebind-ok: out-param +_TOOL_ITEMS_ADAPTER: Final = TypeAdapter(tuple[object, ...]) + + +def _is_str_keyed_dict( + value: object, +) -> TypeIs[dict[str, object]]: # guard-ok: JSON tool objects # mutable-ok: fallbacks reuse tools edited in place + return isinstance(value, dict) + + +def _normalize_custom_field_on_tool(tool: object) -> None: + if not _is_str_keyed_dict(tool): + return + custom: Final = tool.pop("custom", None) + if not _is_str_keyed_dict(custom) or "defer_loading" in tool: + return + deferred: Final = custom.get("defer_loading") + if isinstance(deferred, bool): + tool["defer_loading"] = deferred # rebind-ok: a fallback reuses the caller's tool, as on main + + def normalize_custom_field_on_tools(request_body: dict) -> None: """ Drop the ``custom`` field from each tool, first hoisting a boolean @@ -590,15 +614,8 @@ def normalize_custom_field_on_tools(request_body: dict) -> None: tools: Final = request_body.get("tools") if not tools or not isinstance(tools, list): return - for tool in tools: - if not isinstance(tool, dict): - continue - custom: dict[str, object] | None = tool.pop("custom", None) - if not isinstance(custom, dict) or "defer_loading" in tool: - continue - deferred: object = custom.get("defer_loading") - if isinstance(deferred, bool): - tool["defer_loading"] = deferred + for tool in _TOOL_ITEMS_ADAPTER.validate_python(tools): + _normalize_custom_field_on_tool(tool) _TOOL_DICTS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...]) @@ -1314,6 +1331,12 @@ def _mantle_api_base_from_env() -> str | None: return next((base[: -len(suffix)] for suffix in _MANTLE_OPENAI_BASE_SUFFIXES if base.endswith(suffix)), base) +DROP_DISABLED_REASONING_EFFORT_WARNING: Final = ( + "Dropping %s=%r for Bedrock model=%r (drop_params=True) because the model map marks that level unsupported. " + "Send a level the model supports." +) + + def bedrock_reasoning_effort_disabled(model: str, effort: str) -> bool: from litellm.utils import is_explicitly_disabled_factory diff --git a/litellm/llms/bedrock/count_tokens/mantle_handler.py b/litellm/llms/bedrock/count_tokens/mantle_handler.py index 34e622b5e8d..5cd26235ba2 100644 --- a/litellm/llms/bedrock/count_tokens/mantle_handler.py +++ b/litellm/llms/bedrock/count_tokens/mantle_handler.py @@ -8,6 +8,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.anthropic.common_utils import tools_without_custom_field from litellm.llms.anthropic.count_tokens.transformation import COUNT_TOKEN_OPTION_NAMES, AnthropicCountTokensConfig from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, run_aws_signing from litellm.llms.bedrock.common_utils import BedrockError, build_mantle_messages_url @@ -26,6 +27,7 @@ class MantleCountTokensRequest(TypedDict): _COUNT_REQUEST: Final = TypeAdapter(MantleCountTokensRequest) _COUNT_RESPONSE: Final = TypeAdapter(dict[str, JsonValue]) _COUNT_OPTIONS: Final[TypeAdapter[Mapping[str, JsonValue]]] = TypeAdapter(Mapping[str, JsonValue]) +_TOOLS: Final = TypeAdapter(list[dict[str, JsonValue]]) class BedrockMantleCountTokensHandler(BaseAWSLLM): @@ -59,10 +61,15 @@ class BedrockMantleCountTokensHandler(BaseAWSLLM): self.get_aws_region_name, optional_params=litellm_params, model=resolved_model, model_id=None ) endpoint_url: Final = await run_aws_signing(self.get_mantle_count_tokens_endpoint, aws_region_name) + tools: Final = request.get("tools") body: Final = self._anthropic_config.transform_request_to_count_tokens( model=resolved_model, messages=request["messages"], - tools=request.get("tools"), + tools=None + if tools is None + else _TOOLS.validate_python( + tools_without_custom_field(tools, hoist_defer_loading=True, drop_custom=True) + ), system=request.get("system"), optional_params=_COUNT_OPTIONS.validate_python( {name: value for name, value in request_data.items() if name in COUNT_TOKEN_OPTION_NAMES} diff --git a/litellm/llms/bedrock/count_tokens/transformation.py b/litellm/llms/bedrock/count_tokens/transformation.py index bf9c119f6f2..df5a3827c1f 100644 --- a/litellm/llms/bedrock/count_tokens/transformation.py +++ b/litellm/llms/bedrock/count_tokens/transformation.py @@ -12,6 +12,7 @@ from typing import Final, Literal from pydantic import JsonValue from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.anthropic.common_utils import tools_without_custom_field from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.common_utils import get_bedrock_base_model @@ -194,7 +195,15 @@ class BedrockCountTokensConfig(BaseAWSLLM): # For InvokeModel, we need to provide the raw body that would be sent to the model # Remove the 'model' field from the body as it's not part of the model input - body_data: Final = {k: v for k, v in request_data.items() if k != "model"} + tools: Final = request_data.get("tools") + body_data: Final = { + **{k: v for k, v in request_data.items() if k != "model"}, + **( + {"tools": tools_without_custom_field(tools, hoist_defer_loading=True, drop_custom=True)} + if isinstance(tools, list) + else {} + ), + } if "messages" in body_data: # Bedrock validates the body against the model's InvokeModel schema; diff --git a/litellm/llms/bedrock/embed/embedding.py b/litellm/llms/bedrock/embed/embedding.py index 4d27d94664f..2f0ed4d0e82 100644 --- a/litellm/llms/bedrock/embed/embedding.py +++ b/litellm/llms/bedrock/embed/embedding.py @@ -9,9 +9,11 @@ from collections.abc import Callable, Mapping from typing import TYPE_CHECKING, Final, get_args, overload import httpx +from pydantic import TypeAdapter import litellm from litellm.constants import BEDROCK_EMBEDDING_PROVIDERS_LITERAL +from litellm.litellm_core_utils.hidden_params import set_hidden_param from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.cohere.embed.handler import embedding as cohere_embedding from litellm.llms.custom_httpx.http_handler import ( @@ -36,7 +38,7 @@ from ..base_aws_llm import ( pop_aws_auth_params, run_aws_signing, ) -from ..common_utils import BedrockError +from ..common_utils import BedrockError, split_bedrock_region_path from .amazon_nova_transformation import AmazonNovaEmbeddingConfig from .amazon_titan_g1_transformation import AmazonTitanG1Config from .amazon_titan_multimodal_transformation import ( @@ -64,12 +66,21 @@ def _sign_get_request( return request.prepare() +_OPTIONAL_STR: Final[TypeAdapter[str | None]] = TypeAdapter(str | None) + + +def _without_region_path(model: str) -> str: + region, model_id = split_bedrock_region_path(model) + return model if region is None else model_id + + class BedrockEmbedding(BaseAWSLLM): @overload def _load_credentials( self, optional_params: dict, # mutable-ok: the implementation pops the aws_* keys out of the caller's dict in place bearer_token: None = None, + region_from_model: str | None = None, ) -> tuple[Credentials, str]: ... @overload @@ -77,15 +88,18 @@ class BedrockEmbedding(BaseAWSLLM): self, optional_params: dict, # mutable-ok: the implementation pops the aws_* keys out of the caller's dict in place bearer_token: str, + region_from_model: str | None = None, ) -> tuple[None, str]: ... def _load_credentials( self, optional_params: dict, bearer_token: str | None = None, + region_from_model: str | None = None, ) -> tuple[Credentials | None, str]: auth_params: Final = pop_aws_auth_params(optional_params) - aws_region_name = optional_params.pop("aws_region_name", None) + explicit_region_name: Final = optional_params.pop("aws_region_name", None) + aws_region_name = region_from_model if explicit_region_name is None else explicit_region_name ### SET REGION NAME ### if aws_region_name is None: @@ -184,14 +198,17 @@ class BedrockEmbedding(BaseAWSLLM): provider: BEDROCK_EMBEDDING_PROVIDERS_LITERAL, is_async_invoke: bool | None = False, batch_data: list[dict] | None = None, + aws_region_name: str | None = None, ) -> EmbeddingResponse | None: """ Transforms the response from the Bedrock embedding provider to the OpenAI format. """ returned_response: EmbeddingResponse | None = None + bedrock_model: Final = _without_region_path(model) + is_async_job: Final = bool(is_async_invoke) and len(response_list) == 1 and "invocationArn" in response_list[0] # Handle async invoke responses (single response with invocationArn) - if is_async_invoke and len(response_list) == 1 and "invocationArn" in response_list[0]: + if is_async_job: if provider == "twelvelabs": returned_response = TwelveLabsMarengoEmbeddingConfig().transform_async_invoke_response( response=response_list[0], model=model @@ -227,15 +244,15 @@ class BedrockEmbedding(BaseAWSLLM): ) else: # Handle regular invoke responses - if model == "amazon.titan-embed-image-v1": + if bedrock_model == "amazon.titan-embed-image-v1": returned_response = AmazonTitanMultimodalEmbeddingG1Config().transform_response( response_list=response_list, model=model, batch_data=batch_data ) - elif model == "amazon.titan-embed-text-v1": + elif bedrock_model == "amazon.titan-embed-text-v1": returned_response = AmazonTitanG1Config().transform_response(response_list=response_list, model=model) - elif model == "amazon.titan-embed-text-v2:0": + elif bedrock_model == "amazon.titan-embed-text-v2:0": returned_response = AmazonTitanV2Config().transform_response(response_list=response_list, model=model) - elif model == "amazon.titan-embed-g1-text-02": + elif bedrock_model == "amazon.titan-embed-g1-text-02": returned_response = AmazonTitanG1Config().transform_response(response_list=response_list, model=model) elif provider == "twelvelabs": returned_response = TwelveLabsMarengoEmbeddingConfig().transform_response( @@ -251,6 +268,8 @@ class BedrockEmbedding(BaseAWSLLM): ########################################################## if returned_response is None: raise Exception(f"Unable to map model response to known provider format. model={model}") + if aws_region_name is not None and not is_async_job: + set_hidden_param(returned_response, "region_name", aws_region_name) return returned_response def _single_func_embeddings( @@ -319,6 +338,7 @@ class BedrockEmbedding(BaseAWSLLM): provider=provider, is_async_invoke=is_async_invoke, batch_data=batch_data, + aws_region_name=aws_region_name, ) async def _async_single_func_embeddings( @@ -390,6 +410,7 @@ class BedrockEmbedding(BaseAWSLLM): provider=provider, is_async_invoke=is_async_invoke, batch_data=batch_data, + aws_region_name=aws_region_name, ) def embeddings( @@ -409,26 +430,29 @@ class BedrockEmbedding(BaseAWSLLM): litellm_params: dict, api_key: str | None = None, ) -> EmbeddingResponse: + model_id_param: Final = _OPTIONAL_STR.validate_python(optional_params.pop("model_id", None)) or None + has_async_invoke: Final = "async_invoke/" in model + model_without_async_route: Final = model.replace("async_invoke/", "", 1) + path_region: Final = split_bedrock_region_path(model_without_async_route)[0] if model_id_param is None else None + bedrock_model: Final = _without_region_path(model_without_async_route) credentials, aws_region_name = self._load_credentials( - optional_params, bearer_token=bedrock_bearer_token(api_key) + optional_params, + bearer_token=bedrock_bearer_token(api_key), + region_from_model=self._get_aws_region_from_model_arn(model_id_param) or path_region, ) ### TRANSFORMATION ### - unencoded_model_id: Final = optional_params.pop("model_id", None) or model # default to model if not passed + unencoded_model_id: Final = model_id_param or bedrock_model modelId: Final = urllib.parse.quote(unencoded_model_id, safe="") aws_region_name = self._get_aws_region_name( optional_params={"aws_region_name": aws_region_name}, - model=model, + model=bedrock_model, model_id=unencoded_model_id, ) - # Check async invoke needs to be used - has_async_invoke: Final = "async_invoke/" in model - if has_async_invoke: - model = model.replace("async_invoke/", "", 1) - provider: Final = self.get_bedrock_embedding_provider(model) + provider: Final = self.get_bedrock_embedding_provider(bedrock_model) if provider is None: raise Exception( - f"Unable to determine bedrock embedding provider for model: {model}. " + f"Unable to determine bedrock embedding provider for model: {bedrock_model}. " f"Supported providers: {list(get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL))}" ) inference_params = copy.deepcopy(optional_params) @@ -441,9 +465,9 @@ class BedrockEmbedding(BaseAWSLLM): batch_data: list | None = None if provider == "cohere": data = BedrockCohereEmbeddingConfig().transform_request( - model=model, input=input, inference_params=inference_params + model=bedrock_model, input=input, inference_params=inference_params ) - elif provider == "amazon" and model in [ + elif provider == "amazon" and bedrock_model in [ "amazon.titan-embed-image-v1", "amazon.titan-embed-text-v1", "amazon.titan-embed-text-v2:0", @@ -451,28 +475,28 @@ class BedrockEmbedding(BaseAWSLLM): ]: batch_data = [] for i in input: - if model == "amazon.titan-embed-image-v1": + if bedrock_model == "amazon.titan-embed-image-v1": transformed_request: AmazonEmbeddingRequest = ( AmazonTitanMultimodalEmbeddingG1Config().transform_request( input=i, inference_params=inference_params ) ) - elif model == "amazon.titan-embed-text-v1": + elif bedrock_model == "amazon.titan-embed-text-v1": transformed_request = AmazonTitanG1Config().transform_request( input=i, inference_params=inference_params ) - elif model == "amazon.titan-embed-text-v2:0": + elif bedrock_model == "amazon.titan-embed-text-v2:0": transformed_request = AmazonTitanV2Config().transform_request( input=i, inference_params=inference_params ) - elif model == "amazon.titan-embed-g1-text-02": + elif bedrock_model == "amazon.titan-embed-g1-text-02": transformed_request = AmazonTitanG1Config().transform_request( input=i, inference_params=inference_params ) else: raise Exception( "Unmapped model. Received={}. Expected={}".format( - model, + bedrock_model, [ "amazon.titan-embed-image-v1", "amazon.titan-embed-text-v1", @@ -485,7 +509,7 @@ class BedrockEmbedding(BaseAWSLLM): elif provider == "twelvelabs": batch_data = [] for i in input: - twelvelabs_request = TwelveLabsMarengoEmbeddingConfig(model=model).transform_request( + twelvelabs_request = TwelveLabsMarengoEmbeddingConfig(model=bedrock_model).transform_request( input=i, inference_params=inference_params, async_invoke_route=has_async_invoke, @@ -527,7 +551,7 @@ class BedrockEmbedding(BaseAWSLLM): extra_headers=extra_headers, endpoint_url=endpoint_url, aws_region_name=aws_region_name, - model=model, + model=model_without_async_route, logging_obj=logging_obj, api_key=api_key, provider=provider, @@ -541,7 +565,7 @@ class BedrockEmbedding(BaseAWSLLM): extra_headers=extra_headers, endpoint_url=endpoint_url, aws_region_name=aws_region_name, - model=model, + model=model_without_async_route, logging_obj=logging_obj, api_key=api_key, provider=provider, @@ -570,8 +594,9 @@ class BedrockEmbedding(BaseAWSLLM): ## ROUTING ## # Convert CaseInsensitiveDict to regular dict for httpx compatibility headers_for_request: Final = dict(prepped.headers) if hasattr(prepped, "headers") else {} + set_hidden_param(model_response, "region_name", aws_region_name) return cohere_embedding( - model=model, + model=model_without_async_route, input=input, model_response=model_response, logging_obj=logging_obj, diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py index e1e15861a90..e5d6a168d57 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py @@ -7,6 +7,7 @@ from pydantic import TypeAdapter import litellm from litellm._logging import verbose_logger +from litellm.anthropic_beta_gated_fields import log_beta_gated_strips, strip_beta_gated_native_request from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers from litellm.constants import ( BEDROCK_MIN_THINKING_BUDGET_TOKENS, @@ -102,6 +103,9 @@ class AmazonAnthropicClaudeMessagesConfig( def should_strip_billing_metadata(self) -> bool: return True + def tools_for_upstream(self, tools: Sequence[object], model: str, drop_params: object) -> Sequence[object]: + return tools + def drops_dangerous_tool_use_beta_without_safeguards(self) -> bool: return True @@ -546,6 +550,9 @@ class AmazonAnthropicClaudeMessagesConfig( "edits": retained_edits, } + def strips_beta_gated_fields_from_final_headers(self) -> bool: + return False + def _get_bedrock_invoke_anthropic_beta_headers( self, model: str, @@ -581,9 +588,7 @@ class AmazonAnthropicClaudeMessagesConfig( ), custom_llm_provider="bedrock", is_mid_conversation_output_config_used=is_mid_conversation_output_config_used, - is_thinking_display_updates_used=anthropic_model_info.is_thinking_display_updates_used( - anthropic_messages_request.get("thinking") - ), + thinking=anthropic_messages_request.get("thinking"), is_mid_conversation_tool_change_used=anthropic_model_info.is_mid_conversation_tool_change_used( outgoing_messages_typed ), @@ -754,7 +759,9 @@ class AmazonAnthropicClaudeMessagesConfig( ), "display": "updates", } - self._normalize_system_role_messages(anthropic_messages_request, model=model) + self._normalize_system_role_messages( + anthropic_messages_request, model=model, drop_params=getattr(litellm_params, "drop_params", None) + ) ######################################################### ############## BEDROCK Invoke SPECIFIC TRANSFORMATION ### ######################################################### @@ -822,6 +829,20 @@ class AmazonAnthropicClaudeMessagesConfig( if filtered_betas: anthropic_messages_request["anthropic_beta"] = filtered_betas + final_request: Final = OBJECT_MAPPING_ADAPTER.validate_python(anthropic_messages_request) # pyright: ignore[reportUnknownArgumentType] # built from the base signature's bare dict + gated: Final = strip_beta_gated_native_request( + {}, + final_request, + final_request.get("messages"), + body_betas=filtered_betas, + drop_params=getattr(litellm_params, "drop_params", None), + ) + log_beta_gated_strips(gated.stripped, model=model, provider="bedrock_invoke") + if gated.thinking_stripped: + anthropic_messages_request["thinking"] = gated.thinking + if gated.messages_stripped: + anthropic_messages_request["messages"] = list(gated.messages) + outbound_tools: Final = tools_without_eager_input_streaming(anthropic_messages_request) if outbound_tools is not None: anthropic_messages_request["tools"] = outbound_tools diff --git a/litellm/llms/bedrock/responses/transformation.py b/litellm/llms/bedrock/responses/transformation.py index 7cdd7b08f99..ff44d973d97 100644 --- a/litellm/llms/bedrock/responses/transformation.py +++ b/litellm/llms/bedrock/responses/transformation.py @@ -52,6 +52,7 @@ from litellm.llms.base_llm.responses.codex_compat import drop_unsupported_tools, from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.common_utils import ( BEDROCK_CHAT_COMPLETIONS_ROUTE_PREFIX, + DROP_DISABLED_REASONING_EFFORT_WARNING, BedrockError, bedrock_reasoning_effort_disabled, bedrock_supports_openai_responses, @@ -173,7 +174,7 @@ def _without_disabled_reasoning_effort( ), status_code=400, ) - verbose_logger.debug("Dropping unsupported `reasoning.effort=%s` for Bedrock model=%s.", effort, model) + verbose_logger.warning(DROP_DISABLED_REASONING_EFFORT_WARNING, "reasoning.effort", effort, model) rest: Final = {key: value for key, value in reasoning.items() if key != "effort"} without_reasoning: Final = {key: value for key, value in params.items() if key != "reasoning"} return {**without_reasoning, "reasoning": rest} if rest else without_reasoning diff --git a/litellm/llms/bigmodel/__init__.py b/litellm/llms/bigmodel/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/bigmodel/chat/__init__.py b/litellm/llms/bigmodel/chat/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/bigmodel/chat/transformation.py b/litellm/llms/bigmodel/chat/transformation.py new file mode 100644 index 00000000000..12d5e912c15 --- /dev/null +++ b/litellm/llms/bigmodel/chat/transformation.py @@ -0,0 +1,59 @@ +from collections.abc import Mapping +from typing import Final + +from litellm.llms.zai.chat.transformation import ZAIChatConfig + +from ..common_utils import get_bigmodel_api_key, get_bigmodel_url, is_unsupported_disabled_thinking + +_SUPPORTED_PARAMS: Final = ( + "max_tokens", + "max_completion_tokens", + "stream", + "stream_options", + "temperature", + "top_p", + "stop", + "tools", + "tool_choice", + "response_format", + "thinking", + "reasoning_effort", +) + + +class BigModelChatConfig(ZAIChatConfig): + @property + def custom_llm_provider(self) -> str: + return "bigmodel" + + def get_openai_compatible_provider_info(self, api_base: str | None, api_key: str | None) -> tuple[str, str | None]: + return get_bigmodel_url(api_base, "chat"), get_bigmodel_api_key(api_key) + + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: inherited interface requires a list + return list(_SUPPORTED_PARAMS) + + def map_openai_params( + self, + non_default_params: Mapping[str, object], + optional_params: Mapping[str, object], + model: str, + drop_params: bool, + ) -> dict[str, object]: # mutable-ok: request pipeline adds parameters to the returned body + mapped: Final = { + **optional_params, + **{ + ("max_tokens" if key == "max_completion_tokens" else key): value + for key, value in non_default_params.items() + if key in _SUPPORTED_PARAMS and key != "thinking" + }, + } + if "thinking" not in non_default_params: + return mapped + thinking: Final = non_default_params["thinking"] + if is_unsupported_disabled_thinking(model, thinking): + return { + **mapped, + "reasoning_effort": mapped.get("reasoning_effort", "low"), + "extra_body": {"thinking": {"type": "enabled"}}, + } + return {**mapped, "extra_body": {"thinking": thinking}} diff --git a/litellm/llms/bigmodel/common_utils.py b/litellm/llms/bigmodel/common_utils.py new file mode 100644 index 00000000000..0c9b6da5c8f --- /dev/null +++ b/litellm/llms/bigmodel/common_utils.py @@ -0,0 +1,92 @@ +import re +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final, Literal +from urllib.parse import urlsplit, urlunsplit + +import httpx +from pydantic import JsonValue, TypeAdapter, ValidationError + +import litellm +from litellm.exceptions import ContentPolicyViolationError, ContextWindowExceededError +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.secret_managers.main import get_secret_str + +BIGMODEL_API_BASE: Final = "https://open.bigmodel.cn" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_BUSINESS_CODE_IN_ERROR: Final = re.compile(r"""['"]code['"]\s*:\s*['"]?(\d{4})\b""") +_CODING_CHAT_PATH: Final = "/api/coding/paas/v4" +_API_PATHS: Final = ( + "/api/coding/paas/v4/chat/completions", + _CODING_CHAT_PATH, + "/api/paas/v4/chat/completions", + "/api/anthropic/v1/messages", + "/api/anthropic/v1", + "/api/v1/responses", + "/api/paas/v4", + "/api/anthropic", + "/api/v1", + "/api", +) +_ENDPOINT_PATHS: Final = MappingProxyType( + { + "chat": "/api/paas/v4", + "responses": "/api/v1/responses", + "messages": "/api/anthropic/v1/messages", + } +) + + +def get_bigmodel_api_key(api_key: str | None) -> str | None: + return api_key or get_secret_str("BIGMODEL_API_KEY") + + +def get_bigmodel_url(api_base: str | None, endpoint: Literal["chat", "responses", "messages"]) -> str: + resolved_base: Final = api_base or get_secret_str("BIGMODEL_API_BASE") or BIGMODEL_API_BASE + parsed: Final = urlsplit(resolved_base) + path: Final = parsed.path.rstrip("/") + suffix: Final = next((candidate for candidate in _API_PATHS if path.endswith(candidate)), "") + endpoint_path: Final = ( + _CODING_CHAT_PATH if endpoint == "chat" and suffix.startswith(_CODING_CHAT_PATH) else _ENDPOINT_PATHS[endpoint] + ) + return urlunsplit( + (parsed.scheme, parsed.netloc, f"{path.removesuffix(suffix)}{endpoint_path}", parsed.query, parsed.fragment) + ) + + +def bigmodel_json_body(response: httpx.Response) -> Mapping[str, JsonValue]: + try: + return _JSON_OBJECT.validate_json(response.content) + except ValidationError: + raise BaseLLMException(message=response.text, status_code=response.status_code, headers=response.headers) + + +def is_unsupported_disabled_thinking(model: str, thinking: object) -> bool: + return thinking == {"type": "disabled"} and litellm.utils.supports_factory( + model=model, custom_llm_provider="bigmodel", key="thinking_always_on" + ) + + +def bigmodel_business_error( + code: JsonValue, message: str, model: str, response: httpx.Response | None +) -> ContextWindowExceededError | ContentPolicyViolationError | None: + match code: + case 1261 | "1261": + return ContextWindowExceededError(message=message, model=model, llm_provider="bigmodel", response=response) + case 1301 | "1301": + return ContentPolicyViolationError(message=message, model=model, llm_provider="bigmodel", response=response) + case _: + return None + + +def raise_bigmodel_business_error(model: object, error_str: str, original_exception: object) -> None: + found: Final = _BUSINESS_CODE_IN_ERROR.search(error_str) + response: Final = getattr(original_exception, "response", None) + error: Final = bigmodel_business_error( + found.group(1) if found else None, + error_str, + str(model), + response if isinstance(response, httpx.Response) else None, + ) + if error is not None: + raise error diff --git a/litellm/llms/bigmodel/messages/__init__.py b/litellm/llms/bigmodel/messages/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/bigmodel/messages/transformation.py b/litellm/llms/bigmodel/messages/transformation.py new file mode 100644 index 00000000000..82ce59631f1 --- /dev/null +++ b/litellm/llms/bigmodel/messages/transformation.py @@ -0,0 +1,206 @@ +import json +from collections.abc import AsyncIterator, Mapping, Sequence +from typing import Final + +import httpx +from openai._streaming import ServerSentEvent, SSEDecoder +from pydantic import JsonValue, TypeAdapter, ValidationError + +from litellm.exceptions import AuthenticationError +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.llms.openai_like.messages.transformation import ( + DEFAULT_ANTHROPIC_API_VERSION, + OpenAILikeAnthropicMessagesConfig, +) +from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse +from litellm.types.router import GenericLiteLLMParams + +from ..common_utils import ( + bigmodel_json_body, + get_bigmodel_api_key, + get_bigmodel_url, + is_unsupported_disabled_thinking, +) + +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_OUTPUT_CONFIG: Final = TypeAdapter(dict[str, object]) + + +def _with_lowest_effort_thinking( + params: Mapping[str, object], +) -> dict[str, object]: # mutable-ok: the parent transform takes a request dict + requested_output_config: Final = params.get("output_config") + output_config: Final = { + **({} if "reasoning_effort" in params else {"effort": "low"}), + **( + _OUTPUT_CONFIG.validate_python(requested_output_config) if isinstance(requested_output_config, dict) else {} + ), + } + return {**params, "thinking": {"type": "adaptive"}, **({"output_config": output_config} if output_config else {})} + + +def _normalize_message(body: Mapping[str, JsonValue]) -> dict[str, JsonValue]: # mutable-ok: native JSON response + usage: Final = body.get("usage") + if not isinstance(usage, dict): + return dict(body) + server_tools: Final = usage.get("server_tool_use") + if not isinstance(server_tools, dict) or "web_fetch_requests" in server_tools: + return dict(body) + normalized: Final[JsonValue] = { + **body, + "usage": {**usage, "server_tool_use": {"web_fetch_requests": 0, **server_tools}}, + } + return normalized + + +class _BigModelMessagesStream(httpx.AsyncByteStream): + def __init__(self, response: httpx.Response) -> None: + self._response = response + + async def __aiter__(self) -> AsyncIterator[bytes]: + async for event in SSEDecoder().aiter_bytes(self._response.aiter_bytes()): + yield _normalize_event(event) + + async def aclose(self) -> None: + await self._response.aclose() + + +def _json_object(data: str) -> Mapping[str, JsonValue] | None: + try: + return _JSON_OBJECT.validate_json(data) + except ValidationError: + return None + + +def _normalized_data(data: str) -> str: + body: Final = _json_object(data) + if body is None: + return data + message: Final = body.get("message") + normalized: Final[JsonValue] = ( + {**body, "message": _normalize_message(message)} + if body.get("type") == "message_start" and isinstance(message, dict) + else _normalize_message(body) + ) + return data if normalized == body else json.dumps(normalized, ensure_ascii=False) + + +def _normalize_event(event: ServerSentEvent) -> bytes: + metadata: Final = "".join( + f"{name}: {value}\n" + for name, value in (("event", event.event), ("id", event.id), ("retry", event.retry)) + if value is not None + ) + data_lines: Final = "".join(f"data: {line}\n" for line in _normalized_data(event.data).split("\n")) + return f"{metadata}{data_lines}\n".encode() + + +class BigModelMessagesConfig(OpenAILikeAnthropicMessagesConfig): + @property + def custom_llm_provider(self) -> str: + return "bigmodel" + + def is_native_messages_route(self) -> bool: + return True + + def should_strip_billing_metadata(self) -> bool: + return True + + def transform_anthropic_messages_request( + self, + model: str, + messages: list[dict[str, object]], # mutable-ok: matches dict-typed base signature + anthropic_messages_optional_request_params: dict[str, object], # mutable-ok: matches dict-typed base signature + litellm_params: GenericLiteLLMParams, + headers: dict[str, str], # mutable-ok: matches dict-typed base signature + ) -> dict[str, object]: # mutable-ok: matches dict-typed base signature + request_params: Final = ( + _with_lowest_effort_thinking(anthropic_messages_optional_request_params) + if is_unsupported_disabled_thinking(model, anthropic_messages_optional_request_params.get("thinking")) + else anthropic_messages_optional_request_params + ) + return super().transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the parent transform returns a bare dict + model, messages, request_params, litellm_params, headers + ) + + def transform_anthropic_messages_response( + self, model: str, raw_response: httpx.Response, logging_obj: Logging + ) -> AnthropicMessagesResponse: + normalized: Final = httpx.Response( + raw_response.status_code, + json=_normalize_message(bigmodel_json_body(raw_response)), + request=raw_response.request, + ) + return super().transform_anthropic_messages_response(model, normalized, logging_obj) + + def get_async_streaming_response_iterator( + self, + model: str, + httpx_response: httpx.Response, + request_body: dict[str, object], # mutable-ok: inherited native stream API takes a request dict + litellm_logging_obj: Logging, + ) -> AsyncIterator[bytes]: + normalized: Final = httpx.Response( + httpx_response.status_code, + headers={ + key: value + for key, value in httpx_response.headers.items() + if key.lower() not in ("content-encoding", "content-length") + }, + stream=_BigModelMessagesStream(httpx_response), + request=httpx_response.request, + ) + return super().get_async_streaming_response_iterator( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # native parent yields bytes but its iterator annotation omits the element type + model, normalized, request_body, litellm_logging_obj + ) + + @staticmethod + def _translate_reasoning_effort_to_anthropic( + model: str, optional_params: Mapping[str, object], max_tokens: int | None, custom_llm_provider: str + ) -> None: + return None + + @staticmethod + def _translate_adaptive_effort_for_non_adaptive_model( + model: str, optional_params: Mapping[str, object], max_tokens: int | None, custom_llm_provider: str + ) -> None: + return None + + @staticmethod + def _drop_incompatible_temperature_for_thinking( + model: str, optional_params: Mapping[str, object], custom_llm_provider: str + ) -> None: + return None + + def validate_anthropic_messages_environment( + self, + headers: Mapping[str, str], + model: str, + messages: Sequence[object], + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + api_key: str | None = None, + api_base: str | None = None, + ) -> tuple[dict[str, str], str | None]: # mutable-ok: HTTP handler merges forwarded headers + resolved_key: Final = get_bigmodel_api_key(api_key) + if resolved_key is None: + raise AuthenticationError( + message="Set BIGMODEL_API_KEY or pass api_key", llm_provider="bigmodel", model=model + ) + return { + **{key: value for key, value in headers.items() if key.lower() not in ("x-api-key", "authorization")}, + "x-api-key": resolved_key, + "anthropic-version": DEFAULT_ANTHROPIC_API_VERSION, + "content-type": "application/json", + }, api_base + + def get_complete_url( + self, + api_base: str | None, + api_key: str | None, + model: str, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + stream: bool | None = None, + ) -> str: + return get_bigmodel_url(api_base, "messages") diff --git a/litellm/llms/bigmodel/responses/__init__.py b/litellm/llms/bigmodel/responses/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/bigmodel/responses/transformation.py b/litellm/llms/bigmodel/responses/transformation.py new file mode 100644 index 00000000000..80c53682d16 --- /dev/null +++ b/litellm/llms/bigmodel/responses/transformation.py @@ -0,0 +1,263 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final + +import httpx +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError +from typing_extensions import Never + +from litellm.exceptions import AuthenticationError, UnsupportedParamsError +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.types.llms.openai import ( + ResponseInputParam, + ResponsesAPIOptionalRequestParams, + ResponsesAPIResponse, + ResponsesAPIStreamingResponse, +) +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders + +from ..common_utils import bigmodel_business_error, bigmodel_json_body, get_bigmodel_api_key, get_bigmodel_url + +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_SUPPORTED_PARAMS: Final = ( + "input", + "model", + "instructions", + "stream", + "temperature", + "top_p", + "max_output_tokens", + "tools", + "tool_choice", + "parallel_tool_calls", + "include", + "reasoning", + "text", + "prompt_cache_key", + "previous_response_id", + "store", + "extra_headers", + "extra_query", + "extra_body", + "timeout", +) + + +_STATUS_BY_BUSINESS_CODE: Final = MappingProxyType( + { + **dict.fromkeys((1000, 1001, 1003, 1005), 401), + **dict.fromkeys((1210, 1211, 1212, 1213, 1214, 1215, 1221, 1222, 1261, 1301), 400), + 1220: 403, + **dict.fromkeys( + (1113, 1302, 1305, 1308, 1309, 1310, 1311, 1313, 1314, 1315, 1316, 1317, 1318, 1319, 1320, 1321), + 429, + ), + } +) + + +class _RequestedTools(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + tools: JsonValue = None + tool_choice: JsonValue = None + parallel_tool_calls: JsonValue = None + + +def _requested_tools(logging_obj: Logging) -> _RequestedTools: + try: + return _RequestedTools.model_validate(logging_obj.model_call_details.get("optional_params") or {}) + except ValidationError: + return _RequestedTools() + + +def _envelope_status_code(code: JsonValue) -> int: + if not isinstance(code, int) or isinstance(code, bool): + return 500 + if 400 <= code <= 599: + return code + return _STATUS_BY_BUSINESS_CODE.get(code, 500) + + +def _normalize_output_item(item: JsonValue) -> JsonValue: + if not isinstance(item, dict) or item.get("type") not in ("message", "reasoning"): + return item + content: Final = item.get("content") + normalized: Final[JsonValue] = {"summary": [], **item} if item["type"] == "reasoning" else item + if not isinstance(content, dict): + return normalized + return {**normalized, "content": [content]} + + +def _normalize_response( + body: Mapping[str, JsonValue], + requested: _RequestedTools, +) -> dict[str, JsonValue]: # mutable-ok: JSON serialization boundary + usage: Final = body.get("usage") + output: Final = body.get("output") + tools: Final = body.get("tools") + tool_choice: Final = body.get("tool_choice") + parallel_tool_calls: Final = body.get("parallel_tool_calls") + input_tokens: Final = usage.get("input_tokens") if isinstance(usage, dict) else None + output_tokens: Final = usage.get("output_tokens") if isinstance(usage, dict) else None + normalized_usage: Final = ( + { + "total_tokens": input_tokens + output_tokens, + **usage, + } + if isinstance(usage, dict) and isinstance(input_tokens, int) and isinstance(output_tokens, int) + else usage + ) + normalized: Final[JsonValue] = { + **body, + "parallel_tool_calls": ( + requested.parallel_tool_calls is not False if parallel_tool_calls is None else parallel_tool_calls + ), + "tools": (requested.tools or []) if tools is None else tools, + "tool_choice": (requested.tool_choice or "auto") if tool_choice is None else tool_choice, + **({"usage": normalized_usage} if "usage" in body else {}), + **({"output": [_normalize_output_item(item) for item in output]} if isinstance(output, list) else {}), + } + return normalized + + +def _normalize_stream_part(part: JsonValue) -> JsonValue: + if not isinstance(part, dict): + return part + if part.get("type") == "output_text": + return { + "annotations": [], + "logprobs": None, + **part, + } + if part.get("type") == "reasoning_text": + return {"reasoning": part.get("text", ""), **part} + return part + + +class BigModelResponsesAPIConfig(OpenAIResponsesAPIConfig): + @property + def custom_llm_provider(self) -> LlmProviders: + return LlmProviders.BIGMODEL + + def supports_native_file_search(self) -> bool: + return False + + def supports_native_websocket(self) -> bool: + return False + + def should_fake_stream( + self, model: str | None, stream: bool | None, custom_llm_provider: str | None = None + ) -> bool: + return False + + def may_answer_stream_with_json(self) -> bool: + return True + + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: inherited API requires a list + return list(_SUPPORTED_PARAMS) + + def map_openai_params( + self, + response_api_optional_params: ResponsesAPIOptionalRequestParams, + model: str, + drop_params: bool, + ) -> dict[str, object]: # mutable-ok: request pipeline adds parameters to the returned body + return {key: value for key, value in response_api_optional_params.items() if key in _SUPPORTED_PARAMS} + + def validate_environment( + self, headers: Mapping[str, str], model: str, litellm_params: GenericLiteLLMParams | None + ) -> dict[str, str]: # mutable-ok: HTTP handler merges forwarded headers + resolved_key: Final = get_bigmodel_api_key(litellm_params.api_key if litellm_params else None) + if resolved_key is None: + raise AuthenticationError( + message="Set BIGMODEL_API_KEY or pass api_key", llm_provider="bigmodel", model=model + ) + return { + **headers, + "Authorization": f"Bearer {resolved_key}", + "Content-Type": "application/json", + } + + def get_complete_url(self, api_base: str | None, litellm_params: Mapping[str, object]) -> str: + return get_bigmodel_url(api_base, "responses") + + def transform_response_api_response( + self, model: str, raw_response: httpx.Response, logging_obj: Logging + ) -> ResponsesAPIResponse: + body: Final = bigmodel_json_body(raw_response) + if body.get("success") is False: + message: Final = str(body.get("msg") or raw_response.text) + raise bigmodel_business_error(body.get("code"), message, model, None) or BaseLLMException( + message=message, + status_code=_envelope_status_code(body.get("code")), + headers=raw_response.headers, + ) + normalized: Final = _normalize_response(body, _requested_tools(logging_obj)) + normalized_response: Final = httpx.Response( + status_code=raw_response.status_code, + headers={ + key: value + for key, value in raw_response.headers.items() + if key.lower() not in ("content-encoding", "content-length") + }, + json=normalized, + request=raw_response.request, + ) + return super().transform_response_api_response(model, normalized_response, logging_obj) + + def transform_get_response_api_response( + self, raw_response: httpx.Response, logging_obj: Logging + ) -> ResponsesAPIResponse: + return self.transform_response_api_response("", raw_response, logging_obj) + + def transform_streaming_response( + self, model: str, parsed_chunk: Mapping[str, object], logging_obj: Logging + ) -> ResponsesAPIStreamingResponse: + chunk: Final = _JSON_OBJECT.validate_python(parsed_chunk) + response: Final = chunk.get("response") + item: Final = chunk.get("item") + normalized: Final[JsonValue] = { + **chunk, + **( + {"response": _normalize_response(response, _requested_tools(logging_obj))} + if isinstance(response, dict) + else {} + ), + **({"item": _normalize_output_item(item)} if "item" in chunk else {}), + } + event_model: Final = self.get_event_model_class(str(normalized.get("type"))) + match chunk.get("type"): + case "response.output_text.done": + return event_model.model_validate({"logprobs": [], **normalized}) + case "response.content_part.done": + return event_model.model_validate({**normalized, "part": _normalize_stream_part(chunk.get("part"))}) + case "error": + error: Final = chunk.get("error") + error_fields: Final = error if isinstance(error, dict) else chunk + normalized_error: Final[JsonValue] = { + **normalized, + "error": {"type": "api_error", **error_fields, "code": error_fields.get("code") or "unknown_error"}, + } + return event_model.model_validate(normalized_error) + case _: + return event_model.model_validate(normalized) + + def transform_cancel_response_api_request( + self, response_id: str, api_base: str, litellm_params: GenericLiteLLMParams, headers: Mapping[str, str] + ) -> Never: + raise UnsupportedParamsError(message="BigModel does not support cancelling responses", status_code=400) + + def transform_compact_response_api_request( + self, + model: str, + input: str | ResponseInputParam, + response_api_optional_request_params: Mapping[str, object], + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: Mapping[str, str], + ) -> Never: + raise UnsupportedParamsError(message="BigModel does not support compacting responses", status_code=400) diff --git a/litellm/llms/cohere/embed/transformation.py b/litellm/llms/cohere/embed/transformation.py index 590164ffa0f..1477e45997b 100644 --- a/litellm/llms/cohere/embed/transformation.py +++ b/litellm/llms/cohere/embed/transformation.py @@ -10,11 +10,13 @@ Convers Docs - https://docs.cohere.com/v2/reference/embed """ -from collections.abc import Iterable, Mapping, Sized -from typing import Final, Protocol, cast +from collections.abc import Iterable, Iterator, Mapping, Sequence, Sized +from enum import Enum +from itertools import chain, starmap +from typing import Final, Protocol, TypeAlias, cast import httpx -from pydantic import ConfigDict, TypeAdapter +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter import litellm from litellm import COHERE_DEFAULT_EMBEDDING_INPUT_TYPE @@ -32,14 +34,117 @@ from litellm.utils import is_base64_encoded from ..common_utils import CohereError _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) -_EMBEDDINGS_BY_TYPE: Final = TypeAdapter(Mapping[str, Iterable[object]], config=ConfigDict(hide_input_in_errors=True)) _DECODED_JSON: Final = TypeAdapter(object) class _SupportsEncode(Protocol): """Tokenizer handle: the embedding usage path only encodes text to measure its token length.""" - def encode(self, text: str, /) -> Sized: ... + def encode_ordinary(self, text: str, /) -> Sized: ... + + +class CohereImageTokenBilling(Enum): + SEPARATE_FROM_INPUT_TOKENS = "separate_from_input_tokens" + FOLDED_INTO_INPUT_TOKENS = "folded_into_input_tokens" + + +class CohereEmbeddingRowShape(Enum): + OPENAI = "openai" + TYPED = "typed" + + +class _CohereBilledUnits(BaseModel): + model_config = ConfigDict(frozen=True, hide_input_in_errors=True) + + input_tokens: int | None = None + image_tokens: int | None = None + images: int | None = None + + +class _CohereEmbedMeta(BaseModel): + model_config = ConfigDict(frozen=True, hide_input_in_errors=True) + + billed_units: _CohereBilledUnits = Field(default_factory=_CohereBilledUnits) + + +_CohereEmbeddings: TypeAlias = Mapping[str, Sequence[object]] | Sequence[object] + + +class _CohereEmbedBody(BaseModel): + model_config = ConfigDict(frozen=True, hide_input_in_errors=True) + + embeddings: _CohereEmbeddings + meta: _CohereEmbedMeta = Field(default_factory=_CohereEmbedMeta) + + +def _embedding_rows(vectors: Sequence[object]) -> Iterator[Mapping[str, object]]: + return ({"object": "embedding", "index": index, "embedding": vector} for index, vector in enumerate(vectors)) + + +def _typed_embedding_rows(embedding_type: str, vectors: Sequence[object]) -> Iterator[Mapping[str, object]]: + return ({**row, "type": embedding_type} for row in _embedding_rows(vectors)) + + +def _response_rows(embeddings: _CohereEmbeddings, row_shape: CohereEmbeddingRowShape) -> Iterable[Mapping[str, object]]: + if not isinstance(embeddings, Mapping): + return _embedding_rows(embeddings) + match row_shape: + case CohereEmbeddingRowShape.OPENAI: + return chain.from_iterable(map(_embedding_rows, embeddings.values())) + case CohereEmbeddingRowShape.TYPED: + return chain.from_iterable(starmap(_typed_embedding_rows, embeddings.items())) + + +def _prompt_usage(prompt_tokens: int, prompt_tokens_details: PromptTokensDetailsWrapper | None) -> Usage: + return Usage( + prompt_tokens=prompt_tokens, + completion_tokens=0, + total_tokens=prompt_tokens, + prompt_tokens_details=prompt_tokens_details, + ) + + +def _billed_usage(billed_units: _CohereBilledUnits, image_billing: CohereImageTokenBilling) -> Usage: + match image_billing: + case CohereImageTokenBilling.SEPARATE_FROM_INPUT_TOKENS: + image_units: Final = (billed_units.image_tokens or 0) + (billed_units.images or 0) + return _prompt_usage( + (billed_units.input_tokens or 0) + image_units, + PromptTokensDetailsWrapper( + text_tokens=billed_units.input_tokens, + image_tokens=image_units or None, + image_count=billed_units.images, + ), + ) + case CohereImageTokenBilling.FOLDED_INTO_INPUT_TOKENS: + return _prompt_usage(billed_units.input_tokens or 0, None) + + +def _cohere_embed_usage( + billed_units: _CohereBilledUnits, + input_texts: Iterable[str], + encoding: _SupportsEncode, + image_billing: CohereImageTokenBilling, +) -> Usage: + if billed_units.input_tokens is None and billed_units.image_tokens is None and billed_units.images is None: + return _prompt_usage(sum(len(encoding.encode_ordinary(text)) for text in input_texts), None) + return _billed_usage(billed_units, image_billing) + + +def cohere_embedding_response( + body: Mapping[str, object], + model: str | None, + encoding: _SupportsEncode, + input_texts: Iterable[str], + image_billing: CohereImageTokenBilling, + row_shape: CohereEmbeddingRowShape, +) -> EmbeddingResponse: + cohere_body: Final = _CohereEmbedBody.model_validate(body) + return EmbeddingResponse( + model=model, + data=list(_response_rows(cohere_body.embeddings, row_shape)), + usage=_cohere_embed_usage(cohere_body.meta.billed_units, input_texts, encoding, image_billing), + ) class CohereEmbeddingConfig(BaseEmbeddingConfig): @@ -145,44 +250,16 @@ class CohereEmbeddingConfig(BaseEmbeddingConfig): ), ) - def _calculate_usage(self, input: list[str], encoding: _SupportsEncode, meta: dict) -> Usage: - input_tokens = 0 - - text_tokens: Final[int | None] = meta.get("billed_units", {}).get("input_tokens") - - image_tokens: Final[int | None] = meta.get("billed_units", {}).get("images") - - prompt_tokens_details: PromptTokensDetailsWrapper | None = None - if image_tokens is None and text_tokens is None: - for text in input: - input_tokens += len(encoding.encode(text)) - else: - prompt_tokens_details = PromptTokensDetailsWrapper( - image_tokens=image_tokens, - text_tokens=text_tokens, - ) - if image_tokens: - input_tokens += image_tokens - if text_tokens: - input_tokens += text_tokens - - return Usage( - prompt_tokens=input_tokens, - completion_tokens=0, - total_tokens=input_tokens, - prompt_tokens_details=prompt_tokens_details, - ) - - def _transform_response( + def transform_native_embed_response( self, response: httpx.Response, api_key: str | None, logging_obj: LiteLLMLoggingObj, - data: dict | CohereEmbeddingRequest, - model_response: EmbeddingResponse, + data: Mapping[str, object] | CohereEmbeddingRequest, model: str, encoding: _SupportsEncode, - input: list, + input: Sequence[str], + row_shape: CohereEmbeddingRowShape, ) -> EmbeddingResponse: response_json: Final = _DECODED_JSON.validate_python(response.json()) ## LOGGING @@ -192,38 +269,15 @@ class CohereEmbeddingConfig(BaseEmbeddingConfig): additional_args={"complete_input_dict": data}, original_response=response_json, ) - """ - response - { - 'object': "list", - 'data': [ - - ] - 'model', - 'usage' - } - """ - payload: Final = _JSON_OBJECT.validate_python(response_json) - embeddings: Final = _EMBEDDINGS_BY_TYPE.validate_python(payload["embeddings"]) - output_data: Final = [] - for k, embedding_list in embeddings.items(): - for idx, embedding in enumerate(embedding_list): - output_data.append({"object": "embedding", "index": idx, "embedding": embedding}) - model_response.object = "list" - model_response.data = output_data - model_response.model = model - input_tokens = 0 - for text in input: - input_tokens += len(encoding.encode(text)) - - setattr( - model_response, - "usage", - self._calculate_usage(input, encoding, dict(_JSON_OBJECT.validate_python(payload.get("meta", {})))), + return cohere_embedding_response( + _JSON_OBJECT.validate_python(response_json), + model, + encoding, + input, + CohereImageTokenBilling.SEPARATE_FROM_INPUT_TOKENS, + row_shape, ) - return model_response - def transform_embedding_response( self, model: str, @@ -235,15 +289,15 @@ class CohereEmbeddingConfig(BaseEmbeddingConfig): optional_params: dict, litellm_params: dict, ) -> EmbeddingResponse: - return self._transform_response( + return self.transform_native_embed_response( response=raw_response, api_key=api_key, logging_obj=logging_obj, data=request_data, - model_response=model_response, model=model, encoding=litellm.encoding, input=logging_obj.model_call_details["input"], + row_shape=CohereEmbeddingRowShape.OPENAI, ) def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException: diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 0b64885df45..54adfb5c692 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -764,18 +764,23 @@ class AsyncHTTPHandler: follow_redirects: bool | None = None, timeout: float | httpx.Timeout | None = None, max_response_bytes: int | None = None, + stream: bool = False, ): # Set follow_redirects to UseClientDefault if None _follow_redirects: Final = follow_redirects if follow_redirects is not None else USE_CLIENT_DEFAULT + query: Final = httpx.QueryParams(params).merge(HTTPHandler.extract_query_params(url)) + request_headers: Final = httpx.Headers(headers) - params = params or {} - params.update(HTTPHandler.extract_query_params(url)) + if stream: + return await self._send_streaming_get( + url, params=query, headers=request_headers, follow_redirects=follow_redirects, timeout=timeout + ) if max_response_bytes is not None: return await self._get_with_response_limit( url, - params=httpx.QueryParams(params), - headers=httpx.Headers(headers), + params=query, + headers=request_headers, max_bytes=max_response_bytes, follow_redirects=self.client.follow_redirects if follow_redirects is None else follow_redirects, timeout=self.client.timeout if timeout is None else httpx.Timeout(timeout), @@ -783,13 +788,41 @@ class AsyncHTTPHandler: response: Final = await self.client.get( url, - params=params, - headers=headers, + params=query, + headers=request_headers, follow_redirects=_follow_redirects, timeout=timeout if timeout is not None else USE_CLIENT_DEFAULT, ) return response + async def _send_streaming_get( + self, + url: str, + *, + params: httpx.QueryParams, + headers: httpx.Headers, + follow_redirects: bool | None, + timeout: float | httpx.Timeout | None, + ) -> httpx.Response: + start_time: Final = time.time() + request: Final = self.client.build_request( + "GET", url, params=params, headers=headers, timeout=USE_CLIENT_DEFAULT if timeout is None else timeout + ) + try: + response: Final = await self.client.send( + request, + stream=True, + follow_redirects=USE_CLIENT_DEFAULT if follow_redirects is None else follow_redirects, + ) + _anchor_handler_to(response, self) + response.raise_for_status() + except httpx.TimeoutException as e: + _raise_timeout(e, start_time) + except httpx.HTTPStatusError as e: + await _raise_masked_async_error(e, True) + else: + return response + async def _get_with_response_limit( self, url: str, @@ -1468,15 +1501,20 @@ class HTTPHandler: headers: Mapping[str, Any] | None = None, follow_redirects: bool | None = None, timeout: float | httpx.Timeout | None = None, + stream: bool = False, ): # Set follow_redirects to UseClientDefault if None _follow_redirects: Final = follow_redirects if follow_redirects is not None else USE_CLIENT_DEFAULT - params = params or {} - params.update(self.extract_query_params(url)) + query: Final = httpx.QueryParams(params).merge(self.extract_query_params(url)) + + if stream: + return self._send_streaming_get( + url, params=query, headers=headers, follow_redirects=follow_redirects, timeout=timeout + ) response: Final = self.client.get( url, - params=params, + params=query, headers=headers, follow_redirects=_follow_redirects, timeout=timeout if timeout is not None else USE_CLIENT_DEFAULT, @@ -1484,6 +1522,34 @@ class HTTPHandler: return response + def _send_streaming_get( + self, + url: str, + *, + params: httpx.QueryParams, + headers: Mapping[str, str] | None, + follow_redirects: bool | None, + timeout: float | httpx.Timeout | None, + ) -> httpx.Response: + start_time: Final = time.time() + request: Final = self.client.build_request( + "GET", url, params=params, headers=headers, timeout=USE_CLIENT_DEFAULT if timeout is None else timeout + ) + try: + response: Final = self.client.send( + request, + stream=True, + follow_redirects=USE_CLIENT_DEFAULT if follow_redirects is None else follow_redirects, + ) + _anchor_handler_to(response, self) + response.raise_for_status() + except httpx.TimeoutException as e: + _raise_timeout(e, start_time) + except httpx.HTTPStatusError as e: + _raise_masked_sync_error(e, True) + else: + return response + @staticmethod def extract_query_params(url: str) -> dict[str, str]: """ diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index e7da6586b97..c970b507288 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -50,7 +50,11 @@ from litellm.litellm_core_utils.audio_utils.subtitle_utils import ( SUBTITLE_RESPONSE_FORMATS, synthesize_subtitle_document, ) -from litellm.litellm_core_utils.core_helpers import as_str_mapping, set_provider_response_headers_in_hidden_params +from litellm.litellm_core_utils.core_helpers import ( + as_str_mapping, + qualify_agentic_followup_model, + set_provider_response_headers_in_hidden_params, +) from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header @@ -367,12 +371,37 @@ class _SignedBodyKwargs(TypedDict, total=False): json: ReadOnly[dict[str, object]] +def _is_sized_json_body(response: httpx.Response) -> bool: + headers: Final = httpx.Headers(response.headers) + media_types: Final = (value.split(";")[0].strip().lower() for value in headers.get_list("content-type")) + return "content-length" in headers and "application/json" in media_types + + +def _is_json_document(content: bytes) -> bool: + try: + return isinstance(json.loads(content), dict) + except ValueError: + return False + + def _signed_body_kwargs(*, signed_body: bytes | None, data: dict[str, object]) -> _SignedBodyKwargs: if signed_body is not None: return {"data": signed_body} return {"json": data} +def _answered_json(response: httpx.Response) -> bool: + return "application/json" in response.headers.get("content-type", "").lower() + + +def _stream_query_params(stream: bool, starting_after: int | None) -> Mapping[str, object]: + if not stream: + return {} + if starting_after is None: + return {"stream": "true"} + return {"stream": "true", "starting_after": starting_after} + + def _google_genai_streaming_hidden_params( *, api_base: str, @@ -2552,6 +2581,7 @@ class BaseLLMHTTPHandler: "preset_cache_key": None, "stream_response": {}, "model_info": kwargs.get("model_info"), + **litellm_params.per_second_pricing(), **vertex_location_params, **anthropic_messages_optional_request_params, }, @@ -3025,6 +3055,17 @@ class BaseLLMHTTPHandler: }, ) + replay_as_stream: Final = partial( + MockResponsesAPIStreamingIterator, + model=model, + logging_obj=logging_obj, + responses_api_provider_config=responses_api_provider_config, + litellm_metadata=litellm_metadata, + custom_llm_provider=custom_llm_provider, + request_data=request_context, + call_type=CallTypes.responses.value, + ) + try: if is_stream_request: response = sync_httpx_client.post( @@ -3036,7 +3077,13 @@ class BaseLLMHTTPHandler: ) if caller_requested_stream: if fake_stream is True: - return MockResponsesAPIStreamingIterator( + return replay_as_stream(response=response) + if not ( + responses_api_provider_config.may_answer_stream_with_json() + and _is_sized_json_body(response) + and _is_json_document(response.read()) + ): + return SyncResponsesAPIStreamingIterator( response=response, model=model, logging_obj=logging_obj, @@ -3046,17 +3093,6 @@ class BaseLLMHTTPHandler: request_data=request_context, call_type=CallTypes.responses.value, ) - - return SyncResponsesAPIStreamingIterator( - response=response, - model=model, - logging_obj=logging_obj, - responses_api_provider_config=responses_api_provider_config, - litellm_metadata=litellm_metadata, - custom_llm_provider=custom_llm_provider, - request_data=request_context, - call_type=CallTypes.responses.value, - ) response.read() else: response = sync_httpx_client.post( @@ -3071,6 +3107,9 @@ class BaseLLMHTTPHandler: provider_config=responses_api_provider_config, ) + if is_stream_request and caller_requested_stream: + return replay_as_stream(response=response) + initial_response: Final = responses_api_provider_config.transform_response_api_response( model=model, raw_response=response, @@ -3226,6 +3265,17 @@ class BaseLLMHTTPHandler: }, ) + replay_as_stream: Final = partial( + MockResponsesAPIStreamingIterator, + model=model, + logging_obj=logging_obj, + responses_api_provider_config=responses_api_provider_config, + litellm_metadata=litellm_metadata, + custom_llm_provider=custom_llm_provider, + request_data=request_context, + call_type=CallTypes.responses.value, + ) + try: if is_stream_request: response = await async_httpx_client.post( @@ -3239,7 +3289,13 @@ class BaseLLMHTTPHandler: if caller_requested_stream: if fake_stream is True: - return MockResponsesAPIStreamingIterator( + return replay_as_stream(response=response) + if not ( + responses_api_provider_config.may_answer_stream_with_json() + and _is_sized_json_body(response) + and _is_json_document(await response.aread()) + ): + return ResponsesAPIStreamingIterator( response=response, model=model, logging_obj=logging_obj, @@ -3249,17 +3305,6 @@ class BaseLLMHTTPHandler: request_data=request_context, call_type=CallTypes.responses.value, ) - - return ResponsesAPIStreamingIterator( - response=response, - model=model, - logging_obj=logging_obj, - responses_api_provider_config=responses_api_provider_config, - litellm_metadata=litellm_metadata, - custom_llm_provider=custom_llm_provider, - request_data=request_context, - call_type=CallTypes.responses.value, - ) await response.aread() else: response = await async_httpx_client.post( @@ -3276,6 +3321,9 @@ class BaseLLMHTTPHandler: provider_config=responses_api_provider_config, ) + if is_stream_request and caller_requested_stream: + return replay_as_stream(response=response) + initial_response: Final = responses_api_provider_config.transform_response_api_response( model=model, raw_response=response, @@ -3493,12 +3541,20 @@ class BaseLLMHTTPHandler: logging_obj: LiteLLMLoggingObj, custom_llm_provider: str | None = None, extra_headers: dict[str, object] | None = None, + extra_query: Mapping[str, object] | None = None, extra_body: dict[str, object] | None = None, + stream: bool = False, + starting_after: int | None = None, + litellm_metadata: Mapping[str, object] = MappingProxyType({}), timeout: float | httpx.Timeout | None = None, client: HTTPHandler | AsyncHTTPHandler | None = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> ResponsesAPIResponse | Coroutine[object, object, ResponsesAPIResponse]: + ) -> ( + ResponsesAPIResponse + | SyncResponsesAPIStreamingIterator + | Coroutine[object, object, ResponsesAPIResponse | ResponsesAPIStreamingIterator] + ): """ Get a response by ID Uses GET /v1/responses/{response_id} endpoint in the responses API @@ -3511,7 +3567,11 @@ class BaseLLMHTTPHandler: logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, extra_headers=extra_headers, + extra_query=extra_query, extra_body=extra_body, + stream=stream, + starting_after=starting_after, + litellm_metadata=litellm_metadata, timeout=timeout, client=client, shared_session=shared_session, @@ -3540,20 +3600,23 @@ class BaseLLMHTTPHandler: litellm_params=litellm_params, headers=headers, ) + query: Final = {**data, **(extra_query or {}), **_stream_query_params(stream, starting_after)} ## LOGGING logging_obj.pre_call( input="", api_key="", additional_args={ - "complete_input_dict": data, + "complete_input_dict": query, "api_base": api_base, "headers": headers, }, ) try: - response: Final = sync_httpx_client.get(url=url, headers=headers, params=data) + response: Final = sync_httpx_client.get( + url=url, headers=headers, params=query, stream=stream, timeout=timeout if stream else None + ) response.raise_for_status() except Exception as e: raise self._handle_error( @@ -3561,6 +3624,20 @@ class BaseLLMHTTPHandler: provider_config=responses_api_provider_config, ) + if stream and not _answered_json(response): + return SyncResponsesAPIStreamingIterator( + response=response, + model="", + logging_obj=logging_obj, + responses_api_provider_config=responses_api_provider_config, + litellm_metadata=dict(litellm_metadata), + custom_llm_provider=custom_llm_provider, + request_data={"litellm_params": dict(litellm_params)}, + replays_stored_stream=True, + ) + + if stream: + response.read() return responses_api_provider_config.transform_get_response_api_response( raw_response=response, logging_obj=logging_obj, @@ -3574,11 +3651,15 @@ class BaseLLMHTTPHandler: logging_obj: LiteLLMLoggingObj, custom_llm_provider: str | None = None, extra_headers: dict[str, object] | None = None, + extra_query: Mapping[str, object] | None = None, extra_body: dict[str, object] | None = None, + stream: bool = False, + starting_after: int | None = None, + litellm_metadata: Mapping[str, object] = MappingProxyType({}), timeout: float | httpx.Timeout | None = None, client: HTTPHandler | AsyncHTTPHandler | None = None, shared_session: Optional["ClientSession"] = None, - ) -> ResponsesAPIResponse: + ) -> ResponsesAPIResponse | ResponsesAPIStreamingIterator: """ Async version of get_responses """ @@ -3616,20 +3697,23 @@ class BaseLLMHTTPHandler: litellm_params=litellm_params, headers=headers, ) + query: Final = {**data, **(extra_query or {}), **_stream_query_params(stream, starting_after)} ## LOGGING logging_obj.pre_call( input="", api_key="", additional_args={ - "complete_input_dict": data, + "complete_input_dict": query, "api_base": api_base, "headers": headers, }, ) try: - response: Final = await async_httpx_client.get(url=url, headers=headers, params=data) + response: Final = await async_httpx_client.get( + url=url, headers=headers, params=query, stream=stream, timeout=timeout if stream else None + ) response.raise_for_status() except Exception as e: verbose_logger.debug("Error retrieving response: %s", e) @@ -3638,6 +3722,20 @@ class BaseLLMHTTPHandler: provider_config=responses_api_provider_config, ) + if stream and not _answered_json(response): + return ResponsesAPIStreamingIterator( + response=response, + model="", + logging_obj=logging_obj, + responses_api_provider_config=responses_api_provider_config, + litellm_metadata=dict(litellm_metadata), + custom_llm_provider=custom_llm_provider, + request_data={"litellm_params": dict(litellm_params)}, + replays_stored_stream=True, + ) + + if stream: + await response.aread() return responses_api_provider_config.transform_get_response_api_response( raw_response=response, logging_obj=logging_obj, @@ -6182,9 +6280,7 @@ class BaseLLMHTTPHandler: if patch.messages is None: raise ValueError("Agentic loop plan missing patched messages") - full_model_name = patch.model or model - if "/" not in full_model_name: - full_model_name = f"{custom_llm_provider}/{full_model_name}" + full_model_name: Final = qualify_agentic_followup_model(patch.model, model, custom_llm_provider) optional_params_for_followup: Final = dict(optional_params) optional_params_for_followup.update(patch.optional_params) @@ -6827,7 +6923,7 @@ class BaseLLMHTTPHandler: try: ssl_context: Final = realtime_ssl_for_url(url) provider_backend: Final = await provider_config.open_backend(url, headers) - backend_ws: Final = ( + backend_ws: Final = provider_config.wrap_backend( provider_backend if provider_backend is not None else await self._open_realtime_backend_ws(websockets, url, headers, ssl_context) @@ -6865,9 +6961,7 @@ class BaseLLMHTTPHandler: await backend_ws.send(_session_config) realtime_streaming.session_configuration_request = _session_config - # For providers that defer setup until client session.update, optionally - # send synthetic session.created to unblock clients waiting on connect. - if not provider_config.requires_session_configuration(): + if not _session_config: synthetic_session: Final = provider_config.transform_session_created_event( model=model, logging_session_id=logging_obj.litellm_trace_id, @@ -11056,7 +11150,7 @@ class BaseLLMHTTPHandler: litellm_params=dict(litellm_params), ) - url: Final = api_base + url: Final = vector_store_provider_config.get_vector_store_list_url(api_base) params: Final[dict[str, object]] = {} if after is not None: @@ -11135,7 +11229,7 @@ class BaseLLMHTTPHandler: litellm_params=dict(litellm_params), ) - url: Final = api_base + url: Final = vector_store_provider_config.get_vector_store_list_url(api_base) params: Final[dict[str, object]] = {} if after is not None: diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index 48118c23ed7..60081c38073 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, cast, overload import httpx from pydantic import BaseModel, ConfigDict, TypeAdapter +from litellm.anthropic_beta_gated_fields import log_beta_gated_strips, strip_beta_gated_thinking from litellm.constants import RESPONSE_FORMAT_TOOL_NAME from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( handle_invalid_parallel_tool_calls, @@ -514,6 +515,14 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): self.update_optional_params_with_thinking_tokens( non_default_params=non_default_params, optional_params=mapped_params ) + gated_thinking: Final = strip_beta_gated_thinking( + mapped_params.get("thinking"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # mapped_params is the base config's bare dict + frozenset(), + drop_params=drop_params, + ) + if gated_thinking.stripped: + log_beta_gated_strips(gated_thinking.stripped, model=model, provider="databricks") + mapped_params["thinking"] = gated_thinking.thinking return mapped_params diff --git a/litellm/llms/databricks/messages/transformation.py b/litellm/llms/databricks/messages/transformation.py index e22ad85b5f9..90b63d860d3 100644 --- a/litellm/llms/databricks/messages/transformation.py +++ b/litellm/llms/databricks/messages/transformation.py @@ -144,7 +144,7 @@ class DatabricksAnthropicMessagesConfig(DatabricksBase, AnthropicMessagesConfig) ("content-type" if key.lower() == "content-type" else key): value for key, value in {**auth_headers, **client_anthropic_headers}.items() } - finalized_headers: Final[dict[str, str]] = self._finalize_messages_headers( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the shared helper returns a bare dict # mutable-ok: becomes the outbound headers + finalized_headers: Final[dict[str, str]] = self.finalize_messages_headers( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the shared helper returns a bare dict # mutable-ok: becomes the outbound headers headers=normalized_headers, optional_params=optional_params, messages=messages, @@ -191,6 +191,8 @@ class DatabricksAnthropicMessagesConfig(DatabricksBase, AnthropicMessagesConfig) litellm_params=litellm_params, headers=headers, ) - self._normalize_system_role_messages(anthropic_messages_request, model=model) # pyright: ignore[reportUnknownMemberType] # the shared helper takes a bare dict + self._normalize_system_role_messages( # pyright: ignore[reportUnknownMemberType] # the shared helper takes a bare dict + anthropic_messages_request, model=model, drop_params=getattr(litellm_params, "drop_params", None) + ) self._remove_scope_from_cache_control(anthropic_messages_request) # pyright: ignore[reportUnknownMemberType] # the shared helper takes a bare dict return anthropic_messages_request diff --git a/litellm/llms/fireworks_ai/responses/transformation.py b/litellm/llms/fireworks_ai/responses/transformation.py index db234c09930..86d9df9618a 100644 --- a/litellm/llms/fireworks_ai/responses/transformation.py +++ b/litellm/llms/fireworks_ai/responses/transformation.py @@ -1,10 +1,9 @@ -from collections.abc import Mapping, Sequence +from collections.abc import Mapping from types import MappingProxyType from typing import TYPE_CHECKING, Final from urllib.parse import unquote import httpx -from openai.types.responses import EasyInputMessageParam, ResponseInputContentParam, ResponseInputItemParam from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.fireworks_ai.common_utils import ( @@ -15,12 +14,18 @@ from litellm.llms.fireworks_ai.common_utils import ( with_fireworks_session_affinity, without_caller_user, ) +from litellm.llms.openai.responses.instruction_items import developer_items_as_system, instruction_items_folded from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ResponseInputParam from litellm.types.responses.main import DeleteResponseResult from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders +from litellm.utils import ( + declared_mid_conversation_system_support, + deployment_declared_mid_conversation_system_support, + deployment_model_info, +) if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -45,90 +50,6 @@ def _forwarded_user_params(litellm_params: GenericLiteLLMParams) -> Mapping[str, ) -_INSTRUCTION_ROLES: Final = frozenset({"system", "developer"}) - - -def _role(item: ResponseInputItemParam) -> str | None: - match item: - case {"role": str(role)}: - return role - case _: - return None - - -def _developer_item_as_system(item: ResponseInputItemParam) -> ResponseInputItemParam: - if "role" not in item or item["role"] != "developer": - return item - return EasyInputMessageParam(role="system", content=item["content"], type="message") - - -def _developer_items_as_system(input: str | ResponseInputParam) -> str | ResponseInputParam: - if isinstance(input, str): - return input - return [_developer_item_as_system(item) for item in input] - - -def _text_part(part: ResponseInputContentParam) -> str | None: - match part: - case {"type": "input_text", "text": str(text)}: - return text - case _: - return None - - -def _text_only_content(item: ResponseInputItemParam) -> str | None: - match item: - case {"role": "system" | "developer", "content": str(text)}: - return text - case {"role": "system" | "developer", "content": [*parts]}: - texts: Final = tuple(map(_text_part, parts)) - return None if any(text is None for text in texts) else "\n\n".join(text for text in texts if text) - case _: - return None - - -def _leading_instruction_block_length(roles: Sequence[str | None]) -> int: - return next((index for index, role in enumerate(roles) if role not in _INSTRUCTION_ROLES), len(roles)) - - -def _closing_instruction_block_start(roles: Sequence[str | None], leading_length: int) -> int: - last_conversation_index: Final = next( - (index for index in range(len(roles) - 1, leading_length - 1, -1) if roles[index] not in _INSTRUCTION_ROLES), - None, - ) - if last_conversation_index is None or roles[last_conversation_index] != "assistant": - return len(roles) - return last_conversation_index + 1 - - -def _hoisted_indices(roles: Sequence[str | None]) -> tuple[int, ...]: - leading_length: Final = _leading_instruction_block_length(roles) - closing_start: Final = _closing_instruction_block_start(roles, leading_length) - return tuple( - index for index, role in enumerate(roles[:closing_start]) if index < leading_length or role == "developer" - ) - - -def _with_instruction_items_folded( - input: str | ResponseInputParam, instructions: str | None -) -> tuple[str | None, str | ResponseInputParam]: - if isinstance(input, str): - return instructions, input - items: Final = tuple(input) - folded: Final = MappingProxyType( - { - index: text - for index in _hoisted_indices(tuple(map(_role, items))) - if (text := _text_only_content(items[index])) is not None - } - ) - joined: Final = "\n\n".join(chunk for chunk in (instructions, *folded.values()) if chunk) - return ( - instructions if not folded else joined or None, - [_developer_item_as_system(item) for index, item in enumerate(items) if index not in folded], - ) - - class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig): @property def custom_llm_provider(self) -> LlmProviders: @@ -158,29 +79,28 @@ class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig): self, model: str, input: str | ResponseInputParam, - response_api_optional_request_params: dict, # mutable-ok: overrides the base class signature + response_api_optional_request_params: dict[str, object], # mutable-ok: overrides the base class signature litellm_params: GenericLiteLLMParams, - headers: dict, # mutable-ok: overrides the base class signature - ) -> dict: # mutable-ok: overrides the base class signature - instructions_param: Final[object] = response_api_optional_request_params.get("instructions") + headers: dict[str, str], # mutable-ok: overrides the base class signature + ) -> dict[str, object]: # mutable-ok: overrides the base class signature validated_input: Final = self._validate_input_param(input) - instructions, folded_input = ( - _with_instruction_items_folded(validated_input, instructions_param) - if isinstance(instructions_param, str | None) - else (instructions_param, _developer_items_as_system(validated_input)) - ) - instruction_entries: Final = () if instructions is None else (("instructions", instructions),) - folded_params: Final = { - key: value - for key, value in ( - *((key, value) for key, value in response_api_optional_request_params.items() if key != "instructions"), - *instruction_entries, + model_info: Final = deployment_model_info(litellm_params) + declared: Final = declared_mid_conversation_system_support(model, self.custom_llm_provider.value, model_info) + folded: Final = ( + None + if deployment_declared_mid_conversation_system_support(model_info) is True + else instruction_items_folded( + validated_input, response_api_optional_request_params, "user" if declared is False else "system" ) - } + ) + placed_input, placed_params = folded or ( + developer_items_as_system(validated_input), + response_api_optional_request_params, + ) request: Final = super().transform_responses_api_request( model=resolve_fireworks_resource_name(model), - input=folded_input, - response_api_optional_request_params=folded_params, + input=placed_input, + response_api_optional_request_params=dict(placed_params), litellm_params=litellm_params, headers=headers, ) diff --git a/litellm/llms/gigachat/chat/transformation.py b/litellm/llms/gigachat/chat/transformation.py index 65921b628fb..81a186eadf4 100644 --- a/litellm/llms/gigachat/chat/transformation.py +++ b/litellm/llms/gigachat/chat/transformation.py @@ -38,6 +38,15 @@ else: _EMPTY_FUNCTION: Final[Mapping[str, object]] = MappingProxyType({}) _JSON_VALUE: Final = TypeAdapter(object) +_CONTENT_PART: Final = TypeAdapter(Mapping[str, object]) + + +def _file_part_data(part: Mapping[str, object]) -> str | None: + match part: + case {"type": "file", "file": {"file_data": str() as file_data}} if file_data: + return file_data + case _: + return None def is_valid_json(value: str) -> bool: @@ -295,7 +304,7 @@ class GigaChatConfig(BaseConfig): def _transform_list_content(self, content: Sequence) -> tuple[str, Sequence[object]]: """ - Extract text and image attachments from a multimodal message content list. + Extract text, image and file attachments from a multimodal message content list. Args: content: List of content parts (OpenAI multimodal format) @@ -320,6 +329,10 @@ class GigaChatConfig(BaseConfig): file_id = self._upload_image(upload_url) if file_id: attachments.append(file_id) + elif (file_data := _file_part_data(_CONTENT_PART.validate_python(part))) and ( + uploaded_file_id := self._upload_image(file_data) + ): + attachments.append(uploaded_file_id) text: Final = "\n".join(texts) if texts else "" return text, attachments diff --git a/litellm/llms/github_copilot/authenticator.py b/litellm/llms/github_copilot/authenticator.py index 59e1804f453..ee0e7af7e9b 100644 --- a/litellm/llms/github_copilot/authenticator.py +++ b/litellm/llms/github_copilot/authenticator.py @@ -1,11 +1,12 @@ import json import os import time +from dataclasses import dataclass from datetime import datetime -from typing import Any, Final +from typing import Any, Final, TypeVar import httpx -from pydantic import ConfigDict, TypeAdapter, with_config +from pydantic import BaseModel, ConfigDict, TypeAdapter, with_config from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm._logging import verbose_logger @@ -13,7 +14,6 @@ from litellm.litellm_core_utils.asyncify import can_block_current_thread from litellm.llms.custom_httpx.http_handler import get_httpx_client from .common_utils import ( - APIKeyExpiredError, GetAccessTokenError, GetAPIKeyError, GetDeviceCodeError, @@ -27,6 +27,44 @@ DEFAULT_GITHUB_ACCESS_TOKEN_URL: Final = "https://github.com/login/oauth/access_ DEFAULT_GITHUB_API_KEY_URL: Final = "https://api.github.com/copilot_internal/v2/token" +class CopilotEndpoints(BaseModel): + model_config = ConfigDict(frozen=True, strict=True, extra="ignore", hide_input_in_errors=True) + + api: str | None = None + + +class CopilotCachedKey(BaseModel): + model_config = ConfigDict(frozen=True, strict=True, extra="ignore", hide_input_in_errors=True) + + token: str + expires_at: float + endpoints: CopilotEndpoints | None = None + + +class CopilotCachedEndpoints(BaseModel): + model_config = ConfigDict(frozen=True, strict=True, extra="ignore", hide_input_in_errors=True) + + endpoints: CopilotEndpoints | None = None + + +ApiKeyFileView = TypeVar("ApiKeyFileView", CopilotCachedKey, CopilotCachedEndpoints) + + +@dataclass(frozen=True, slots=True) +class _UnsavedApiKey: + access_token: str + key: CopilotCachedKey + + +_UNSAVED_API_KEYS: Final[dict[str, _UnsavedApiKey]] = {} # mutable-ok: refreshed keys api-key.json could not hold + + +def _read_token_file(path: str) -> str: + with open(path, "rb") as f: + raw: Final = f.read() + return raw.decode(json.detect_encoding(raw)) + + @with_config(ConfigDict(extra="allow", strict=True)) class _DeviceCode(TypedDict): device_code: ReadOnly[str] @@ -87,12 +125,13 @@ class Authenticator: GetAccessTokenError: If unable to obtain an access token after retries. """ try: - with open(self.access_token_file, "r") as f: - access_token = f.read().strip() - if access_token: - return access_token + saved_token: Final = _read_token_file(self.access_token_file).strip() + if saved_token: + return saved_token except OSError: verbose_logger.warning("No existing access token found or error reading file") + except ValueError as e: + verbose_logger.warning("Ignoring %s, not a text access token: %s", self.access_token_file, e) if not can_block_current_thread(): raise GetAccessTokenError( @@ -135,64 +174,83 @@ class Authenticator: Raises: GetAPIKeyError: If unable to obtain an API key. """ - try: - with open(self.api_key_file, "r") as f: - api_key_info = json.load(f) - if api_key_info.get("expires_at", 0) > datetime.now().timestamp(): - return api_key_info.get("token") - else: - verbose_logger.warning("API key expired, refreshing") - raise APIKeyExpiredError( - message="API key expired", - status_code=401, - ) - except OSError: - verbose_logger.warning("No API key file found or error opening file") - except (json.JSONDecodeError, KeyError) as e: - verbose_logger.warning("Error reading API key from file: %s", e) - except APIKeyExpiredError: - pass # Already logged in the try block + unsaved: Final = self._unsaved_api_key() + if unsaved is not None: + return unsaved.token + cached: Final = self._read_api_key_file(CopilotCachedKey) + if cached is not None and cached.expires_at > datetime.now().timestamp(): + return cached.token + if cached is not None: + verbose_logger.warning("API key expired, refreshing") try: - api_key_info = self._refresh_api_key() - with open(self.api_key_file, "w") as f: - json.dump(api_key_info, f) - token: Final = api_key_info.get("token") - if token: - return token - else: - raise GetAPIKeyError( - message="API key response missing token", - status_code=401, - ) - except OSError as e: - verbose_logger.error("Error saving API key to file: %s", e) - raise GetAPIKeyError( - message=f"Failed to save API key: {e}", - status_code=500, - ) + api_key_info: Final = self._refresh_api_key() except RefreshAPIKeyError as e: raise GetAPIKeyError( message=f"Failed to refresh API key: {e}", status_code=401, + ) from e + serialized: Final = json.dumps(api_key_info) + try: + with open(self.api_key_file, "w") as f: + f.write(serialized) + except OSError as e: + verbose_logger.warning( + "Keeping the refreshed API key in memory, it could not be saved to %s: %s", self.api_key_file, e ) + self._keep_unsaved_api_key(serialized) + token: Final = api_key_info.get("token") + if not token: + raise GetAPIKeyError( + message="API key response missing token", + status_code=401, + ) + return token def get_api_base(self) -> str | None: """ - Get the API endpoint from the api-key.json file. + Get the API endpoint of the current API key. Returns: Optional[str]: The GitHub Copilot API endpoint, or None if not found. """ - try: - with open(self.api_key_file, "r") as f: - api_key_info: Final = json.load(f) - endpoints: Final = api_key_info.get("endpoints", {}) - api_endpoint: Final = endpoints.get("api") - return api_endpoint - except (OSError, json.JSONDecodeError, KeyError) as e: - verbose_logger.warning("Error reading API endpoint from file: %s", e) + cached: Final = self._unsaved_api_key() or self._read_api_key_file(CopilotCachedEndpoints) + if cached is None or cached.endpoints is None: return None + return cached.endpoints.api + + def _read_api_key_file(self, view: type[ApiKeyFileView]) -> ApiKeyFileView | None: + try: + return view.model_validate_json(_read_token_file(self.api_key_file)) + except OSError: + verbose_logger.warning("No API key file found or error opening file") + return None + except ValueError as e: + verbose_logger.warning("Ignoring %s, not a Copilot API key file: %s", self.api_key_file, e) + return None + + def _stored_access_token(self) -> str | None: + try: + access_token: Final = _read_token_file(self.access_token_file).strip() + except (OSError, ValueError): + return None + return access_token or None + + def _unsaved_api_key(self) -> CopilotCachedKey | None: + unsaved: Final = _UNSAVED_API_KEYS.get(self.api_key_file) + if unsaved is None or unsaved.key.expires_at <= time.time(): + return None + return unsaved.key if unsaved.access_token == self._stored_access_token() else None + + def _keep_unsaved_api_key(self, serialized: str) -> None: + access_token: Final = self._stored_access_token() + if access_token is None: + return + try: + key: Final = CopilotCachedKey.model_validate_json(serialized) + except ValueError: + return + _UNSAVED_API_KEYS[self.api_key_file] = _UnsavedApiKey(access_token=access_token, key=key) def _refresh_api_key(self) -> dict[str, Any]: """ diff --git a/litellm/llms/hosted_vllm/responses/transformation.py b/litellm/llms/hosted_vllm/responses/transformation.py index 8489bc0da84..0831ed3b37a 100644 --- a/litellm/llms/hosted_vllm/responses/transformation.py +++ b/litellm/llms/hosted_vllm/responses/transformation.py @@ -9,10 +9,13 @@ the chat completions → responses conversion pipeline. from typing import Final from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.openai.responses.instruction_items import instruction_items_folded from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import ResponseInputParam from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders +from litellm.utils import declared_mid_conversation_system_support, deployment_model_info class HostedVLLMResponsesAPIConfig(OpenAIResponsesAPIConfig): @@ -69,6 +72,31 @@ class HostedVLLMResponsesAPIConfig(OpenAIResponsesAPIConfig): return join_api_base(api_base, "v1/responses") + def transform_responses_api_request( + self, + model: str, + input: str | ResponseInputParam, + response_api_optional_request_params: dict[str, object], # mutable-ok: overrides the base class signature + litellm_params: GenericLiteLLMParams, + headers: dict[str, str], # mutable-ok: overrides the base class signature + ) -> dict[str, object]: # mutable-ok: overrides the base class signature + declared: Final = declared_mid_conversation_system_support( + model, self.custom_llm_provider.value, deployment_model_info(litellm_params) + ) + folded: Final = ( + instruction_items_folded(self._validate_input_param(input), response_api_optional_request_params, "user") + if declared is False + else None + ) + placed_input, placed_params = folded or (input, response_api_optional_request_params) + return super().transform_responses_api_request( + model=model, + input=placed_input, + response_api_optional_request_params=dict(placed_params), + litellm_params=litellm_params, + headers=headers, + ) + def supports_native_websocket(self) -> bool: """Hosted vLLM does not support native WebSocket for Responses API""" return False diff --git a/litellm/llms/manus/responses/transformation.py b/litellm/llms/manus/responses/transformation.py index ed06391a260..91832899141 100644 --- a/litellm/llms/manus/responses/transformation.py +++ b/litellm/llms/manus/responses/transformation.py @@ -241,7 +241,7 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig): api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, - ) -> tuple[str, dict]: + ) -> tuple[str, dict[str, str]]: """ Transform the get response API request into a URL and data. @@ -252,7 +252,7 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig): """ encoded_response_id: Final = encode_url_path_segment(response_id, field_name="response_id") url: Final = f"{api_base}/{encoded_response_id}" - data: Final[dict] = {} + data: Final[dict[str, str]] = {} return url, data def transform_get_response_api_response( diff --git a/litellm/llms/oci/chat/transformation.py b/litellm/llms/oci/chat/transformation.py index e8d37f857e6..8257a75b4a5 100644 --- a/litellm/llms/oci/chat/transformation.py +++ b/litellm/llms/oci/chat/transformation.py @@ -20,6 +20,7 @@ import litellm from litellm._logging import verbose_logger from litellm.constants import DEFAULT_OCI_CHAT_MAX_TOKENS from litellm.litellm_core_utils.logging_utils import track_llm_api_timing +from litellm.litellm_core_utils.prompt_templates.common_utils import messages_with_file_parts_as_image_urls from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException @@ -681,7 +682,9 @@ class OCIChatConfig(BaseConfig): servingMode=serving_mode, chatRequest=OCIChatRequestPayload( apiFormat=vendor.value, - messages=adapt_messages_to_generic_oci_standard(messages), + messages=adapt_messages_to_generic_oci_standard( + list(messages_with_file_parts_as_image_urls(messages)) + ), **self._get_optional_params(vendor, optional_params, model), ), ) diff --git a/litellm/llms/openai/chat/gpt_reasoning_transformation.py b/litellm/llms/openai/chat/gpt_reasoning_transformation.py index 0311f6c6590..8c9551d01c0 100644 --- a/litellm/llms/openai/chat/gpt_reasoning_transformation.py +++ b/litellm/llms/openai/chat/gpt_reasoning_transformation.py @@ -5,8 +5,10 @@ from collections.abc import Collection from typing import ClassVar, Final import litellm +from litellm._logging import verbose_logger from litellm.utils import ( declared_value_factory, + has_catalog_entry, is_explicitly_disabled_factory, supports_factory, ) @@ -16,6 +18,25 @@ from .gpt_transformation import OpenAIGPTConfig _GPT_SERIES_VERSION: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?(?=[.-]|$)") +def refused_reasoning_effort_warning(*, suggest_declared_levels: bool, suggest_base_model: bool) -> str: + return "".join( + ( + "Dropping %s=%r for model=%r (drop_params=True) because the model map does not mark that level " + "supported for %r.", + ( + " If the deployment supports it, list every level it supports in model_info.reasoning_effort_levels." + if suggest_declared_levels + else "" + ), + ( + " If the deployment name is not its underlying model, set model_info.base_model to that model." + if suggest_base_model + else "" + ), + ) + ) + + def _catalogue_declares_default_effort() -> bool: """Whether the loaded cost map carries default_reasoning_effort for ANY entry. @@ -93,6 +114,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig): OPT_IN_REASONING_EFFORTS: ClassVar[frozenset[str]] = frozenset(("xhigh",)) ROUTE_SPECIFIC_REASONING_EFFORTS: ClassVar[frozenset[str]] = frozenset(("max",)) + LOOKS_UP_BASE_MODEL: ClassVar[bool] = False @classmethod def is_model_gpt_5_model(cls, model: str) -> bool: @@ -307,7 +329,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig): model: str, drop_params: bool, *, - declared_reasoning_efforts: Collection[str] = (), + declared_reasoning_efforts: Collection[str] | None = None, ) -> dict: if self.is_model_gpt_5_search_model(model): if "max_tokens" in non_default_params: @@ -338,10 +360,21 @@ class OpenAIGPT5Config(OpenAIGPTConfig): if ( requested_effort is not None - and requested_effort not in declared_reasoning_efforts + and requested_effort not in (declared_reasoning_efforts or ()) and self.refuses_reasoning_effort_level(model, requested_effort) ): if litellm.drop_params or drop_params: + verbose_logger.warning( + refused_reasoning_effort_warning( + suggest_declared_levels=declared_reasoning_efforts is not None, + suggest_base_model=self.LOOKS_UP_BASE_MODEL + and not has_catalog_entry(self._model_map_lookup_name(model), None), + ), + "reasoning_effort", + requested_effort, + model, + self._model_map_lookup_name(model), + ) non_default_params.pop("reasoning_effort", None) optional_params.pop("reasoning_effort", None) else: diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index b7396e8fb03..22af51aaabe 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -36,7 +36,11 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import ( ) from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator -from litellm.llms.base_llm.base_utils import BaseLLMModelInfo +from litellm.llms.base_llm.base_utils import ( + BaseLLMModelInfo, + hoist_developer_messages_into_leading_system_message, + map_developer_role_to_system_role, +) from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( @@ -58,7 +62,7 @@ from litellm.types.utils import ( ModelResponseStream, chat_completion_tool_call_from_dict, ) -from litellm.utils import convert_to_model_response_object +from litellm.utils import convert_to_model_response_object, declared_mid_conversation_system_support from ..common_utils import OpenAIError from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config @@ -75,6 +79,45 @@ else: _NO_TOOLS_UPDATE: Final[Mapping[str, object]] = MappingProxyType({}) + + +def targets_openai_hosted_endpoint(custom_llm_provider: str | None, api_base: str | None) -> bool: + if custom_llm_provider != "openai": + return False + resolved_api_base: Final = ( + api_base or litellm.api_base or os.getenv("OPENAI_BASE_URL") or os.getenv("OPENAI_API_BASE") + ) + if not resolved_api_base: + return True + hostname: Final = urlparse(resolved_api_base).hostname + if hostname is None: + return True + return hostname == "openai.com" or hostname.endswith(".openai.com") + + +def _roles(messages: Sequence[AllMessageValues]) -> tuple[str, ...]: + return tuple(message["role"] for message in messages) + + +def translate_developer_role_for_openai_compatible_endpoint( + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, +) -> Sequence[AllMessageValues]: + in_place: Final = map_developer_role_to_system_role(messages=messages) + if targets_openai_hosted_endpoint(custom_llm_provider, api_base): + return in_place + placed: Final = hoist_developer_messages_into_leading_system_message(messages=messages) + if _roles(placed) == _roles(in_place): + return in_place + if declared_mid_conversation_system_support(model, custom_llm_provider, model_info) is not False: + return in_place + return placed + + _OBJECT_MAPPING: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) _CHOICE_MAPPINGS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) _DECODED_JSON: Final = TypeAdapter(object) @@ -437,20 +480,30 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): ) return messages, tools + def translate_developer_role_to_system_role( + self, + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: + """ + A developer message only changes role, except on a model whose deployment or cost + map declares `supports_mid_conversation_system` false, where the instructions are + placed in one leading system message for its chat template. + """ + return translate_developer_role_for_openai_compatible_endpoint( + messages, model=model, custom_llm_provider=custom_llm_provider, api_base=api_base, model_info=model_info + ) + def _targets_openai_hosted_endpoint( self, custom_llm_provider: str | None, api_base: str | None, ) -> bool: - if custom_llm_provider != "openai": - return False - resolved_api_base = api_base or litellm.api_base or os.getenv("OPENAI_BASE_URL") or os.getenv("OPENAI_API_BASE") - if not resolved_api_base: - return True - hostname: Final = urlparse(resolved_api_base).hostname - if hostname is None: - return True - return hostname == "openai.com" or hostname.endswith(".openai.com") + return targets_openai_hosted_endpoint(custom_llm_provider, api_base) def _should_preserve_cache_control_for_endpoint( self, diff --git a/litellm/llms/openai/chat/o_series_transformation.py b/litellm/llms/openai/chat/o_series_transformation.py index c7552d47985..64285aafdb0 100644 --- a/litellm/llms/openai/chat/o_series_transformation.py +++ b/litellm/llms/openai/chat/o_series_transformation.py @@ -11,7 +11,7 @@ Translations handled by LiteLLM: - Logprobs => drop param (if user opts in to dropping param) """ -from collections.abc import Coroutine +from collections.abc import Coroutine, Sequence from typing import Final, Literal, cast, overload import litellm @@ -37,7 +37,15 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig): def get_config(cls): return super().get_config() - def translate_developer_role_to_system_role(self, messages: list[AllMessageValues]) -> list[AllMessageValues]: + def translate_developer_role_to_system_role( + self, + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: """ O-series models support `developer` role. """ diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index 216725b9449..a9d2dcdffb8 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -104,6 +104,19 @@ def is_openai_backed_api_base(api_base: str) -> bool: return hostname is not None and (hostname == _OPENAI_API_HOST or hostname.endswith(f".{_OPENAI_API_HOST}")) +_OPENAI_AGENTS_API_RESOURCES: Final = frozenset({"agents", "vaults"}) + + +def openai_api_resource(path: str) -> str | None: + segments: Final = tuple(segment for segment in path.split("/") if segment) + resource_segments: Final = segments[1:] if segments[:1] == ("v1",) else segments + return next(iter(resource_segments), None) + + +def is_openai_agents_api_path(path: str) -> bool: + return openai_api_resource(path) in _OPENAI_AGENTS_API_RESOURCES + + class OpenAIError(BaseLLMException): def __init__( self, diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 5a3ab8aebea..af2e85c5f20 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -1,6 +1,6 @@ import time import types -from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Iterator, Mapping +from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Final, Literal, Optional, cast import httpx @@ -50,7 +50,11 @@ from litellm.utils import ( from ...types.llms.openai import * from ..base import BaseLLM from .chat.gpt_reasoning_transformation import OpenAIGPT5Config -from .chat.gpt_transformation import OpenAIGPTConfig, OpenAIUnknownModelConfig +from .chat.gpt_transformation import ( + OpenAIGPTConfig, + OpenAIUnknownModelConfig, + translate_developer_role_for_openai_compatible_endpoint, +) from .chat.o_series_transformation import OpenAIOSeriesConfig from .common_utils import ( BaseOpenAILLM, @@ -206,6 +210,19 @@ class OpenAIConfig(BaseConfig): else: return self._gpt_config_for_model(model).get_supported_openai_params(model=model) + def translate_developer_role_to_system_role( + self, + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: + return translate_developer_role_for_openai_compatible_endpoint( + messages, model=model, custom_llm_provider=custom_llm_provider, api_base=api_base, model_info=model_info + ) + def _gpt_config_for_model(self, model: str) -> OpenAIGPTConfig: if type(self) is OpenAIConfig and not OpenAIGPTConfig.is_openai_catalog_model(model): return OpenAIUnknownModelConfig() diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index b7e4cc6868f..5be488f0f6e 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -4,11 +4,17 @@ This file contains the calling OpenAI's `/v1/realtime` endpoint. This requires websockets, and is currently only supported on LiteLLM Proxy. """ +import asyncio import ssl -from typing import TYPE_CHECKING, Final, cast +from collections.abc import Mapping +from typing import TYPE_CHECKING, Final, Literal, cast + +from pydantic import TypeAdapter from litellm._logging import redact_string, verbose_logger from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES +from litellm.exceptions import AuthenticationError +from litellm.responses.streaming_iterator import map_stream_error_to_exception from litellm.types.realtime import RealtimeQueryParams from ....litellm_core_utils.litellm_logging import Logging as LiteLLMLogging @@ -25,6 +31,54 @@ from ..openai import OpenAIChatCompletion if TYPE_CHECKING: from fastapi import WebSocket + from openai.types.realtime import RealtimeError + from websockets.asyncio.client import ClientConnection + +_SERVER_EVENT_FIELDS: Final = TypeAdapter(Mapping[str, object]) +_CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL: Final = "invalid_model" +_CREDENTIALS_REJECTED: Final = "invalid_api_key" + + +def first_event_error(first_event: str | bytes) -> "RealtimeError | None": + event: Final = _SERVER_EVENT_FIELDS.validate_json(first_event) + if event.get("type") != "error": + return None + from openai.types.realtime import RealtimeErrorEvent + + return RealtimeErrorEvent.model_validate(event).error + + +def _confirm_first_event( + first_event: str | bytes, model: str, sent_credentials: bool, model_listed_outside_realtime: bool +) -> Literal[True]: + error: Final = first_event_error(first_event) + if error is None or ( + error.code == _CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL and not model_listed_outside_realtime + ): + return True + if error.code == _CREDENTIALS_REJECTED or not sent_credentials: + raise AuthenticationError(message=error.message, llm_provider="openai", model=model) + raise map_stream_error_to_exception(error, model=model, custom_llm_provider="openai") + + +async def confirm_session_started( + connection: "ClientConnection", model: str, sent_credentials: bool, model_listed_outside_realtime: bool +) -> Literal[True]: + return _confirm_first_event(await connection.recv(), model, sent_credentials, model_listed_outside_realtime) + + +async def confirm_gateway_session_started( + connection: "ClientConnection", + model: str, + sent_credentials: bool, + model_listed_outside_realtime: bool, + first_event_wait_seconds: float, +) -> Literal[True]: + try: + first_event: Final = await asyncio.wait_for(connection.recv(), first_event_wait_seconds) + except asyncio.TimeoutError: + return True + return _confirm_first_event(first_event, model, sent_credentials, model_listed_outside_realtime) class OpenAIRealtime(OpenAIChatCompletion): diff --git a/litellm/llms/openai/responses/instruction_items.py b/litellm/llms/openai/responses/instruction_items.py new file mode 100644 index 00000000000..82bf4329b34 --- /dev/null +++ b/litellm/llms/openai/responses/instruction_items.py @@ -0,0 +1,98 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final, Literal, TypeAlias + +from openai.types.responses import EasyInputMessageParam, ResponseInputContentParam, ResponseInputItemParam + +from litellm.llms.base_llm.base_utils import hoisted_instruction_indices +from litellm.types.llms.openai import ResponseInputParam + +UnfoldedDeveloperRole: TypeAlias = Literal["system", "user"] + +_INSTRUCTION_ROLES: Final = frozenset({"system", "developer"}) + + +def _role(item: ResponseInputItemParam) -> str | None: + match item: + case {"role": str(role)}: + return role + case _: + return None + + +def _developer_item_as(item: ResponseInputItemParam, role: UnfoldedDeveloperRole) -> ResponseInputItemParam: + if "role" not in item or item["role"] != "developer" or "content" not in item: + return item + return EasyInputMessageParam(role=role, content=item["content"], type="message") + + +def developer_items_as_system(input: str | ResponseInputParam) -> str | ResponseInputParam: + if isinstance(input, str): + return input + return [_developer_item_as(item, "system") for item in input] + + +def _text_part(part: ResponseInputContentParam) -> str | None: + match part: + case {"type": "input_text", "text": str(text)}: + return text + case _: + return None + + +def _text_only_content(item: ResponseInputItemParam) -> str | None: + match item: + case {"role": "system" | "developer", "content": str(text)}: + return text + case {"role": "system" | "developer", "content": [*parts]}: + texts: Final = tuple(map(_text_part, parts)) + return None if any(text is None for text in texts) else "\n\n".join(text for text in texts if text) + case _: + return None + + +def _folded_instructions_and_items( + items: tuple[ResponseInputItemParam, ...], instructions: str | None, unfolded_developer_role: UnfoldedDeveloperRole +) -> tuple[str | None, ResponseInputParam]: + folded: Final = MappingProxyType( + { + index: text + for index in hoisted_instruction_indices(tuple(map(_role, items))) + if (text := _text_only_content(items[index])) is not None + } + ) + joined: Final = "\n\n".join(chunk for chunk in (instructions, *folded.values()) if chunk) + return ( + instructions if not folded else joined or None, + [_developer_item_as(item, unfolded_developer_role) for index, item in enumerate(items) if index not in folded], + ) + + +def instruction_items_folded( + input: str | ResponseInputParam, + params: Mapping[str, object], + unfolded_developer_role: UnfoldedDeveloperRole, +) -> tuple[ResponseInputParam, Mapping[str, object]] | None: + """ + For a model whose chat template takes a single leading system message, the + leading system and developer items and every later text developer item folded into + top-level `instructions`, the developer items left over sent with + `unfolded_developer_role`, or None when the request needs no folding. + """ + instructions: Final = params.get("instructions") + if isinstance(input, str) or not isinstance(instructions, str | None): + return None + items: Final = tuple(input) + if not any(_role(item) in _INSTRUCTION_ROLES for item in items): + return None + folded_instructions, folded_items = _folded_instructions_and_items(items, instructions, unfolded_developer_role) + instruction_entries: Final = () if folded_instructions is None else (("instructions", folded_instructions),) + return folded_items, MappingProxyType( + { + key: value + for key, value in ( + *((key, value) for key, value in params.items() if key != "instructions"), + *instruction_entries, + ) + } + ) diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 45fc1fc3ff7..9b8515a9ecd 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -1,4 +1,4 @@ -from collections.abc import Callable, Collection, Mapping, Sequence +from collections.abc import Callable, Collection, Iterable, Mapping, Sequence from functools import lru_cache from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Protocol, cast, get_type_hints @@ -10,7 +10,11 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger -from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook, targets_openai_api +from litellm.integrations.anthropic_cache_control_hook import ( + AnthropicCacheControlHook, + openai_prompt_cache_breakpoint_support, + targets_openai_api, +) from litellm.litellm_core_utils.core_helpers import normalize_drop_params, process_response_headers from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( @@ -20,6 +24,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( cache_control_as_prompt_cache_breakpoint, drop_non_python_regex_patterns, flatten_combinators_and_drop_non_python_regex_patterns, + prompt_cache_breakpoint_for_wire, with_prompt_cache_breakpoint, ) from litellm.litellm_core_utils.safe_json_loads import safe_json_loads @@ -27,6 +32,7 @@ from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_a from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.llms.openai.chat.gpt_reasoning_transformation import ( is_gpt_reasoning_series_name, + refused_reasoning_effort_warning, unsupported_reasoning_effort_message, ) from litellm.responses.litellm_completion_transformation.custom_tools import TOOL_CALL_ITEM_ID_PREFIX_BY_TYPE @@ -50,6 +56,9 @@ from ..common_utils import OpenAIError from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS: Final = 16 +DROP_UNSUPPORTED_REASONING_WARNING: Final = ( + "Dropping reasoning=%r for model=%r (drop_params=True) because its model cost map entry lacks supports_reasoning." +) _RAW_RESPONSE_JSON: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True)) if TYPE_CHECKING: @@ -66,6 +75,7 @@ _PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS: Final = frozenset({LlmProviders.AZURE, _PROVIDERS_REPLAYING_ONLY_THEIR_OWN_REASONING: Final = _PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS _ROLES_READING_PROMPT_CACHE_BREAKPOINTS: Final = frozenset({"system", "developer", "user"}) _PART_TYPES_READING_PROMPT_CACHE_BREAKPOINTS: Final = frozenset({"input_text", "input_image", "input_file"}) +_INPUT_PART_LIST_KEYS: Final = frozenset({"content", "output"}) _COMPACT_TRACKING_PARAMS: Final = frozenset({"metadata", "user"}) _COMPACT_REJECTED_PARAMS: Final = frozenset( { @@ -93,6 +103,13 @@ _INPUT_ITEM: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, _CONTENT_PARTS: Final[TypeAdapter[Sequence[Mapping[str, object]]]] = TypeAdapter(Sequence[Mapping[str, object]]) +class _DeclaredModel(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True) + + base_model: object = None + model_info: object = None + + class _DeploymentHost(BaseModel): """`responses()` builds the params from raw kwargs, where callers may spell the host `base_url`.""" @@ -216,6 +233,63 @@ def _with_breakpoint_on_last_reading_part( return (*parts[:last], marked_last, *parts[last + 1 :]) +def _json_object_or_none(value: object) -> Mapping[str, object] | None: + return value if isinstance(value, Mapping) else None + + +def _json_array_or_empty(value: object) -> Sequence[object]: + return value if isinstance(value, (list, tuple)) else () + + +def _item_with_wire_safe_prompt_cache_breakpoints(item: object, keeps_markers: bool) -> object: + fields: Final = _json_object_or_none(item) + if fields is None or not _carries_prompt_cache_breakpoint(fields): + return item + wire_safe_fields: Final = { + key: _parts_with_wire_safe_prompt_cache_breakpoints(key, value, keeps_markers) + for key, value in fields.items() + if key != "prompt_cache_breakpoint" + } + unchanged: Final = "prompt_cache_breakpoint" not in fields and _same_objects( + wire_safe_fields.values(), fields.values() + ) + return item if unchanged else wire_safe_fields + + +def _same_objects(rebuilt: Iterable[object], originals: Iterable[object]) -> bool: + return all(new is old for new, old in zip(rebuilt, originals, strict=True)) + + +def _carries_prompt_cache_breakpoint(fields: Mapping[str, object]) -> bool: + return "prompt_cache_breakpoint" in fields or any( + _parts_carry_prompt_cache_breakpoint(fields.get(key)) for key in _INPUT_PART_LIST_KEYS + ) + + +def _parts_carry_prompt_cache_breakpoint(parts: object) -> bool: + return any(isinstance(part, Mapping) and "prompt_cache_breakpoint" in part for part in _json_array_or_empty(parts)) + + +def _parts_with_wire_safe_prompt_cache_breakpoints(key: str, value: object, keeps_markers: bool) -> object: + if key not in _INPUT_PART_LIST_KEYS or not _parts_carry_prompt_cache_breakpoint(value): + return value + parts: Final = _json_array_or_empty(value) + wire_safe_parts: Final = tuple(_part_with_wire_safe_prompt_cache_breakpoint(part, keeps_markers) for part in parts) + return value if _same_objects(wire_safe_parts, parts) else list(wire_safe_parts) + + +def _part_with_wire_safe_prompt_cache_breakpoint(part: object, keeps_markers: bool) -> object: + fields: Final = _json_object_or_none(part) + if fields is None or "prompt_cache_breakpoint" not in fields: + return part + marker: Final = fields["prompt_cache_breakpoint"] + wire_marker: Final = prompt_cache_breakpoint_for_wire(marker, drop_params=True) if keeps_markers else None + if keeps_markers and wire_marker == marker: + return part + unmarked: Final = {key: value for key, value in fields.items() if key != "prompt_cache_breakpoint"} + return with_prompt_cache_breakpoint(unmarked, wire_marker) + + class _ReasoningSupportEntry(LiteLLMBaseModel): litellm_provider: str | None = None supports_reasoning: bool | None = None @@ -347,12 +421,19 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): return declared return info["key"] in _bundled_openai_reasoning_models() + @staticmethod + def _is_known_non_reasoning_model(model: str) -> bool: + from litellm.utils import is_known_non_reasoning_model + + return is_known_non_reasoning_model(model.split("/")[-1], LlmProviders.OPENAI.value) + @staticmethod def _requests_reasoning_effort(reasoning: object) -> bool: - effort: Final = ( - reasoning.get("effort") if isinstance(reasoning, Mapping) else getattr(reasoning, "effort", None) - ) - return effort is not None + return OpenAIResponsesAPIConfig._requested_effort(reasoning) is not None + + @staticmethod + def _requested_effort(reasoning: object) -> object: + return reasoning.get("effort") if isinstance(reasoning, Mapping) else getattr(reasoning, "effort", None) @staticmethod def _enforce_min_max_output_tokens(max_output_tokens: "int | None") -> "int | None": @@ -394,17 +475,20 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): ) -> dict: return dict( self.map_openai_params_for_deployment( - response_api_optional_params, model, drop_params, declared_reasoning_efforts=() + response_api_optional_params, model, drop_params, declared_reasoning_efforts=None ) ) + def honors_declared_reasoning_efforts(self) -> bool: + return type(self).map_openai_params is OpenAIResponsesAPIConfig.map_openai_params + def map_openai_params_for_deployment( self, response_api_optional_params: ResponsesAPIOptionalRequestParams, model: str, drop_params: bool, *, - declared_reasoning_efforts: Collection[str], + declared_reasoning_efforts: Collection[str] | None, ) -> Mapping[str, object]: """No mapping applied since inputs are in OpenAI spec already. @@ -424,7 +508,9 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): and not self._supports_reasoning_param(model=model) ): if drop_params or litellm.drop_params: - params.pop("reasoning", None) + dropped_reasoning: Final = params.pop("reasoning", None) + if self._requested_effort(dropped_reasoning) != "none" or not self._is_known_non_reasoning_model(model): + verbose_logger.warning(DROP_UNSUPPORTED_REASONING_WARNING, dropped_reasoning, model) else: raise litellm.UnsupportedParamsError( message=( @@ -442,7 +528,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): if ( isinstance(reasoning, dict) and isinstance(requested_effort, str) - and requested_effort not in declared_reasoning_efforts + and requested_effort not in (declared_reasoning_efforts or ()) and self._effort_level_is_disabled(lookup_name, requested_effort) ): if not (drop_params or litellm.drop_params): @@ -450,6 +536,15 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): message=unsupported_reasoning_effort_message("reasoning.effort", requested_effort, model), status_code=400, ) + verbose_logger.warning( + refused_reasoning_effort_warning( + suggest_declared_levels=declared_reasoning_efforts is not None, suggest_base_model=False + ), + "reasoning.effort", + requested_effort, + model, + lookup_name, + ) remaining: Final = reasoning.copy() remaining.pop("effort", None) if remaining: @@ -544,12 +639,11 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): validated_input: Final = self._validate_input_param( self._drop_bridge_minted_reasoning_items(input), drops_set_created_by=drops_params(litellm_params) ) - marked_input: Final = ( - self._input_with_prompt_cache_breakpoints(validated_input) - if self._reads_prompt_cache_breakpoints( - model=model, litellm_params=litellm_params, prompt_cache_options=prompt_cache_options - ) - else validated_input + reads_breakpoints: Final = self._reads_prompt_cache_breakpoints( + model=model, litellm_params=litellm_params, prompt_cache_options=prompt_cache_options + ) + marker_safe_input: Final = self._input_with_wire_safe_prompt_cache_breakpoints( + model=model, input=validated_input, litellm_params=litellm_params, reads_breakpoints=reads_breakpoints ) maps_anthropic_tool_search: Final = self._targets_openai_api(litellm_params) and any( is_anthropic_tool_search_chat_tool(tool) for tool in tools or () @@ -559,20 +653,71 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): if maps_anthropic_tool_search else tools ) - replayable_input: Final = ( - input_without_unanswered_anthropic_tool_search_calls(marked_input) - if maps_anthropic_tool_search - else marked_input - ) stripped_input, stripped_tools = self.remove_cache_control_flag_from_input_and_tools( - model=model, input=replayable_input, tools=mapped_tools + model=model, + input=self._replayable_input(marker_safe_input, reads_breakpoints, maps_anthropic_tool_search), + tools=mapped_tools, ) + if marker_safe_input is not validated_input: + self._strip_caller_input_as_if_unmarked( + model=model, + caller_input=validated_input, + reads_breakpoints=reads_breakpoints, + maps_anthropic_tool_search=maps_anthropic_tool_search, + ) object_schema_tools: Final = self._tools_with_object_parameters(model=model, tools=stripped_tools) sanitized_tools: Final = self._sanitized_tool_schemas_for_openai( model=model, tools=object_schema_tools, litellm_params=litellm_params ) return self._drop_foreign_tool_call_item_ids(stripped_input), sanitized_tools + def _replayable_input( + self, input: str | ResponseInputParam, reads_breakpoints: bool, maps_anthropic_tool_search: bool + ) -> str | ResponseInputParam: + marked_input: Final = self._input_with_prompt_cache_breakpoints(input) if reads_breakpoints else input + if not maps_anthropic_tool_search: + return marked_input + return input_without_unanswered_anthropic_tool_search_calls(marked_input) + + def _strip_caller_input_as_if_unmarked( + self, + model: str, + caller_input: str | ResponseInputParam, + reads_breakpoints: bool, + maps_anthropic_tool_search: bool, + ) -> None: + self.remove_cache_control_flag_from_input_and_tools( + model=model, input=self._replayable_input(caller_input, reads_breakpoints, maps_anthropic_tool_search) + ) + + def _input_with_wire_safe_prompt_cache_breakpoints( + self, + model: str, + input: str | ResponseInputParam, + litellm_params: GenericLiteLLMParams, + reads_breakpoints: bool, + ) -> str | ResponseInputParam: + if not isinstance(input, list) or not drops_params(litellm_params): + return input + keeps_markers: Final = reads_breakpoints or self._keeps_prompt_cache_breakpoints( + model=model, litellm_params=litellm_params + ) + wire_safe_items: Final = tuple( + _item_with_wire_safe_prompt_cache_breakpoints(item, keeps_markers) for item in input + ) + if _same_objects(wire_safe_items, input): + return input + wire_safe_input: Final = list(wire_safe_items) + return cast("ResponseInputParam", wire_safe_input) # cast-ok: items keep their shape, minus a rejected marker + + def _keeps_prompt_cache_breakpoints(self, model: str, litellm_params: GenericLiteLLMParams) -> bool: + provider: Final = _provider_name(self.custom_llm_provider) + base_model: Final = self._declared_base_model(model=model, litellm_params=litellm_params) + known_support: Final = ( + openai_prompt_cache_breakpoint_support(name, provider) for name in dict.fromkeys((base_model, model)) + ) + return next((support for support in known_support if support is not None), None) is not False + def _targets_openai_api(self, litellm_params: GenericLiteLLMParams) -> bool: host: Final = _DeploymentHost.model_validate(litellm_params) return self.custom_llm_provider == LlmProviders.OPENAI and targets_openai_api(host.api_base or host.base_url) @@ -688,14 +833,15 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): @staticmethod def _without_foreign_tool_call_item_id(item: object) -> object: - if not isinstance(item, dict): + fields: Final = _json_object_or_none(item) + if fields is None: return item - item_type: Final = item.get("type") - item_id: Final = item.get("id") + item_type: Final = fields.get("type") + item_id: Final = fields.get("id") genuine_prefix: Final = TOOL_CALL_ITEM_ID_PREFIX_BY_TYPE.get(item_type) if isinstance(item_type, str) else None if genuine_prefix is None or not isinstance(item_id, str) or item_id.startswith(genuine_prefix): return item - return {key: value for key, value in item.items() if key != "id"} + return {key: value for key, value in fields.items() if key != "id"} def _sanitized_tool_schemas_for_openai( self, @@ -739,6 +885,13 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): base_model: Final[object] = model_info.get("base_model") if isinstance(model_info, dict) else None return base_model if isinstance(base_model, str) and base_model else model + @staticmethod + def _declared_base_model(model: str, litellm_params: GenericLiteLLMParams) -> str: + declared: Final = _DeclaredModel.model_validate(litellm_params) + model_info: Final = _json_object_or_none(declared.model_info) + candidates: Final = (declared.base_model, None if model_info is None else model_info.get("base_model")) + return next((name for name in candidates if isinstance(name, str) and name), model) + @staticmethod def _sanitized_tool_entry( entry: Mapping[str, object], @@ -1094,7 +1247,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, - ) -> tuple[str, dict]: + ) -> tuple[str, dict[str, str]]: """ Transform the get response API request into a URL and data @@ -1103,7 +1256,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): """ encoded_response_id: Final = encode_url_path_segment(response_id, field_name="response_id") url: Final = f"{api_base}/{encoded_response_id}" - data: Final[dict] = {} + data: Final[dict[str, str]] = {} return url, data def transform_get_response_api_response( diff --git a/litellm/llms/openai_like/messages/transformation.py b/litellm/llms/openai_like/messages/transformation.py index 8daae2a8e6a..a5d825a39d7 100644 --- a/litellm/llms/openai_like/messages/transformation.py +++ b/litellm/llms/openai_like/messages/transformation.py @@ -37,6 +37,9 @@ class OpenAILikeAnthropicMessagesConfig(AnthropicMessagesConfig): def is_native_messages_route(self) -> bool: return False + def hoists_wrapped_defer_loading(self, model: str) -> bool: + return False + def validate_anthropic_messages_environment( self, headers: dict[str, str], diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index fe88ac03e04..ad47b334ed4 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -270,6 +270,13 @@ "/v1/responses" ] }, + "kyma": { + "base_url": "https://kymaapi.com/v1", + "api_key_env": "KYMA_API_KEY", + "provider_info": { + "supports_function_calling": true + } + }, "streamlake": { "base_url": "https://vanchin.streamlake.ai/api/gateway/v1/endpoints", "api_key_env": "STREAMLAKE_API_KEY", diff --git a/litellm/llms/opensearch/__init__.py b/litellm/llms/opensearch/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/opensearch/vector_stores/__init__.py b/litellm/llms/opensearch/vector_stores/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/opensearch/vector_stores/transformation.py b/litellm/llms/opensearch/vector_stores/transformation.py new file mode 100644 index 00000000000..2d862bc8ea6 --- /dev/null +++ b/litellm/llms/opensearch/vector_stores/transformation.py @@ -0,0 +1,415 @@ +import hashlib +import json +import re +from collections.abc import Mapping, Sequence +from functools import partial +from typing import TYPE_CHECKING, Final, Literal, NoReturn +from urllib.parse import quote, urlsplit + +import httpx +from pydantic import ConfigDict, Field, TypeAdapter, ValidationError + +from litellm._logging import verbose_logger +from litellm.exceptions import AuthenticationError, BadRequestError +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params +from litellm.litellm_core_utils.optional_imports import ensure_optional_import +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.vector_store.transformation import ( + BaseQueryEmbeddingVectorStoreConfig, + VectorStoreEmbeddingExecutor, +) +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, SignsRequestsWithAWS, sign_aws_json_post +from litellm.types.llms.base import LiteLLMBaseModel +from litellm.types.llms.bedrock import AwsAuthParams +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import EmbeddingResponse +from litellm.types.vector_stores import ( + BaseVectorStoreAuthCredentials, + VectorStoreCreateOptionalRequestParams, + VectorStoreIndexEndpoints, + VectorStoreResultContent, + VectorStoreSearchOptionalRequestParams, + VectorStoreSearchResponse, + VectorStoreSearchResult, +) + +if TYPE_CHECKING: + from botocore.credentials import Credentials + + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + +DEFAULT_EMBEDDING_FIELD: Final = "embedding" +DEFAULT_TEXT_FIELD: Final = "text" +DEFAULT_MAX_NUM_RESULTS: Final = 10 +MAX_MAX_NUM_RESULTS: Final = 50 +_PARAM_PREFIX: Final = "opensearch_" +_AWS_HOSTS: Final = ( + re.compile(r"\.(?P[a-z0-9-]+)\.(?Paoss|aoss-fips|es)\.amazonaws\.com(?:\.cn)?\Z"), + re.compile(r"\.(?Paoss|aos)\.(?P[a-z0-9-]+)\.on\.aws\Z"), +) +_ACCOUNT_ID_LABEL: Final = re.compile(r"\A\d{12}\.") +_QUERY_VECTOR: Final = TypeAdapter(tuple[float, ...]) +_ATTRIBUTE_VALUE_TYPES: Final = (str, int, float, bool) +_SEARCH_ONLY_MESSAGE: Final = ( + "The OpenSearch vector store is search-only. Create the k-NN index in OpenSearch, " + "then register it here with its name as opensearch_index_name or vector_store_id." +) +_PER_ACCOUNT_MESSAGE: Final = ( + "api_base is an OpenSearch Serverless per-account endpoint, which needs a collection header the vector store " + "does not send. Use the collection endpoint instead, for example https://..aoss.amazonaws.com." +) +_SIGV4_ONLY_MESSAGE: Final = ( + "The OpenSearch vector store signs every request with AWS SigV4 and does not support api_key. " + "Remove api_key and give the vector store AWS credentials instead." +) +_CREDENTIALS_HINT: Final = ( + "Set aws_access_key_id and aws_secret_access_key, aws_role_name, or aws_profile_name on the vector store, " + "or run LiteLLM with an AWS identity." +) + + +def _bad_request(message: str) -> BadRequestError: + return BadRequestError(message=message, model=None, llm_provider="opensearch") + + +def _unauthenticated(reason: str) -> AuthenticationError: + return AuthenticationError( + message=f"No usable AWS credentials to sign the OpenSearch request ({reason}). {_CREDENTIALS_HINT}", + llm_provider="opensearch", + model=None, + ) + + +def _usable_credentials(credentials: object) -> "Credentials": + from botocore.credentials import Credentials + + if isinstance(credentials, Credentials): + return credentials + raise _unauthenticated("none were found") + + +class _OpenSearchParams(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + litellm_embedding_model: str | None = None + opensearch_embedding_field: str | None = None + opensearch_index_name: str | None = None + opensearch_text_field: str | None = None + + @property + def embedding_field(self) -> str: + return self.opensearch_embedding_field or DEFAULT_EMBEDDING_FIELD + + @property + def text_field(self) -> str: + return self.opensearch_text_field or DEFAULT_TEXT_FIELD + + def index_name(self, vector_store_id: str) -> str: + return self.opensearch_index_name or vector_store_id + + def require_embedding_model(self) -> str: + if not self.litellm_embedding_model: + raise _bad_request( + "litellm_embedding_model is required for the OpenSearch vector store. It must be the model that " + f"produced the vectors in '{self.embedding_field}', for example openai/text-embedding-3-small." + ) + return self.litellm_embedding_model + + +class _SigningRegion(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + aws_region_name: str | None = None + + +class _SearchOptions(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + max_num_results: int | None = None + filters: Mapping[str, object] | None = None + ranking_options: Mapping[str, object] | None = None + rewrite_query: bool | None = None + + @property + def unsupported(self) -> tuple[str, ...]: + requested: Final = ( + ("filters", self.filters), + ("ranking_options", self.ranking_options), + ("rewrite_query", self.rewrite_query), + ) + return tuple(name for name, value in requested if value) + + @property + def k(self) -> int: + if self.max_num_results is None: + return DEFAULT_MAX_NUM_RESULTS + if not 1 <= self.max_num_results <= MAX_MAX_NUM_RESULTS: + raise _bad_request( + f"max_num_results must be between 1 and {MAX_MAX_NUM_RESULTS}, got {self.max_num_results}" + ) + return self.max_num_results + + +_KNOWN_PARAMS: Final = frozenset(name for name in _OpenSearchParams.model_fields if name.startswith(_PARAM_PREFIX)) + + +class _LoggedSearch(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + query: str | tuple[str, ...] = "" + litellm_params: _OpenSearchParams = _OpenSearchParams() + + @property + def search_query(self) -> str: + return BaseQueryEmbeddingVectorStoreConfig.query_text(self.query) + + +class _Hit(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + id: str = Field(alias="_id") + score: float | None = Field(default=None, alias="_score") + source: Mapping[str, object] = Field(default_factory=dict, alias="_source") + + +class _Hits(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + hits: Sequence[_Hit] + + +class _SearchResponse(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + hits: _Hits + + +def aws_signing_target(url: str) -> tuple[Literal["aoss", "es"], str | None]: + hostname: Final = urlsplit(url).hostname or "" + match: Final = next(filter(None, (pattern.search(hostname) for pattern in _AWS_HOSTS)), None) + if match is None: + return "es", None + return ("aoss" if match["service"].startswith("aoss") else "es"), match["region"] + + +def _search_result(hit: _Hit, params: _OpenSearchParams) -> VectorStoreSearchResult: + text: Final = hit.source.get(params.text_field) + filename: Final = hit.source.get("filename") + return VectorStoreSearchResult( + score=hit.score, + content=[VectorStoreResultContent(type="text", text=text)] if isinstance(text, str) else [], + file_id=hit.id, + filename=filename if isinstance(filename, str) else None, + attributes={ + key: value + for key, value in hit.source.items() + if key not in (params.text_field, params.embedding_field) and isinstance(value, _ATTRIBUTE_VALUE_TYPES) + }, + ) + + +class OpenSearchVectorStoreConfig(BaseQueryEmbeddingVectorStoreConfig, SignsRequestsWithAWS): + def __init__(self, aws: BaseAWSLLM | None = None) -> None: + self._aws: Final = aws if aws is not None else BaseAWSLLM() + + def get_auth_credentials(self, litellm_params: Mapping[str, object]) -> BaseVectorStoreAuthCredentials: + return BaseVectorStoreAuthCredentials() + + def get_vector_store_endpoints_by_type(self) -> VectorStoreIndexEndpoints: + return VectorStoreIndexEndpoints(read=[], write=[]) + + def validate_environment( + self, headers: Mapping[str, object], litellm_params: GenericLiteLLMParams | None + ) -> dict[str, object]: # mutable-ok: the shared HTTP handler requires writable headers + params: Final = dict(litellm_params or {}) + self._signing_credentials(params, self._signing_region(params, str(params.get("api_base") or ""))) + return {**headers, "Content-Type": "application/json"} + + def _signing_region(self, params: Mapping[str, object], api_base: str) -> str: + host_region: Final = aws_signing_target(api_base)[1] + return self._aws.get_aws_region_name_for_non_llm_api_calls( + host_region or _SigningRegion.model_validate(params).aws_region_name + ) + + def _signing_credentials(self, params: Mapping[str, object], region: str) -> "Credentials": + if params.get("api_key"): + raise _bad_request(_SIGV4_ONLY_MESSAGE) + auth: Final = AwsAuthParams.model_validate(params) + if bool(auth.aws_access_key_id) != bool(auth.aws_secret_access_key): + raise _unauthenticated("aws_access_key_id and aws_secret_access_key must be set together") + ensure_optional_import("botocore") + from botocore.exceptions import ( + CredentialRetrievalError, + NoCredentialsError, + PartialCredentialsError, + ProfileNotFound, + ) + + try: + return _usable_credentials(self._aws.resolve_credentials(auth, region)) + except CredentialRetrievalError as error: + verbose_logger.warning("OpenSearch vector store AWS credential source failed: %s", error) + raise _unauthenticated("the AWS credential source failed, see the LiteLLM logs") from None + except (NoCredentialsError, PartialCredentialsError, ProfileNotFound) as error: + raise _unauthenticated(str(error)) from None + + def get_complete_url(self, api_base: str | None, litellm_params: Mapping[str, object]) -> str: + parsed: Final = urlsplit(api_base or "") + if parsed.scheme not in ("http", "https") or not parsed.hostname: + raise _bad_request( + "api_base must be the OpenSearch endpoint URL, for example " + "https://..aoss.amazonaws.com or https://." + ) + if _ACCOUNT_ID_LABEL.match(parsed.hostname) and aws_signing_target(api_base or "")[0] == "aoss": + raise _bad_request(_PER_ACCOUNT_MESSAGE) + return (api_base or "").rstrip("/") + + def get_vector_store_url(self, api_base: str, encoded_vector_store_id: str) -> NoReturn: + raise _bad_request(_SEARCH_ONLY_MESSAGE) + + def get_vector_store_list_url(self, api_base: str) -> NoReturn: + raise _bad_request(_SEARCH_ONLY_MESSAGE) + + @staticmethod + def _params( + litellm_params: Mapping[str, object], + optional_params: VectorStoreSearchOptionalRequestParams, + extra_body: Mapping[str, object] | None, + ) -> tuple[_OpenSearchParams, int]: + unknown: Final = sorted( + key for key in litellm_params if key.startswith(_PARAM_PREFIX) and key not in _KNOWN_PARAMS + ) + if unknown: + raise _bad_request( + f"Unknown OpenSearch vector store parameter(s): {', '.join(unknown)}. " + f"Supported: {', '.join(sorted(_KNOWN_PARAMS))}." + ) + if extra_body: + raise _bad_request("The OpenSearch vector store does not support extra_body.") + try: + options: Final = _SearchOptions.model_validate(optional_params) + except ValidationError as error: + raise _bad_request(f"Invalid vector store search parameters: {error}") from None + if options.unsupported and not drops_unsupported_params(litellm_params.get("drop_params")): + raise _bad_request( + f"The OpenSearch vector store does not support {', '.join(options.unsupported)} yet. " + "Remove it from the request, or set drop_params to drop it." + ) + try: + return _OpenSearchParams.model_validate(litellm_params), options.k + except ValidationError: + raise _bad_request( + "opensearch_embedding_field, opensearch_index_name, opensearch_text_field, and litellm_embedding_model " + "must be strings." + ) from None + + @staticmethod + def _search_request( + vector_store_id: str, + embedding: EmbeddingResponse, + k: int, + params: _OpenSearchParams, + api_base: str, + ) -> tuple[str, dict[str, object]]: # mutable-ok: the provider contract returns a writable JSON request body + vector: Final = _QUERY_VECTOR.validate_python(embedding.data[0]["embedding"]) # pyright: ignore[reportUnknownMemberType] # EmbeddingResponse.data is an untyped list, the vector is validated here + return ( + f"{api_base}/{quote(params.index_name(vector_store_id), safe='')}/_search", + { + "size": k, + "query": {"knn": {params.embedding_field: {"vector": vector, "k": k}}}, + "_source": {"excludes": [params.embedding_field]}, + }, + ) + + def transform_search_vector_store_request( + self, + vector_store_id: str, + query: str | Sequence[str], + vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, + api_base: str, + litellm_logging_obj: "LiteLLMLoggingObj", + litellm_params: Mapping[str, object], + extra_body: Mapping[str, object] | None = None, + embedding_executor: VectorStoreEmbeddingExecutor | None = None, + ) -> tuple[str, dict[str, object]]: # mutable-ok: the provider contract returns a writable JSON request body + params, k = self._params(litellm_params, vector_store_search_optional_params, extra_body) + embedding: Final = self.query_embedding_executor(embedding_executor, None).embed( + params.require_embedding_model(), self.query_text(query), self.query_embedding_configuration(litellm_params) + ) + return self._search_request(vector_store_id, embedding, k, params, api_base) + + async def atransform_search_vector_store_request( + self, + vector_store_id: str, + query: str | Sequence[str], + vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, + api_base: str, + litellm_logging_obj: "LiteLLMLoggingObj", + litellm_params: Mapping[str, object], + extra_body: Mapping[str, object] | None = None, + embedding_executor: VectorStoreEmbeddingExecutor | None = None, + ) -> tuple[str, dict[str, object]]: # mutable-ok: the provider contract returns a writable JSON request body + params, k = self._params(litellm_params, vector_store_search_optional_params, extra_body) + embedding: Final = await self.query_embedding_executor(embedding_executor, None).aembed( + params.require_embedding_model(), self.query_text(query), self.query_embedding_configuration(litellm_params) + ) + return self._search_request(vector_store_id, embedding, k, params, api_base) + + def sign_request( + self, + headers: Mapping[str, str], + optional_params: Mapping[str, object], + request_data: Mapping[str, object], + api_base: str, + api_key: str | None = None, + ) -> tuple[dict[str, str], bytes]: # mutable-ok: the shared HTTP handler requires writable headers + region: Final = self._signing_region(optional_params, api_base) + body: Final = json.dumps(request_data) + signed: Final = sign_aws_json_post( + get_credentials=partial(self._signing_credentials, optional_params, region), + service_name=aws_signing_target(api_base)[0], + aws_region_name=region, + url=api_base, + body=body, + headers={ + "Content-Type": "application/json", + "x-amz-content-sha256": hashlib.sha256(body.encode()).hexdigest(), + }, + ) + signed_headers: Final = dict(signed.headers.items()) + signed_names: Final = frozenset(name.lower() for name in signed_headers) + return { + **{name: value for name, value in headers.items() if name.lower() not in signed_names}, + **signed_headers, + }, body.encode() + + def transform_search_vector_store_response( + self, response: httpx.Response, litellm_logging_obj: "LiteLLMLoggingObj" + ) -> VectorStoreSearchResponse: + if not response.is_success: + raise BaseLLMException(status_code=response.status_code, message=response.text, headers=response.headers) + try: + hits: Final = _SearchResponse.model_validate_json(response.content).hits.hits + except ValidationError: + raise BaseLLMException( + status_code=502, + message="OpenSearch answered the search without a hits.hits list.", + headers=response.headers, + ) from None + logged: Final = _LoggedSearch.model_validate(litellm_logging_obj.model_call_details) + return VectorStoreSearchResponse( + object="vector_store.search_results.page", + search_query=logged.search_query, + data=[_search_result(hit, logged.litellm_params) for hit in hits], + ) + + def validate_create_vector_store(self) -> NoReturn: + raise _bad_request(_SEARCH_ONLY_MESSAGE) + + def transform_create_vector_store_request( + self, vector_store_create_optional_params: VectorStoreCreateOptionalRequestParams, api_base: str + ) -> NoReturn: + raise _bad_request(_SEARCH_ONLY_MESSAGE) + + def transform_create_vector_store_response(self, response: httpx.Response) -> NoReturn: + raise _bad_request(_SEARCH_ONLY_MESSAGE) diff --git a/litellm/llms/sap/chat/transformation.py b/litellm/llms/sap/chat/transformation.py index 622c28f3c90..0270f814328 100755 --- a/litellm/llms/sap/chat/transformation.py +++ b/litellm/llms/sap/chat/transformation.py @@ -11,6 +11,7 @@ import httpx from pydantic import BaseModel import litellm +from litellm.litellm_core_utils.prompt_templates.common_utils import message_with_file_parts_as_image_urls from litellm.litellm_core_utils.url_utils import join_api_base from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse @@ -104,7 +105,7 @@ def _message_role_mapping(role: str) -> type[ChatMessage]: def _messages_to_sap_template(messages: list[AllMessageValues]) -> list: template: Final = [] for message in messages: - folded_message = _fold_message_cache_control(message) + folded_message = _fold_message_cache_control(message_with_file_parts_as_image_urls(message)) message_role_class = _message_role_mapping(str(folded_message["role"])) validated_message = validate_dict(folded_message, message_role_class) template.append(validated_message) diff --git a/litellm/llms/snowflake/chat/transformation.py b/litellm/llms/snowflake/chat/transformation.py index 030413030eb..928dcc3bc52 100644 --- a/litellm/llms/snowflake/chat/transformation.py +++ b/litellm/llms/snowflake/chat/transformation.py @@ -112,7 +112,7 @@ def _convert_image_url_to_anthropic(block: Mapping[str, object]) -> object: pdf/text data URIs become document blocks rather than malformed image blocks. """ image_url: Final = block.get("image_url") - url: Final = image_url if isinstance(image_url, str) else _image_url_field(image_url, "url") + url: Final = image_url if isinstance(image_url, str) else _str_field(image_url, "url") if not url: return block @@ -121,18 +121,31 @@ def _convert_image_url_to_anthropic(block: Mapping[str, object]) -> object: if select_anthropic_content_block_type_for_file(_data_uri_media_type(url)) == "document" else create_anthropic_image_param( image_url if isinstance(image_url, dict) else url, - format=_image_url_field(image_url, "format"), + format=_str_field(image_url, "format"), is_bedrock_invoke=True, ) ) + return _with_cache_control(converted, block) + + +def _convert_file_to_anthropic(block: Mapping[str, object]) -> object: + file_data: Final = _str_field(block.get("file"), "file_data") + if not file_data: + return block + return _with_cache_control( + anthropic_process_openai_file_message({"type": "file", "file": {"file_data": file_data}}), block + ) + + +def _with_cache_control(converted: Mapping[str, object], block: Mapping[str, object]) -> Mapping[str, object]: cache_control: Final = block.get("cache_control") if cache_control is None: return converted return {**converted, "cache_control": cache_control} -def _image_url_field(image_url: object, key: str) -> str | None: - value: Final = image_url.get(key) if isinstance(image_url, dict) else None +def _str_field(container: object, key: str) -> str | None: + value: Final = container.get(key) if isinstance(container, dict) else None return value if isinstance(value, str) else None @@ -141,15 +154,23 @@ def _data_uri_media_type(url: str) -> str: return match.group(1) if match else "" -def _convert_image_url_blocks_to_anthropic(content: object) -> object: +def _convert_media_block_to_anthropic(block: object) -> object: + mapping: Final = as_str_mapping(block) + if mapping is None: + return block + match mapping.get("type"): + case "image_url": + return _convert_image_url_to_anthropic(mapping) + case "file": + return _convert_file_to_anthropic(mapping) + case _: + return block + + +def _convert_media_blocks_to_anthropic(content: object) -> object: if not isinstance(content, list): return content - return [ - _convert_image_url_to_anthropic(block) - if isinstance(block, Mapping) and block.get("type") == "image_url" - else block - for block in content - ] + return [_convert_media_block_to_anthropic(block) for block in content] def _convert_tool_result_to_anthropic( @@ -391,7 +412,7 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig): conversation.append( { "role": role, - "content": _convert_image_url_blocks_to_anthropic(content), + "content": _convert_media_blocks_to_anthropic(content), } ) diff --git a/litellm/llms/soniox/common_utils.py b/litellm/llms/soniox/common_utils.py index 8d83b4c9218..796f6ca12e4 100644 --- a/litellm/llms/soniox/common_utils.py +++ b/litellm/llms/soniox/common_utils.py @@ -3,7 +3,7 @@ Shared utilities for the Soniox provider (https://soniox.com). """ from collections.abc import Mapping, Sequence -from typing import Final, TypeAlias +from typing import Final, Literal, TypeAlias from litellm.litellm_core_utils.audio_utils.subtitle_utils import ( SubtitleToken, @@ -17,6 +17,8 @@ SonioxToken: TypeAlias = Mapping[str, object] # Soniox API base URL. SONIOX_API_BASE: Final[str] = "https://api.soniox.com" +SONIOX_REALTIME_MODEL_PREFIX: Final = "stt-rt-" + # Default polling interval in seconds when waiting for an async transcription # to finish. Mirrors the Soniox SDK default. SONIOX_DEFAULT_POLL_INTERVAL: Final[float] = 1.0 @@ -54,6 +56,10 @@ class SonioxException(BaseLLMException): """Provider-specific exception class for Soniox.""" +def soniox_health_check_mode(model: str) -> Literal["realtime"] | None: + return "realtime" if model.startswith(SONIOX_REALTIME_MODEL_PREFIX) else None + + def get_soniox_api_key(api_key: str | None = None) -> str | None: """Resolve the Soniox API key from arg or env var.""" # Local import to avoid a circular import: litellm.secret_managers.main diff --git a/litellm/llms/soniox/realtime/__init__.py b/litellm/llms/soniox/realtime/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/soniox/realtime/transformation.py b/litellm/llms/soniox/realtime/transformation.py new file mode 100644 index 00000000000..ed7590b6f39 --- /dev/null +++ b/litellm/llms/soniox/realtime/transformation.py @@ -0,0 +1,609 @@ +import asyncio +import time +from collections import deque +from collections.abc import Awaitable, Callable, Mapping +from itertools import chain, groupby +from types import MappingProxyType, TracebackType +from typing import Final +from urllib.parse import urlparse, urlunparse + +from pydantic import BaseModel, ConfigDict, JsonValue, ValidationError +from typing_extensions import NotRequired, ReadOnly, Required, Self, TypedDict + +from litellm import verbose_logger +from litellm._uuid import uuid +from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.realtime.transcription_protocol import ( + SESSION_UPDATE_EVENT_TYPES, + RealtimeTranscriptionProtocolError, + TranscriptionSessionUpdate, + completed_event, + decode_pcm16_append, + delta_event, + duration_usage, + error_event, + json_object, + parse_transcription_session_update, + speech_event, + transcription_session, + transcription_session_created_event, + uses_server_vad, +) +from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeBackend +from litellm.llms.custom_httpx.http_handler import realtime_ssl_for_url +from litellm.llms.soniox.common_utils import SonioxException, get_soniox_api_key +from litellm.types.llms.openai import ( + OpenAIRealtimeErrorEvent, + OpenAIRealtimeEvents, + OpenAIRealtimeInputAudioTranscriptionCompleted, + OpenAIRealtimeInputAudioTranscriptionDelta, + OpenAIRealtimeTranscriptionSessionCreated, +) +from litellm.types.realtime import ( + RealtimeInputAudioTranscriptionUsage, + RealtimeResponseTransformInput, + RealtimeResponseTypedDict, +) + +DEFAULT_SONIOX_REALTIME_URL: Final = "wss://stt-rt.soniox.com/transcribe-websocket" +WEBSOCKET_SCHEMES: Final = MappingProxyType({"wss": "wss", "ws": "ws", "https": "wss", "http": "ws"}) +DEFAULT_SAMPLE_RATE: Final = 24_000 +KEEPALIVE_INTERVAL_SECONDS: Final = 5.0 +_MAX_PENDING_FRAMES: Final = 64 +_FINALIZE: Final = '{"type":"finalize"}' +_KEEPALIVE: Final = '{"type":"keepalive"}' +_END_STREAM: Final = "" +_HEALTH_CHECK_SILENCE: Final = bytes(DEFAULT_SAMPLE_RATE // 5) +_CONTROL_FRAMES: Final = frozenset((_FINALIZE, _KEEPALIVE, _END_STREAM)) +SESSION_STARTED_FRAME: Final = '{"type":"litellm.soniox.session_started"}' +_ENDPOINT_TOKEN: Final = "" +_FINALIZED_TOKEN: Final = "" + + +class SonioxProtocolError(RealtimeTranscriptionProtocolError): + pass + + +class SonioxRealtimeOptions(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + language_hints: tuple[str, ...] | None = None + language_hints_strict: bool | None = None + enable_language_identification: bool | None = None + enable_speaker_diarization: bool | None = None + context: JsonValue | None = None + translation: JsonValue | None = None + client_reference_id: str | None = None + max_endpoint_delay_ms: int | None = None + endpoint_sensitivity: float | None = None + endpoint_latency_adjustment_level: int | None = None + + +class SonioxStartRequest(SonioxRealtimeOptions): + model: str + audio_format: str + sample_rate: int + num_channels: int + enable_endpoint_detection: bool + + +class SonioxSpeechLabels(TypedDict, total=False): + speaker: ReadOnly[str] + language: ReadOnly[str] + + +class SonioxTranscriptSegment(SonioxSpeechLabels): + text: Required[ReadOnly[str]] + + +class SonioxTranscriptionDelta(OpenAIRealtimeInputAudioTranscriptionDelta, SonioxSpeechLabels): + pass + + +class SonioxTranscriptionCompleted(OpenAIRealtimeInputAudioTranscriptionCompleted): + segments: NotRequired[ReadOnly[tuple[SonioxTranscriptSegment, ...]]] + + +class _SonioxToken(BaseModel): + model_config = ConfigDict(frozen=True) + + text: str + is_final: bool = False + translation_status: str | None = None + speaker: str | None = None + language: str | None = None + + @property + def labelled(self) -> bool: + return self.speaker is not None or self.language is not None + + +class _SonioxResponse(BaseModel): + model_config = ConfigDict(frozen=True) + + tokens: tuple[_SonioxToken, ...] = () + total_audio_proc_ms: int | None = None + error_code: int | None = None + error_message: str | None = None + finished: bool = False + + +def _sample_rate(update: TranscriptionSessionUpdate | None) -> int: + audio_format: Final = None if update is None else update.audio_format + if audio_format is None: + return DEFAULT_SAMPLE_RATE + if not audio_format.is_pcm16: + raise SonioxProtocolError("Soniox realtime requires pcm16 input audio") + if audio_format.channels not in (None, 1): + raise SonioxProtocolError("Soniox realtime requires mono input audio") + if audio_format.rate is None: + return DEFAULT_SAMPLE_RATE + if audio_format.rate <= 0: + raise SonioxProtocolError("sample rate must be positive") + return audio_format.rate + + +def _language_hints( + language: str | None, configured: tuple[str, ...] | None, strict: bool | None +) -> tuple[str, ...] | None: + if not language or (strict and configured): + return configured + return (language, *(hint for hint in configured or () if hint != language)) + + +def _uses_endpoint_detection(update: TranscriptionSessionUpdate, drop_params: bool) -> bool: + return uses_server_vad(update, "Soniox realtime", drop_params, SonioxProtocolError) + + +def _ignored_settings(update: TranscriptionSessionUpdate) -> tuple[str, ...]: + vad_tuning: Final = (f"turn_detection.{key}" for key in sorted(update.turn_detection or {}) if key != "type") + return (*update.unsupported_transcription_keys, *vad_tuning) + + +def build_start_request( + model: str, update: TranscriptionSessionUpdate | None, options: SonioxRealtimeOptions, drop_params: bool = False +) -> SonioxStartRequest: + return SonioxStartRequest( + **options.model_dump(exclude={"language_hints"}), + language_hints=_language_hints( + None if update is None else update.language, options.language_hints, options.language_hints_strict + ), + model=model, + audio_format="pcm_s16le", + sample_rate=_sample_rate(update), + num_channels=1, + enable_endpoint_detection=update is None or _uses_endpoint_detection(update, drop_params), + ) + + +def _session_created(session_id: str, start_request: SonioxStartRequest) -> OpenAIRealtimeTranscriptionSessionCreated: + session: Final = transcription_session( + session_id=session_id, + model=start_request.model, + sample_rate=start_request.sample_rate, + language=start_request.language_hints[0] if start_request.language_hints else None, + server_vad=start_request.enable_endpoint_detection, + ) + return transcription_session_created_event(session) + + +def _changes_started_rate(update: TranscriptionSessionUpdate, started: SonioxStartRequest) -> bool: + audio_format: Final = update.audio_format + if audio_format is None: + return False + requested_rate: Final = _sample_rate(update) + return audio_format.rate is not None and requested_rate != started.sample_rate + + +def _changes_started_stream(update: TranscriptionSessionUpdate, started: SonioxStartRequest, drop_params: bool) -> bool: + turn_detection_set: Final = update.turn_detection is not None or update.turn_detection_disabled + return ( + _changes_started_rate(update, started) + or ( + update.language is not None + and _language_hints(update.language, started.language_hints, started.language_hints_strict) + != started.language_hints + ) + or (turn_detection_set and _uses_endpoint_detection(update, drop_params) != started.enable_endpoint_detection) + ) + + +def build_soniox_realtime_url(api_base: str | None) -> str: + if api_base is None: + return DEFAULT_SONIOX_REALTIME_URL + parsed: Final = urlparse(api_base) + scheme: Final = WEBSOCKET_SCHEMES.get(parsed.scheme) + if scheme is None: + raise ValueError("Soniox realtime api_base must be a wss://, ws://, https://, or http:// URL") + return urlunparse(parsed._replace(scheme=scheme)) + + +def _labels(token: _SonioxToken) -> SonioxSpeechLabels: + speaker: Final[SonioxSpeechLabels] = {} if token.speaker is None else {"speaker": token.speaker} + language: Final[SonioxSpeechLabels] = {} if token.language is None else {"language": token.language} + return {**speaker, **language} + + +def _delta(item_id: str, token: _SonioxToken) -> SonioxTranscriptionDelta: + return {**delta_event(item_id, token.text), **_labels(token)} + + +def _segment(run: tuple[_SonioxToken, ...]) -> SonioxTranscriptSegment: + return {**_labels(run[0]), "text": "".join(token.text for token in run).strip()} + + +def _segments(tokens: tuple[_SonioxToken, ...]) -> tuple[SonioxTranscriptSegment, ...]: + return tuple(_segment(tuple(run)) for _, run in groupby(tokens, key=lambda token: (token.speaker, token.language))) + + +def _new_item_id() -> str: + return f"item_{uuid.uuid4().hex}" + + +def _completed( + item_id: str, tokens: tuple[_SonioxToken, ...], usage: RealtimeInputAudioTranscriptionUsage | None +) -> OpenAIRealtimeInputAudioTranscriptionCompleted: + completed: Final = completed_event(item_id, "".join(token.text for token in tokens).strip(), usage) + if not any(token.labelled for token in tokens): + return completed + labelled: Final[SonioxTranscriptionCompleted] = {**completed, "segments": _segments(tokens)} + return labelled + + +def _soniox_error(response: _SonioxResponse) -> SonioxException | None: + if response.error_code is None and response.error_message is None: + return None + return SonioxException( + status_code=response.error_code or 500, + message=f"Soniox realtime error {response.error_code}: {response.error_message}", + ) + + +class SonioxEventTransformer: + def __init__(self, *, drops_originals: bool) -> None: + self._drops_originals: Final = drops_originals + self._item_id: str | None = None + self._final_tokens: Final[list[_SonioxToken]] = [] # mutable-ok: a turn's tokens append in O(1) each + self._finalizes: Final[deque[bool]] = deque() # mutable-ok: pending finalizes queue and pop in O(1) each + self._oldest_commit_answered: bool = False + self._processed_ms: int = 0 + self._billed_ms: int = 0 + self._refusal: SonioxException | None = None + + @property + def refusal(self) -> SonioxException | None: + return self._refusal + + def transform(self, payload: str) -> tuple[OpenAIRealtimeEvents, ...]: + try: + response: Final = _SonioxResponse.model_validate_json(payload) + except ValidationError: + raise SonioxProtocolError("Soniox returned an invalid realtime response") from None + self._bill(response.total_audio_proc_ms) + error: Final = _soniox_error(response) + if error is not None: + return (self._report(error),) + return tuple(chain.from_iterable(self._token_events(token) for token in response.tokens)) + + def queue_finalize(self, *, discards_turn: bool) -> None: + if discards_turn and not self._finalizes: + self._item_id = None + self._final_tokens.clear() + self._finalizes.append(discards_turn) + + def take_unbilled_usage(self, stream_ms: int = 0) -> RealtimeInputAudioTranscriptionUsage | None: + billable_ms: Final = max(self._processed_ms, stream_ms) + unbilled_ms: Final = billable_ms - self._billed_ms + if unbilled_ms <= 0: + return None + self._billed_ms = billable_ms + return duration_usage(unbilled_ms / 1000) + + def _report(self, error: SonioxException) -> OpenAIRealtimeErrorEvent: + if self._processed_ms == 0: + self._refusal = error + return error_event(error.message) + + def _bill(self, total_audio_proc_ms: int | None) -> None: + if total_audio_proc_ms is not None and total_audio_proc_ms > self._processed_ms: + self._processed_ms = total_audio_proc_ms + + def _token_events(self, token: _SonioxToken) -> tuple[OpenAIRealtimeEvents, ...]: + if token.text == _FINALIZED_TOKEN: + return self._finalized() + if self._discarding: + return () + if token.text == _ENDPOINT_TOKEN: + return self._end_turn() + started: Final = self._start() + if not token.is_final or (self._drops_originals and token.translation_status == "original"): + return started + self._final_tokens.append(token) + return (*started, _delta(self._current_item_id(), token)) + + @property + def _discarding(self) -> bool: + return bool(self._finalizes) and self._finalizes[0] + + def _finalized(self) -> tuple[OpenAIRealtimeEvents, ...]: + discarded: Final = self._finalizes.popleft() if self._finalizes else None + answered: Final = self._oldest_commit_answered + self._oldest_commit_answered = False + if discarded: + return () + if self._item_id is not None: + return self._complete(speech_stopped=False) + if discarded is None or answered: + return () + return (_completed(_new_item_id(), (), self.take_unbilled_usage()),) + + def _end_turn(self) -> tuple[OpenAIRealtimeEvents, ...]: + events: Final = self._complete(speech_stopped=True) + if events and self._finalizes: + self._oldest_commit_answered = True + return events + + def _start(self) -> tuple[OpenAIRealtimeEvents, ...]: + if self._item_id is not None: + return () + item_id: Final = _new_item_id() + self._item_id = item_id + return (speech_event("input_audio_buffer.speech_started", item_id),) + + def _current_item_id(self) -> str: + assert self._item_id is not None + return self._item_id + + def _complete(self, *, speech_stopped: bool) -> tuple[OpenAIRealtimeEvents, ...]: + item_id: Final = self._item_id + if item_id is None: + return () + tokens: Final = tuple(self._final_tokens) + self._item_id = None + self._final_tokens.clear() + stopped: Final[tuple[OpenAIRealtimeEvents, ...]] = ( + (speech_event("input_audio_buffer.speech_stopped", item_id),) if speech_stopped else () + ) + return (*stopped, _completed(item_id, tokens, self.take_unbilled_usage())) + + +class SonioxRealtimeBackend: + def __init__( + self, + inner: RealtimeBackend, + *, + keepalive_interval: float = KEEPALIVE_INTERVAL_SECONDS, + clock: Callable[[], float] = time.monotonic, + sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, + ) -> None: + self._inner: Final = inner + self._interval: Final = keepalive_interval + self._clock: Final = clock + self._sleep: Final = sleep + self._frames: Final[asyncio.Queue[str | bytes | Exception]] = asyncio.Queue(maxsize=_MAX_PENDING_FRAMES) + self._last_sent: float = clock() + self._started: bool = False + self._ended: bool = False + self._tasks: tuple[asyncio.Task[None], ...] = () + + async def __aenter__(self) -> Self: + await self._inner.__aenter__() + self._tasks = (asyncio.create_task(self._pump()), asyncio.create_task(self._keepalive())) + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + self._stop() + await self._inner.__aexit__(exc_type, exc_value, traceback) + + async def send(self, message: str | bytes) -> None: + self._last_sent = self._clock() + self._ended = self._ended or message == _END_STREAM + await self._inner.send(message) + if not self._started and isinstance(message, str) and message not in _CONTROL_FRAMES: + self._started = True + await self._frames.put(SESSION_STARTED_FRAME) + + async def recv(self, decode: bool | None = None) -> str | bytes: + frame: Final = await self._frames.get() + if isinstance(frame, Exception): + raise frame + return frame + + async def close(self) -> None: + self._stop() + await self._inner.close() + + def _stop(self) -> None: + for task in self._tasks: + task.cancel() + + async def _pump(self) -> None: + while True: + try: + frame = await self._inner.recv() + except Exception as e: # noqa: BLE001 # recv() re-raises it, so an upstream failure never hangs the reader + await self._frames.put(e) + return + await self._frames.put(frame) + + async def _keepalive(self) -> None: + while not self._ended: + await self._sleep(self._interval) + if self._ended or self._clock() - self._last_sent < self._interval: + continue + try: + await self.send(_KEEPALIVE) + except Exception as e: # noqa: BLE001 # stops only keepalives; the pump reports the broken socket + verbose_logger.debug("Soniox realtime: stopping keepalive after send failed: %s", e) + return + + +class SonioxRealtimeConfig(BaseRealtimeConfig): + def __init__( + self, + *, + options: SonioxRealtimeOptions | None = None, + wrap: Callable[[RealtimeBackend], RealtimeBackend] = SonioxRealtimeBackend, + clock: Callable[[], float] = time.monotonic, + drop_params: bool = False, + ) -> None: + self._options: Final = options or SonioxRealtimeOptions() + self._drop_params: Final = drop_params + self._wrap: Final = wrap + self._clock: Final = clock + self._transformer: Final = SonioxEventTransformer(drops_originals=self._options.translation is not None) + self._start_request: SonioxStartRequest | None = None + self._session_id: str = f"sess_{uuid.uuid4().hex}" + self._started_at: float | None = None + + @classmethod + def from_litellm_params(cls, litellm_params: Mapping[str, object], *, drop_params: bool = False) -> Self: + return cls(options=SonioxRealtimeOptions.model_validate(dict(litellm_params)), drop_params=drop_params) + + def requires_session_configuration(self) -> bool: + return True + + def validate_environment( + self, + headers: dict[str, str], # mutable-ok: BaseRealtimeConfig contract + model: str, + api_key: str | None = None, + ) -> dict[str, str]: # mutable-ok: BaseRealtimeConfig contract + resolved_key: Final = get_soniox_api_key(api_key) + if not resolved_key: + raise ValueError("Missing Soniox API key. Set SONIOX_API_KEY or api_key in the deployment litellm_params") + return {**headers, "Authorization": f"Bearer {resolved_key}"} + + def get_complete_url(self, api_base: str | None, model: str, api_key: str | None = None) -> str: + return build_soniox_realtime_url(api_base) + + def wrap_backend(self, backend: RealtimeBackend) -> RealtimeBackend: + return self._wrap(backend) + + def transform_session_created_event( + self, + model: str, + logging_session_id: str, + session_configuration_request: str | None = None, + ) -> OpenAIRealtimeTranscriptionSessionCreated: + self._session_id = logging_session_id + return _session_created(logging_session_id, build_start_request(model, None, self._options)) + + def transform_realtime_request( + self, + message: str, + model: str, + session_configuration_request: str | None = None, + ) -> tuple[str | bytes, ...]: + request: Final = json_object(message, SonioxProtocolError) + event_type: Final = request.get("type") + if event_type is None: + raise SonioxProtocolError("Soniox realtime takes OpenAI realtime events, and this message has no type") + if event_type in SESSION_UPDATE_EVENT_TYPES: + update: Final = parse_transcription_session_update(message, SonioxProtocolError) + ignored: Final = _ignored_settings(update) + if ignored: + verbose_logger.warning("Soniox realtime: ignoring unsupported session settings %s", ignored) + return self._start(model, update) + if event_type == "input_audio_buffer.append": + audio: Final = decode_pcm16_append(request.get("audio"), error=SonioxProtocolError) + return (*self._start(model, None), audio) + if event_type == "input_audio_buffer.commit": + return self._finalize(model, discards_turn=False) + if event_type == "input_audio_buffer.clear": + return () if self._start_request is None else self._finalize(model, discards_turn=True) + if event_type == "input_audio_buffer.end": + return (*self._start(model, None), _END_STREAM) + verbose_logger.debug("Soniox realtime: dropping unsupported client event %s", event_type) + return () + + def _finalize(self, model: str, *, discards_turn: bool) -> tuple[str, ...]: + start: Final = self._start(model, None) + self._transformer.queue_finalize(discards_turn=discards_turn) + return (*start, _FINALIZE) + + def session_refusal(self) -> Exception | None: + return self._transformer.refusal + + def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None: + started_at: Final = self._started_at + if self._transformer.refusal is not None: + return None + if started_at is None: + return self._transformer.take_unbilled_usage() + return self._transformer.take_unbilled_usage(int((self._clock() - started_at) * 1000)) + + def transform_realtime_response( + self, + message: str | bytes, + model: str, + logging_obj: LiteLLMLoggingObj, + realtime_response_transform_input: RealtimeResponseTransformInput, + ) -> RealtimeResponseTypedDict: + payload: Final = message.decode("utf-8") if isinstance(message, bytes) else message + result: Final[RealtimeResponseTypedDict] = { + "response": list(self._backend_events(payload)), + "current_output_item_id": realtime_response_transform_input.get("current_output_item_id"), + "current_response_id": realtime_response_transform_input.get("current_response_id"), + "current_delta_chunks": realtime_response_transform_input.get("current_delta_chunks"), + "current_conversation_id": realtime_response_transform_input.get("current_conversation_id"), + "current_item_chunks": realtime_response_transform_input.get("current_item_chunks"), + "current_delta_type": realtime_response_transform_input.get("current_delta_type"), + "session_configuration_request": realtime_response_transform_input.get("session_configuration_request"), + } + return result + + def _backend_events(self, payload: str) -> tuple[OpenAIRealtimeEvents, ...]: + if payload != SESSION_STARTED_FRAME: + return self._transformer.transform(payload) + assert self._start_request is not None + return (_session_created(self._session_id, self._start_request),) + + def _start(self, model: str, update: TranscriptionSessionUpdate | None) -> tuple[str, ...]: + started: Final = self._start_request + if started is not None: + if update is not None and _changes_started_stream(update, started, self._drop_params): + raise SonioxProtocolError("Soniox realtime can't change the session once the stream has started") + return () + start_request: Final = build_start_request(model, update, self._options, self._drop_params) + self._start_request = start_request + self._started_at = self._clock() + return (start_request.model_dump_json(exclude_none=True),) + + +def _finishes_stream(frame: str | bytes) -> bool: + response: Final = _SonioxResponse.model_validate_json(frame) + error: Final = _soniox_error(response) + if error is not None: + raise error + return response.finished + + +async def soniox_realtime_health_check( + model: str, api_base: str | None, api_key: str | None, options: SonioxRealtimeOptions +) -> bool: + import websockets + + config: Final = SonioxRealtimeConfig() + url: Final = config.get_complete_url(api_base, model) + headers: Final = config.validate_environment({}, model, api_key) + start_request: Final = build_start_request(model, None, options) + async with websockets.connect( + url, + additional_headers=headers, + max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, + ssl=realtime_ssl_for_url(url), + ) as backend: + await backend.send(start_request.model_dump_json(exclude_none=True)) + await backend.send(_HEALTH_CHECK_SILENCE) + await backend.send(_END_STREAM) + async for frame in backend: + if _finishes_stream(frame): + return True + raise SonioxException(status_code=502, message="Soniox closed the realtime stream without finishing it") diff --git a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py index 5bff43828d6..884c82fe331 100644 --- a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py +++ b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py @@ -1,4 +1,5 @@ from collections.abc import Sequence +from dataclasses import dataclass from typing import Final, Literal import httpx @@ -8,7 +9,6 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger from litellm.caching.caching import Cache, LiteLLMCacheType -from litellm.constants import MINIMUM_PROMPT_CACHE_TOKEN_COUNT from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, @@ -16,10 +16,11 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, ) from litellm.llms.openai.openai import AllMessageValues -from litellm.utils import is_prompt_caching_valid_prompt +from litellm.types.llms.openai import ChatCompletionToolParam +from litellm.utils import get_prompt_cache_min_tokens, is_prompt_caching_valid_prompt from ..common_utils import VertexAIError, get_vertex_base_url -from ..vertex_llm_base import VertexBase +from ..vertex_llm_base import VertexBase, publisher_model_location from .known_cached_contents import KnownCachedContents, known_cached_contents from .transformation import ( cached_messages_end_on_supported_turn, @@ -65,6 +66,19 @@ class _CreatedCachedContentRoot(RootModel[_CreatedCachedContent]): model_config = ConfigDict(frozen=True, hide_input_in_errors=True) +_TOOL_PARAMS: Final = ("tools", "tool_choice") + + +def _sync_cache_client(timeout: float | httpx.Timeout | None) -> HTTPHandler: + if timeout is None: + return HTTPHandler() + return HTTPHandler(timeout=httpx.Timeout(timeout) if isinstance(timeout, (float, int)) else timeout) + + +class _CachedContentsNotServedError(VertexAIError): + pass + + def _record_cache_creation_tokens(logging_obj: Logging, created: _CreatedCachedContent) -> None: tokens: Final = (created.get("usageMetadata") or {}).get("totalTokenCount") or 0 logging_obj.vertex_ai_cache_creation_input_tokens = max(tokens, 0) # rebind-ok: set provider count @@ -78,6 +92,61 @@ def _wait_seconds(timeout: float | httpx.Timeout | None) -> float | None: return timeout +@dataclass(frozen=True, slots=True) +class _CacheTarget: + model: str + project: str | None + location: str | None + + +def _cache_target( + custom_llm_provider: Literal["vertex_ai", "vertex_ai_beta", "gemini"], + api_base: str | None, + model: str, + vertex_project: str | None, + vertex_location: str | None, +) -> _CacheTarget: + model_location: Final = ( + None if custom_llm_provider == "gemini" or api_base is None else publisher_model_location(api_base) + ) + if model_location is None: + return _CacheTarget(model=model, project=vertex_project, location=vertex_location) + return _CacheTarget(model=model_location.model, project=model_location.project, location=model_location.location) + + +def _prompt_cache_min_tokens(models: Sequence[str], custom_llm_provider: str) -> int: + return max( + get_prompt_cache_min_tokens( + model if model.startswith(custom_llm_provider) else f"{custom_llm_provider}/{model}" + ) + for model in models + ) + + +def _context_caching_skip_reason( + cached_messages: Sequence[AllMessageValues], + tools: Sequence[ChatCompletionToolParam] | None, + deployment_model: str, + cache_model: str, + custom_llm_provider: Literal["vertex_ai", "vertex_ai_beta", "gemini"], +) -> str | None: + if not cached_messages_end_on_supported_turn(cached_messages): + return ( + "Vertex AI context caching: cached message block ends on a model turn once " + "system messages are extracted, which the cachedContents API rejects." + ) + min_tokens: Final = _prompt_cache_min_tokens((deployment_model, cache_model), custom_llm_provider) + if not is_prompt_caching_valid_prompt( + model=cache_model, + messages=list(cached_messages), + tools=None if tools is None else list(tools), + custom_llm_provider=custom_llm_provider, + min_token_count=min_tokens, + ): + return f"Vertex AI context caching: cached content is below minimum token count ({min_tokens})." + return None + + class ContextCachingEndpoints(VertexBase): """ Covers context caching endpoints for Vertex AI + Google AI Studio @@ -135,6 +204,7 @@ class ContextCachingEndpoints(VertexBase): vertex_project=vertex_project, vertex_location=vertex_location, vertex_api_version=("v1beta1" if custom_llm_provider == "vertex_ai_beta" else "v1"), + collection_endpoint=True, ) def check_cache( @@ -206,6 +276,8 @@ class ContextCachingEndpoints(VertexBase): except httpx.HTTPStatusError as e: if e.response.status_code == 403: return None + if e.response.status_code == 404 and custom_llm_provider == "gemini": + raise _CachedContentsNotServedError(status_code=404, message=e.response.text) raise VertexAIError(status_code=e.response.status_code, message=e.response.text) except Exception as e: raise VertexAIError(status_code=500, message=str(e)) @@ -305,6 +377,8 @@ class ContextCachingEndpoints(VertexBase): except httpx.HTTPStatusError as e: if e.response.status_code == 403: return None + if e.response.status_code == 404 and custom_llm_provider == "gemini": + raise _CachedContentsNotServedError(status_code=404, message=e.response.text) raise VertexAIError(status_code=e.response.status_code, message=e.response.text) except Exception as e: raise VertexAIError(status_code=500, message=str(e)) @@ -370,41 +444,25 @@ class ContextCachingEndpoints(VertexBase): if len(cached_messages) == 0: return messages, optional_params, None - if not cached_messages_end_on_supported_turn(cached_messages): - verbose_logger.debug( - "Vertex AI context caching: cached message block ends on a model turn once " - "system messages are extracted, which the cachedContents API rejects. " - "Skipping context caching." - ) + target: Final = _cache_target(custom_llm_provider, api_base, model, vertex_project, vertex_location) + tools: Final = optional_params.get("tools") + tool_choice: Final = optional_params.get("tool_choice") + skip_reason: Final = _context_caching_skip_reason( + cached_messages, tools, model, target.model, custom_llm_provider + ) + if skip_reason is not None: + verbose_logger.debug("%s Skipping context caching.", skip_reason) return messages, optional_params, None - # Gemini requires a minimum of 1024 tokens for context caching. - # Skip caching if the cached content is too small to avoid API errors. - if not is_prompt_caching_valid_prompt( - model=model, - messages=cached_messages, - tools=optional_params.get("tools"), - custom_llm_provider=custom_llm_provider, - ): - verbose_logger.debug( - "Vertex AI context caching: cached content is below minimum token " - "count (%d). Skipping context caching.", - MINIMUM_PROMPT_CACHE_TOKEN_COUNT, - ) - return messages, optional_params, None - - tools: Final = optional_params.pop("tools", None) - tool_choice: Final = optional_params.pop("tool_choice", None) - ## AUTHORIZATION ## token, url = self._get_token_and_url_context_caching( gemini_api_key=api_key, custom_llm_provider=custom_llm_provider, api_base=api_base, - vertex_project=vertex_project, - vertex_location=vertex_location, + vertex_project=target.project, + vertex_location=target.location, vertex_auth_header=vertex_auth_header, - model=model, + model=target.model, ) headers: Final = { @@ -417,44 +475,42 @@ class ContextCachingEndpoints(VertexBase): if extra_headers is not None: headers.update(extra_headers) - if client is None or not isinstance(client, HTTPHandler): - _params: Final = {} - if timeout is not None: - if isinstance(timeout, float) or isinstance(timeout, int): - timeout = httpx.Timeout(timeout) - _params["timeout"] = timeout - client = HTTPHandler(**_params) - else: - client = client + http_client: Final = client if isinstance(client, HTTPHandler) else _sync_cache_client(timeout) ## CHECK IF CACHED ALREADY generated_cache_key: Final = local_cache_obj.get_cache_key( - messages=cached_messages, tools=tools, tool_choice=tool_choice, model=model - ) - google_cache_name: Final = self.check_cache( - cache_key=generated_cache_key, - client=client, - headers=headers, - api_key=api_key, - api_base=api_base, - logging_obj=logging_obj, - custom_llm_provider=custom_llm_provider, - vertex_project=vertex_project, - vertex_location=vertex_location, - vertex_auth_header=vertex_auth_header, - model=model, + messages=cached_messages, tools=tools, tool_choice=tool_choice, model=target.model ) + try: + google_cache_name: Final = self.check_cache( + cache_key=generated_cache_key, + client=http_client, + headers=headers, + api_key=api_key, + api_base=api_base, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + vertex_project=target.project, + vertex_location=target.location, + vertex_auth_header=vertex_auth_header, + model=target.model, + ) + except _CachedContentsNotServedError: + verbose_logger.debug("Gemini context caching: %s answers 404. Skipping context caching.", url) + return messages, optional_params, None + for tool_param in _TOOL_PARAMS: + optional_params.pop(tool_param, None) if google_cache_name: return non_cached_messages, optional_params, google_cache_name ## TRANSFORM REQUEST cached_content_request_body: Final = transform_openai_messages_to_gemini_context_caching( - model=model, + model=target.model, messages=cached_messages, cache_key=generated_cache_key, custom_llm_provider=custom_llm_provider, - vertex_project=vertex_project, - vertex_location=vertex_location, + vertex_project=target.project, + vertex_location=target.location, ) cached_content_request_body["tools"] = tools @@ -473,7 +529,7 @@ class ContextCachingEndpoints(VertexBase): ) try: - response: Final = client.post( + response: Final = http_client.post( url=url, headers=headers, json=cached_content_request_body, @@ -489,7 +545,7 @@ class ContextCachingEndpoints(VertexBase): _record_cache_creation_tokens(logging_obj, raw_response_cached) self._known_caches.remember( KnownCachedContents.scope( - url, custom_llm_provider, headers, vertex_project=vertex_project, vertex_location=vertex_location + url, custom_llm_provider, headers, vertex_project=target.project, vertex_location=target.location ), generated_cache_key, raw_response_cached.get("name"), @@ -536,41 +592,25 @@ class ContextCachingEndpoints(VertexBase): if len(cached_messages) == 0: return messages, optional_params, None - if not cached_messages_end_on_supported_turn(cached_messages): - verbose_logger.debug( - "Vertex AI context caching: cached message block ends on a model turn once " - "system messages are extracted, which the cachedContents API rejects. " - "Skipping context caching." - ) + target: Final = _cache_target(custom_llm_provider, api_base, model, vertex_project, vertex_location) + tools: Final = optional_params.get("tools") + tool_choice: Final = optional_params.get("tool_choice") + skip_reason: Final = _context_caching_skip_reason( + cached_messages, tools, model, target.model, custom_llm_provider + ) + if skip_reason is not None: + verbose_logger.debug("%s Skipping context caching.", skip_reason) return messages, optional_params, None - # Gemini requires a minimum of 1024 tokens for context caching. - # Skip caching if the cached content is too small to avoid API errors. - if not is_prompt_caching_valid_prompt( - model=model, - messages=cached_messages, - tools=optional_params.get("tools"), - custom_llm_provider=custom_llm_provider, - ): - verbose_logger.debug( - "Vertex AI context caching: cached content is below minimum token " - "count (%d). Skipping context caching.", - MINIMUM_PROMPT_CACHE_TOKEN_COUNT, - ) - return messages, optional_params, None - - tools: Final = optional_params.pop("tools", None) - tool_choice: Final = optional_params.pop("tool_choice", None) - ## AUTHORIZATION ## token, url = self._get_token_and_url_context_caching( gemini_api_key=api_key, custom_llm_provider=custom_llm_provider, api_base=api_base, - vertex_project=vertex_project, - vertex_location=vertex_location, + vertex_project=target.project, + vertex_location=target.location, vertex_auth_header=vertex_auth_header, - model=model, + model=target.model, ) headers: Final = { @@ -590,10 +630,10 @@ class ContextCachingEndpoints(VertexBase): ## CHECK IF CACHED ALREADY generated_cache_key: Final = local_cache_obj.get_cache_key( - messages=cached_messages, tools=tools, tool_choice=tool_choice, model=model + messages=cached_messages, tools=tools, tool_choice=tool_choice, model=target.model ) scope: Final = KnownCachedContents.scope( - url, custom_llm_provider, headers, vertex_project=vertex_project, vertex_location=vertex_location + url, custom_llm_provider, headers, vertex_project=target.project, vertex_location=target.location ) async def lookup_or_create() -> str | None: @@ -605,10 +645,10 @@ class ContextCachingEndpoints(VertexBase): api_base=api_base, logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, - vertex_project=vertex_project, - vertex_location=vertex_location, + vertex_project=target.project, + vertex_location=target.location, vertex_auth_header=vertex_auth_header, - model=model, + model=target.model, ) if google_cache_name: @@ -616,12 +656,12 @@ class ContextCachingEndpoints(VertexBase): ## TRANSFORM REQUEST cached_content_request_body: Final = transform_openai_messages_to_gemini_context_caching( - model=model, + model=target.model, messages=cached_messages, cache_key=generated_cache_key, custom_llm_provider=custom_llm_provider, - vertex_project=vertex_project, - vertex_location=vertex_location, + vertex_project=target.project, + vertex_location=target.location, ) cached_content_request_body["tools"] = tools @@ -659,9 +699,15 @@ class ContextCachingEndpoints(VertexBase): ) return raw_response_cached.get("name") - cache_name: Final = await self._known_caches.single_flight( - scope, generated_cache_key, lookup_or_create, _wait_seconds(timeout) - ) + try: + cache_name: Final = await self._known_caches.single_flight( + scope, generated_cache_key, lookup_or_create, _wait_seconds(timeout) + ) + except _CachedContentsNotServedError: + verbose_logger.debug("Gemini context caching: %s answers 404. Skipping context caching.", url) + return messages, optional_params, None + for tool_param in _TOOL_PARAMS: + optional_params.pop(tool_param, None) return non_cached_messages, optional_params, cache_name def get_cache(self): diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py index 1f0ee728cd1..bfa94396ea7 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py @@ -211,6 +211,12 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert if messages_carry_output_config(messages): beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value) + beta_values.update( + anthropic_model_info.thinking_betas( + optional_params.get("thinking"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # optional_params is the base config's bare dict + ) + ) + if beta_values: request_headers["anthropic-beta"] = ",".join(beta_values) @@ -245,7 +251,9 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert headers=headers, ) - self._normalize_system_role_messages(anthropic_messages_request, model=model) + self._normalize_system_role_messages( + anthropic_messages_request, model=model, drop_params=getattr(litellm_params, "drop_params", None) + ) self._remove_scope_from_cache_control(anthropic_messages_request) diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py index 363e5596cb9..79fd9f9e8b1 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py @@ -145,6 +145,7 @@ class VertexAIAnthropicConfig(AnthropicConfig): file_id_used=self.is_file_id_used(messages), mcp_server_used=self.is_mcp_server_used(optional_params.get("mcp_servers")), custom_llm_provider="vertex_ai", + thinking=optional_params.get("thinking"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # optional_params is the base config's bare dict ) beta_set: Final = set(auto_betas) diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py index 4be41e26326..180761f15d9 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py @@ -6,9 +6,12 @@ Unlike Gemini models which use Google's token counting API, partner models use their respective publisher-specific count-tokens endpoints. """ +from collections.abc import Mapping from typing import Any, Final +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.anthropic.common_utils import OBJECT_LIST_ADAPTER, is_claude_model_target, tools_without_custom_field from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.llms.vertex_ai.common_utils import get_vertex_base_url from litellm.llms.vertex_ai.vertex_llm_base import VertexBase @@ -20,6 +23,22 @@ _COUNT_TOKENS_PUBLISHERS: Final = ( ) +def count_tokens_request_body( + request_data: Mapping[str, object], model: str, drop_params: object +) -> Mapping[str, object]: + tools: Final = request_data.get("tools") + if not isinstance(tools, list): + return request_data + return { + **request_data, + "tools": tools_without_custom_field( + OBJECT_LIST_ADAPTER.validate_python(tools), + hoist_defer_loading=is_claude_model_target("vertex_ai", model), + drop_custom=drops_unsupported_params(drop_params), + ), + } + + def count_tokens_publisher(model: str) -> str | None: return next((publisher for marker, publisher in _COUNT_TOKENS_PUBLISHERS if marker in model), None) @@ -169,7 +188,7 @@ class VertexAIPartnerModelsTokenCounter(VertexBase): response: Final = await async_client.post( endpoint_url, headers=headers, - json=request_data, + json=dict(count_tokens_request_body(request_data, model, litellm_params.get("drop_params"))), timeout=30.0, ) diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py index bb200fd542d..a3a3427670d 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py @@ -9,6 +9,7 @@ import httpx import litellm from litellm import LlmProviders +from litellm.litellm_core_utils.prompt_templates.common_utils import messages_with_file_parts_as_image_urls from litellm.types.llms.vertex_ai import VertexPartnerProvider from litellm.utils import ModelResponse @@ -217,7 +218,7 @@ class VertexAIPartnerModels(VertexBase): return base_llm_http_handler.completion( model=model, stream=stream, - messages=messages, + messages=list(messages_with_file_parts_as_image_urls(messages)), acompletion=acompletion, api_base=api_base, model_response=model_response, diff --git a/litellm/llms/vertex_ai/vertex_llm_base.py b/litellm/llms/vertex_ai/vertex_llm_base.py index 0cdd92bb071..7dd98212f76 100644 --- a/litellm/llms/vertex_ai/vertex_llm_base.py +++ b/litellm/llms/vertex_ai/vertex_llm_base.py @@ -6,9 +6,11 @@ Handles Authentication and generating request urls for Vertex AI and Google AI S import asyncio import json +import re import threading from collections import OrderedDict from collections.abc import Mapping +from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Final, Literal, Protocol from urllib.parse import urlparse @@ -46,6 +48,31 @@ def graft_default_vertex_path(api_base: str, default_url: str) -> str: _graft_default_vertex_path = graft_default_vertex_path + +_PUBLISHER_MODEL_LOCATION: Final = re.compile(r".*/projects/[^/]+/locations/[^/]+(?=/publishers/[^/]+/models/[^/]+/?$)") + + +@dataclass(frozen=True, slots=True) +class PublisherModelLocation: + parent_url: str + project: str + location: str + model: str + + +def publisher_model_location(api_base: str) -> PublisherModelLocation | None: + match: Final = _PUBLISHER_MODEL_LOCATION.match(api_base) + if match is None: + return None + _, _, project, _, location = match.group().rsplit("/", 4) + return PublisherModelLocation( + parent_url=match.group(), + project=project, + location=location, + model=api_base.rstrip("/").rsplit("/", 1)[-1], + ) + + GOOGLE_IMPORT_ERROR_MESSAGE: Final = ( "Google Cloud SDK not found. Install it with: pip install 'litellm[google]' or pip install google-cloud-aiplatform" ) @@ -637,15 +664,18 @@ class VertexBase: vertex_location: str | None = None, vertex_api_version: Literal["v1", "v1beta1"] | None = None, use_psc_endpoint_format: bool = False, + collection_endpoint: bool = False, ) -> tuple[str | None, str]: """ for cloudflare ai gateway - https://github.com/BerriAI/litellm/issues/4317 Handles custom api_base for: - 1. Gemini (Google AI Studio) - constructs /models/{model}:{endpoint} + 1. Gemini (Google AI Studio) - constructs /models/{model}:{endpoint}, or {api_base}/{endpoint} + for a collection endpoint 2. Vertex AI with standard proxies - grafts the default vertex URL path onto the api_base when its path is empty or only an API version (/v1, /v1beta1); - otherwise constructs {api_base}:{endpoint} + otherwise constructs {api_base}:{endpoint}, or, for a collection endpoint under an api_base + naming a publisher model, the collection under that model's projects/{project}/locations/{location} 3. Vertex AI with PSC endpoints - constructs full path structure {api_base}/v1/projects/{project}/locations/{location}/endpoints/{model}:{endpoint} (only when use_psc_endpoint_format=True) @@ -653,22 +683,22 @@ class VertexBase: Args: use_psc_endpoint_format: If True, constructs PSC endpoint URL format. If False (default), uses api_base as-is and appends :{endpoint} + collection_endpoint: True when `endpoint` names a collection resource such as + cachedContents rather than a model action such as generateContent ## Returns - (auth_header, url) - Tuple[Optional[str], str] """ if api_base: if custom_llm_provider == "gemini": - # For Gemini (Google AI Studio), construct the full path like other providers - if model is None: + if model is None and not collection_endpoint: raise ValueError("Model parameter is required for Gemini custom API base URLs") - url = join_api_base(api_base, f"models/{model}:{endpoint}") + url = join_api_base(api_base, endpoint if collection_endpoint else f"models/{model}:{endpoint}") if gemini_api_key is None: raise ValueError( "Missing Gemini API key. Set the GEMINI_API_KEY or GOOGLE_API_KEY environment variable." ) - if gemini_api_key is not None: - auth_header = {"x-goog-api-key": gemini_api_key} + auth_header = {"x-goog-api-key": gemini_api_key} else: # For Vertex AI if use_psc_endpoint_format: @@ -692,7 +722,12 @@ class VertexBase: elif urlparse(api_base).path.rstrip("/") in ("/v1", "/v1beta1") and "/projects/" in urlparse(url).path: url = graft_default_vertex_path(api_base=api_base, default_url=url) else: - url = f"{api_base.rstrip('/')}:{endpoint}" + model_location: Final = publisher_model_location(api_base) if collection_endpoint else None + url = ( + f"{api_base.rstrip('/')}:{endpoint}" + if model_location is None + else join_api_base(model_location.parent_url, endpoint) + ) if stream is True: parsed_stream_url: Final = urlparse(url) stream_query: Final = f"{parsed_stream_url.query}&alt=sse" if parsed_stream_url.query else "alt=sse" diff --git a/litellm/llms/volcengine/responses/transformation.py b/litellm/llms/volcengine/responses/transformation.py index 4076ed06fc2..7f2425d0909 100644 --- a/litellm/llms/volcengine/responses/transformation.py +++ b/litellm/llms/volcengine/responses/transformation.py @@ -313,10 +313,10 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig): api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, - ) -> tuple[str, dict]: + ) -> tuple[str, dict[str, str]]: encoded_response_id: Final = encode_url_path_segment(response_id, field_name="response_id") url: Final = f"{api_base}/{encoded_response_id}" - data: Final[dict] = {} + data: Final[dict[str, str]] = {} return url, data def transform_get_response_api_response( diff --git a/litellm/main.py b/litellm/main.py index 41c4302499e..50ab274a060 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -108,6 +108,10 @@ from litellm.litellm_core_utils.mock_functions import ( from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_content_from_model_response as get_content_from_model_response, # noqa: PLC0414 # public re-export ) +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + messages_have_pdf_file_part, + messages_without_pdf_file_parts, +) from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header from litellm.litellm_core_utils.request_timeout_resolver import ( get_configured_request_timeout, @@ -155,6 +159,7 @@ from litellm.types.litellm_params import ControlOptions, RetryStrategy from litellm.types.llms.openai import AllMessageValues from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( + AgenticLoopParams, BaseUrlRoutingHint, CustomPricingLiteLLMParams, ModelResponseStream, @@ -190,11 +195,13 @@ from litellm.utils import ( get_requester_metadata, get_secret, get_standard_openai_params, + is_explicitly_disabled_factory, mock_completion_streaming_obj, pre_process_non_default_params, read_config_args, should_run_mock_completion, supports_httpx_timeout, + supports_pdf_input, validate_and_fix_openai_messages, validate_and_fix_openai_tools, validate_and_fix_thinking_param, @@ -234,6 +241,7 @@ from .litellm_core_utils.streaming_chunk_builder_utils import ( ChunkProcessor as ChunkProcessor, # noqa: PLC0414 # public re-export ) from .llms.anthropic.chat import AnthropicChatCompletion +from .llms.anthropic.common_utils import is_claude_tool_target from .llms.azure.audio_transcriptions import AzureAudioTranscription from .llms.azure.azure import AzureChatCompletion, check_dynamic_azure_params from .llms.azure.chat.o_series_handler import AzureOpenAIO1ChatCompletion @@ -391,6 +399,125 @@ def _resolve_control_options(kwargs: Mapping[str, object], model: str) -> Contro return assert_never(control) +PROVIDERS_WITHOUT_DOCUMENT_INPUT: Final = frozenset({"ollama", "ollama_chat"}) +PROVIDERS_DROPPING_UNDECLARED_PDF_INPUT: Final = frozenset({"azure_ai"}) + + +def _cost_map_pdf_input_support(model: str, custom_llm_provider: str) -> bool | None: + if custom_llm_provider in PROVIDERS_WITHOUT_DOCUMENT_INPUT or is_explicitly_disabled_factory( + model, custom_llm_provider, "supports_pdf_input" + ): + return False + if supports_pdf_input(model, custom_llm_provider): + return True + return None + + +def _deployment_pdf_input_support(model: str, base_model: str, custom_llm_provider: str) -> bool | None: + supports: Final = ( + _cost_map_pdf_input_support(model, custom_llm_provider), + _cost_map_pdf_input_support(base_model, custom_llm_provider), + ) + if False in supports: + return False + if True in supports: + return True + return None + + +def _declared_pdf_input_support( + model: str, custom_llm_provider: str, model_info: Mapping[str, object] | None, base_model: str | None +) -> bool | None: + match model_info: + case {"supports_pdf_input": bool() as declared}: + return declared + case _ if base_model: + return _deployment_pdf_input_support(model, base_model, custom_llm_provider) + case _: + return _cost_map_pdf_input_support(model, custom_llm_provider) + + +def _pdf_input_unreadable( + model: str, + custom_llm_provider: str, + model_info: Mapping[str, object] | None, + base_model: str | None, + drop_params: bool, +) -> bool: + match _declared_pdf_input_support(model, custom_llm_provider, model_info, base_model): + case False: + return True + case None: + return drop_params and custom_llm_provider in PROVIDERS_DROPPING_UNDECLARED_PDF_INPUT + case True: + return False + + +def _messages_without_unreadable_pdfs( + messages: Sequence[AllMessageValues], + model: str, + custom_llm_provider: str, + model_info: Mapping[str, object] | None, + base_model: str | None, + drop_params: bool, +) -> tuple[AllMessageValues, ...] | None: + if not messages_have_pdf_file_part(messages) or not _pdf_input_unreadable( + model, custom_llm_provider, model_info, base_model, drop_params + ): + return None + if not drop_params: + raise litellm.BadRequestError( + message=( + f"{custom_llm_provider}/{model} does not accept PDF input. " + "Remove the PDF, or set drop_params=True to send the request without it" + ), + model=model, + llm_provider=custom_llm_provider, + ) + readable_messages: Final = messages_without_pdf_file_parts(messages) + if not any(message["role"] == "user" for message in readable_messages): + raise litellm.BadRequestError( + message=( + f"{custom_llm_provider}/{model} does not accept PDF input, " + "and no user message is left once the PDF is dropped" + ), + model=model, + llm_provider=custom_llm_provider, + ) + return readable_messages + + +def _messages_readable_by_model( + messages: Sequence[AllMessageValues], + model: str, + custom_llm_provider: str, + model_info: Mapping[str, object] | None, + base_model: str | None, + drop_params: bool, + ensure_alternating_roles: bool | None, + user_continue_message: ChatCompletionUserMessage | None, + assistant_continue_message: ChatCompletionAssistantMessage | None, +) -> tuple[AllMessageValues, ...] | None: + pdf_free_messages: Final = _messages_without_unreadable_pdfs( + messages=messages, + model=model, + custom_llm_provider=custom_llm_provider, + model_info=model_info, + base_model=base_model, + drop_params=drop_params, + ) + if pdf_free_messages is None: + return None + return tuple( + get_completion_messages( + messages=list(pdf_free_messages), + ensure_alternating_roles=ensure_alternating_roles or False, + user_continue_message=user_continue_message, + assistant_continue_message=assistant_continue_message, + ) + ) + + ####### COMPLETION ENDPOINTS ################ @@ -1135,17 +1262,6 @@ from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the o _ANTHROPIC_ONLY_TOOL_KEYS: Final = frozenset({"input_examples", "eager_input_streaming"}) -def _is_claude_tool_target(custom_llm_provider: str | None, model: str) -> bool: - if custom_llm_provider == "anthropic": - return True - model_lower: Final = model.lower() - if custom_llm_provider == "bedrock": - return "claude" in model_lower or ("arn:" in model_lower and ":bedrock:" in model_lower) - if custom_llm_provider == "azure_ai" or custom_llm_provider == "vertex_ai": - return "claude" in model_lower - return False - - def _history_without_unanswered_tool_search_calls( messages: Sequence[object], custom_llm_provider: str | None, provider_config: BaseConfig ) -> tuple[object, ...] | None: @@ -1282,6 +1398,16 @@ def _get_router_deployment_id(kwargs: dict) -> str | None: return None +def _record_agentic_loop_params( + logging_obj: object, requested_model: str, custom_llm_provider: str, owns_agentic_loop: bool +) -> None: + if not owns_agentic_loop or not isinstance(logging_obj, LiteLLMLoggingObj): + return + logging_obj.record_agentic_loop_params( + AgenticLoopParams(model=requested_model, custom_llm_provider=custom_llm_provider) + ) + + def _register_custom_pricing_for_request( model: str, custom_llm_provider: str, @@ -2672,10 +2798,13 @@ def _complete_custom_openai( # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there api_base = _resolve_openai_api_base(api_base) + if custom_llm_provider == "bigmodel" and not api_key: + raise litellm.AuthenticationError( + message="Set BIGMODEL_API_KEY or pass api_key", llm_provider="bigmodel", model=model + ) organization = ( organization - or litellm.organization - or get_secret("OPENAI_ORGANIZATION") + or (None if custom_llm_provider == "bigmodel" else litellm.organization or get_secret("OPENAI_ORGANIZATION")) or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 ) openai.organization = organization @@ -4230,7 +4359,6 @@ def _complete_sagemaker(ctx: CompletionDispatchContext) -> _CompletionDispatchRe _ADDITIONAL_DROP_PARAMS_ADAPTER: Final = TypeAdapter(list[str]) _OPTIONAL_PARAMS_ADAPTER: Final = TypeAdapter(dict[str, object]) _OLLAMA_EMBEDDING_HEADERS_ADAPTER: Final = TypeAdapter(dict[str, str]) -_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _PROVIDER_ROUTING_HINTS: Final = TypeAdapter(ProviderRoutingHints) _BASE_URL_HINT: Final = TypeAdapter(BaseUrlRoutingHint) @@ -5554,9 +5682,15 @@ def completion( ) headers = kwargs.get("headers", None) or extra_headers - ensure_alternating_roles: Final[bool | None] = kwargs.get("ensure_alternating_roles", None) - user_continue_message: Final[ChatCompletionUserMessage | None] = kwargs.get("user_continue_message", None) - assistant_continue_message: ChatCompletionAssistantMessage | None = kwargs.get("assistant_continue_message", None) + ensure_alternating_roles: Final = cast( # cast-ok: read from untyped kwargs + bool | None, kwargs.get("ensure_alternating_roles", None) + ) + user_continue_message: Final = cast( # cast-ok: read from untyped kwargs + ChatCompletionUserMessage | None, kwargs.get("user_continue_message", None) + ) + assistant_continue_message: Final = cast( # cast-ok: read from untyped kwargs + ChatCompletionAssistantMessage | None, kwargs.get("assistant_continue_message", None) + ) headers = {} if headers is None else dict(headers) if extra_headers is not None: headers.update(extra_headers) @@ -5698,6 +5832,7 @@ def completion( **{k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS | {"provider_specific_header"} if k in kwargs}, "extra_headers": extra_headers, } + requested_model: Final = model model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( model=model, custom_llm_provider=custom_llm_provider, @@ -5709,16 +5844,23 @@ def completion( else None ), ) + _record_agentic_loop_params( + logging_obj=logging, + requested_model=requested_model, + custom_llm_provider=custom_llm_provider, + owns_agentic_loop=not skip_responses_api_bridge, + ) + deployment_model_info: Final = str_keyed_mapping( + cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg + ) ## RESPONSES API BRIDGE LOGIC ## - check early and normalize model name responses_api_model_info, model = responses_api_bridge_check( model=model, custom_llm_provider=custom_llm_provider, web_search_options=web_search_options, api_base=api_base, - deployment_model_info=str_keyed_mapping( - cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg - ), + deployment_model_info=deployment_model_info, ) routing_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict drop_unsupported_reasoning: Final = ( @@ -5749,7 +5891,7 @@ def completion( } ) - if not _is_claude_tool_target(custom_llm_provider=custom_llm_provider, model=model): + if not is_claude_tool_target(custom_llm_provider=custom_llm_provider, model=model): tools = _drop_anthropic_only_tool_keys(tools=tools) if provider_specific_header is not None: @@ -5815,7 +5957,15 @@ def completion( ) if provider_config is not None: - translated_messages: Final = provider_config.translate_developer_role_to_system_role(messages=messages) + translated_messages: Final = list( + provider_config.translate_developer_role_to_system_role( + messages=messages, + model=model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + model_info=deployment_model_info, + ) + ) history_for_route: Final = _history_without_unanswered_tool_search_calls( translated_messages, custom_llm_provider, provider_config ) @@ -5908,15 +6058,17 @@ def completion( def optional_params_for_model_info(deployment_info: Mapping[str, object] | None) -> dict[str, object]: return get_optional_params(**mapping_args, model_info=deployment_info, **mapping_special_params) - optional_params = optional_params_for_model_info( + validated_model_info: Final = ( _MODEL_INFO_ADAPTER.validate_python(model_info) if isinstance(model_info, dict) else None ) + deployment_base_model: Final = base_model if isinstance(base_model, str) else None + optional_params = optional_params_for_model_info(validated_model_info) fakes_stream: Final = partial( _completion_fakes_stream, model, custom_llm_provider, provider_config, - base_model=base_model if isinstance(base_model, str) else None, + base_model=deployment_base_model, bridges_to_responses_api=bridges_to_responses_api is True, ) resolve_timeout: Final = partial( @@ -5954,16 +6106,29 @@ def completion( functions_unsupported_model: Final[list[object]] = TypeAdapter(list[object]).validate_python( optional_params.pop("functions_unsupported_model") ) - request_kwargs: Final = _REQUEST_KWARGS_ADAPTER.validate_python(kwargs) messages = function_call_prompt( messages=messages, functions=functions_unsupported_model, model=model, llm_provider=custom_llm_provider, - drop_params=litellm.drop_params is True - or normalize_drop_params(request_kwargs.get("drop_params")) is True, + drop_params=drop_unsupported_reasoning, ) + readable_messages: Final = _messages_readable_by_model( + messages=cast(list[AllMessageValues], messages), # cast-ok: completion types messages as a bare List + model=model, + custom_llm_provider=custom_llm_provider, + model_info=validated_model_info, + base_model=deployment_base_model, + drop_params=drop_unsupported_reasoning, + ensure_alternating_roles=ensure_alternating_roles, + user_continue_message=user_continue_message, + assistant_continue_message=assistant_continue_message, + ) + messages = ( # rebind-ok: completion() rewrites messages step by step + messages if readable_messages is None else list(readable_messages) + ) + # For logging - save the values of the litellm-specific params passed in requested_litellm_params: Final = get_litellm_params( acompletion=acompletion, @@ -6299,6 +6464,7 @@ def completion( or custom_llm_provider == "nebius" or custom_llm_provider == "wandb" or custom_llm_provider == "clarifai" + or custom_llm_provider == "bigmodel" or custom_llm_provider in litellm.openai_compatible_providers or JSONProviderRegistry.exists(custom_llm_provider) # JSON-configured providers or "ft:gpt-3.5-turbo" in model # finetune gpt-3.5-turbo diff --git a/litellm/messages/dispatch.py b/litellm/messages/dispatch.py index 0e2fc2e3dcb..8e6becc60b4 100644 --- a/litellm/messages/dispatch.py +++ b/litellm/messages/dispatch.py @@ -111,6 +111,7 @@ _FEATURES: Final[detectors.Detectors] = MappingProxyType( Feature.CACHE_CONTROL_INJECTION: detectors.cache_control_injection, Feature.STREAM_COST_INJECTION: route_detectors.stream_cost_injection, Feature.THINKING_SIGNATURE_RETRY: route_detectors.thinking_signature_retry, + Feature.TOOL_CUSTOM_FIELD: route_detectors.tool_custom_field, } ) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3838f7510f0..c6bbb63b6fc 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -4049,6 +4049,7 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_pdf_input": false, "supports_response_schema": true, "supports_tool_choice": true }, @@ -12437,7 +12438,8 @@ "supported_output_modalities": [ "text" ], - "supports_native_streaming": true + "supports_native_streaming": true, + "supports_pdf_input": false }, "azure_ai/FLUX-1.1-pro": { "deprecation_date": "2027-01-05", @@ -12502,6 +12504,7 @@ "output_cost_per_token": 1.85e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12516,6 +12519,7 @@ "output_cost_per_token": 3.828e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12531,6 +12535,7 @@ "output_cost_per_token": 3.52e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12546,6 +12551,7 @@ "output_cost_per_token": 4.84e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12560,6 +12566,7 @@ "output_cost_per_token": 4.84e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12574,6 +12581,7 @@ "output_cost_per_token": 7.26e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12594,6 +12602,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12616,6 +12625,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -12638,6 +12648,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -12660,6 +12671,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -12687,6 +12699,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -12703,6 +12716,7 @@ "output_cost_per_token": 1.32e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12724,6 +12738,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -12744,6 +12759,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -12765,6 +12781,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12833,6 +12850,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12847,6 +12865,7 @@ "output_cost_per_token": 7.1e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/Llama-4-Maverick-17B-128E-Instruct-FP8": { @@ -12859,6 +12878,7 @@ "output_cost_per_token": 1e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": true }, @@ -12872,6 +12892,7 @@ "output_cost_per_token": 7.8e-07, "source": "https://azure.microsoft.com/en-us/blog/introducing-the-llama-4-herd-in-azure-ai-foundry-and-azure-databricks/", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": true }, @@ -12884,6 +12905,7 @@ "mode": "chat", "output_cost_per_token": 3.7e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/Meta-Llama-3.1-70B-Instruct": { @@ -12895,6 +12917,7 @@ "mode": "chat", "output_cost_per_token": 3.54e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/Phi-3-medium-128k-instruct": { @@ -12906,6 +12929,7 @@ "mode": "chat", "output_cost_per_token": 6.8e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -12918,6 +12942,7 @@ "mode": "chat", "output_cost_per_token": 6.8e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -12930,6 +12955,7 @@ "mode": "chat", "output_cost_per_token": 5.2e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -12942,6 +12968,7 @@ "mode": "chat", "output_cost_per_token": 5.2e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -12954,6 +12981,7 @@ "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -12966,6 +12994,7 @@ "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -12978,6 +13007,7 @@ "mode": "chat", "output_cost_per_token": 6.4e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -12990,6 +13020,7 @@ "mode": "chat", "output_cost_per_token": 5.2e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -13002,6 +13033,7 @@ "mode": "chat", "output_cost_per_token": 5.2e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": true }, @@ -13015,6 +13047,7 @@ "output_cost_per_token": 5e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -13027,7 +13060,8 @@ "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true + "supports_function_calling": true, + "supports_pdf_input": false }, "azure_ai/Phi-4-multimodal-instruct": { "deprecation_date": "2027-01-17", @@ -13041,6 +13075,7 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_audio_input": true, "supports_function_calling": true, + "supports_pdf_input": false, "supports_vision": true }, "azure_ai/Phi-4-mini-reasoning": { @@ -13052,7 +13087,8 @@ "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true + "supports_function_calling": true, + "supports_pdf_input": false }, "azure_ai/Phi-4-reasoning": { "deprecation_date": "2027-01-17", @@ -13064,6 +13100,7 @@ "output_cost_per_token": 5e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true, "supports_reasoning": true }, @@ -13106,6 +13143,7 @@ "output_cost_per_token": 1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/doc-intelligence/prebuilt-read": { @@ -13144,6 +13182,7 @@ "mode": "chat", "output_cost_per_token": 5.4e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_reasoning": true, "supports_tool_choice": true }, @@ -13202,6 +13241,7 @@ "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -13217,6 +13257,7 @@ "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -13230,6 +13271,7 @@ "mode": "chat", "output_cost_per_token": 4.56e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/deepseek-v4-pro": { @@ -13242,6 +13284,7 @@ "output_cost_per_token": 3.48e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_reasoning": true, "supports_tool_choice": true, "cache_read_input_token_cost": 1.45e-07, @@ -13257,6 +13300,7 @@ "output_cost_per_token": 5.1e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_reasoning": true, "supports_tool_choice": true, "cache_read_input_token_cost": 2.8e-08, @@ -13273,6 +13317,7 @@ "output_cost_per_token": 1.32e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -13306,6 +13351,7 @@ "output_cost_per_token": 1.5e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", "supports_function_calling": true, + "supports_pdf_input": false, "supports_response_schema": true, "supports_tool_choice": true, "supports_web_search": true @@ -13323,6 +13369,7 @@ "output_cost_per_token_above_200k_tokens": 5e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -13343,6 +13390,7 @@ "output_cost_per_token_above_200k_tokens": 1.2e-05, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/grok/", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -13361,6 +13409,7 @@ "output_cost_per_token": 2.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -13379,6 +13428,7 @@ "output_cost_per_token": 2.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -13395,6 +13445,7 @@ "mode": "chat", "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_response_schema": true, "supports_tool_choice": true, "supports_web_search": true @@ -13409,6 +13460,7 @@ "mode": "chat", "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -13424,6 +13476,7 @@ "output_cost_per_token": 1.5e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", "supports_function_calling": true, + "supports_pdf_input": false, "supports_response_schema": true, "supports_tool_choice": true, "supports_web_search": true @@ -13436,7 +13489,8 @@ "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.00971, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models" + "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false }, "azure_ai/jamba-instruct": { "deprecation_date": "2025-03-01", @@ -13447,6 +13501,7 @@ "mode": "chat", "output_cost_per_token": 7e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/kimi-k2.5": { @@ -13459,6 +13514,7 @@ "output_cost_per_token": 3e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true, @@ -13482,6 +13538,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_reasoning": true, "supports_tool_choice": true, "supports_vision": true, @@ -13498,6 +13555,7 @@ "output_cost_per_token": 4e-08, "source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.ministral-3b-2410-offer?tab=Overview", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/mistral-large": { @@ -13510,6 +13568,7 @@ "output_cost_per_token": 1.2e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/mistral-large-2407": { @@ -13522,6 +13581,7 @@ "output_cost_per_token": 6e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/mistral-large-latest": { @@ -13533,6 +13593,7 @@ "output_cost_per_token": 6e-06, "source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.mistral-ai-large-2407-offer?tab=Overview", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/mistral-large-3": { @@ -13544,6 +13605,7 @@ "output_cost_per_token": 1.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": true }, @@ -13557,6 +13619,7 @@ "output_cost_per_token": 2e-06, "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/mistral-nemo": { @@ -13568,7 +13631,8 @@ "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_function_calling": true + "supports_function_calling": true, + "supports_pdf_input": false }, "azure_ai/mistral-small": { "deprecation_date": "2025-07-31", @@ -13580,6 +13644,7 @@ "output_cost_per_token": 3e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/mistral-small-2503": { @@ -13591,6 +13656,7 @@ "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": true }, @@ -52006,6 +52072,7 @@ ], "supports_assistant_prefill": false, "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -52026,6 +52093,7 @@ ], "supports_assistant_prefill": false, "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -52777,6 +52845,7 @@ "output_cost_per_token": 2.5e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true, "supports_web_search": true }, @@ -52794,6 +52863,7 @@ "global" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_reasoning": true, "supports_tool_choice": true }, @@ -52811,6 +52881,7 @@ "global" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -52989,6 +53060,7 @@ "output_cost_per_token": 3.6e-07, "output_cost_per_token_batches": 1.8e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supports_pdf_input": false, "supports_reasoning": true }, "vertex_ai/openai/gpt-oss-20b-maas": { @@ -53000,6 +53072,7 @@ "mode": "chat", "output_cost_per_token": 2.5e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supports_pdf_input": false, "supports_reasoning": true, "cache_read_input_token_cost": 7e-09, "input_cost_per_token_batches": 3.5e-08, @@ -53131,6 +53204,7 @@ "us-south1" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas": { @@ -53149,6 +53223,7 @@ "global" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "vertex_ai/qwen/qwen3-next-80b-a3b-instruct-maas": { @@ -53164,6 +53239,7 @@ "global" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "vertex_ai/qwen/qwen3-next-80b-a3b-thinking-maas": { @@ -53179,6 +53255,7 @@ "global" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "vertex_ai/veo-3.1-generate-preview": { @@ -61177,6 +61254,126 @@ ], "supports_audio_input": true }, + "soniox/stt-rt-v5": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "soniox/stt-rt-preview": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "soniox/stt-rt-preview-v2": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "soniox/stt-rt-v3": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "soniox/stt-rt-v3-preview": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "soniox/stt-rt-v4": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, "tensormesh/Qwen/Qwen3.5-397B-A17B-FP8": { "litellm_provider": "tensormesh", "mode": "chat", @@ -63655,6 +63852,7 @@ "output_cost_per_token_priority": 9e-06, "source": "https://api.fireworks.ai/v1/serverless/models", "supports_function_calling": true, + "supports_mid_conversation_system": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -63801,6 +63999,7 @@ "output_cost_per_token_priority": 9e-06, "source": "https://api.fireworks.ai/v1/serverless/models", "supports_function_calling": true, + "supports_mid_conversation_system": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -67200,6 +67399,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -67214,6 +67414,7 @@ "mode": "chat", "output_cost_per_token": 1.2e-06, "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -67256,6 +67457,7 @@ "output_cost_per_token": 1.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -67270,6 +67472,7 @@ "output_cost_per_token": 3.1e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -67284,6 +67487,7 @@ "output_cost_per_token": 5.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -67298,6 +67502,7 @@ "output_cost_per_token": 6.25e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -67314,6 +67519,7 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -67329,6 +67535,7 @@ "output_cost_per_token": 3.2e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_reasoning": true, "supports_tool_choice": true }, @@ -67347,6 +67554,7 @@ "supported_output_modalities": [ "text" ], + "supports_pdf_input": false, "supports_response_schema": true, "supports_vision": true }, @@ -67356,7 +67564,8 @@ "output_cost_per_token": 5.4e-06, "litellm_provider": "azure_ai", "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_pdf_input": false }, "azure_ai/deepseek-v3-0324": { "deprecation_date": "2026-07-13", @@ -67364,7 +67573,8 @@ "output_cost_per_token": 4.56e-06, "litellm_provider": "azure_ai", "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_pdf_input": false }, "azure_ai/deepseek-v3.1": { "deprecation_date": "2026-07-13", @@ -67372,7 +67582,8 @@ "output_cost_per_token": 4.94e-06, "litellm_provider": "azure_ai", "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_pdf_input": false }, "azure_ai/grok-3": { "deprecation_date": "2026-05-01", @@ -67380,7 +67591,8 @@ "output_cost_per_token": 1.5e-05, "litellm_provider": "azure_ai", "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_pdf_input": false }, "azure_ai/grok-3-mini": { "deprecation_date": "2026-05-01", @@ -67388,7 +67600,8 @@ "output_cost_per_token": 1.27e-06, "litellm_provider": "azure_ai", "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_pdf_input": false }, "azure_ai/grok-4-fast-non-reasoning": { "deprecation_date": "2026-05-01", @@ -67396,7 +67609,8 @@ "output_cost_per_token": 5e-07, "litellm_provider": "azure_ai", "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_pdf_input": false }, "azure_ai/grok-4-fast-reasoning": { "deprecation_date": "2026-05-01", @@ -67404,7 +67618,8 @@ "output_cost_per_token": 5e-07, "litellm_provider": "azure_ai", "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_pdf_input": false }, "bedrock/us-gov-west-1/nvidia.nemotron-nano-3-30b": { "input_cost_per_token": 7.2e-08, @@ -73074,6 +73289,7 @@ "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_regions": ["global"], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -73689,6 +73905,7 @@ "output_cost_per_token": 3.5e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/microsoft/", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -74743,6 +74960,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": false, "supports_reasoning": true, "supports_tool_choice": true, @@ -74991,6 +75209,7 @@ "output_cost_per_token_above_200k_tokens": 1.2e-05, "source": "https://docs.x.ai/developers/models/grok-4.7", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -78339,6 +78558,1254 @@ "supports_xhigh_reasoning_effort": true, "use_openai_responses_path": true }, + "kyma/claude-fable-5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-05, + "output_cost_per_token": 6.75e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true + }, + "kyma/claude-fable-5.1": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-05, + "output_cost_per_token": 6.75e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true + }, + "kyma/claude-haiku-4-5": { + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "input_cost_per_token": 1.35e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 6.75e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false, + "prompt_cache_min_tokens": 4096 + }, + "kyma/claude-haiku-5.5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-07, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 6.75e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": false, + "prompt_cache_min_tokens": 512 + }, + "kyma/claude-opus-4-7": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 6.75e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 3.375e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 2048 + }, + "kyma/claude-opus-5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 6.75e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 3.375e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 512 + }, + "kyma/claude-opus-5.5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512 + }, + "kyma/claude-sonnet-4-6": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 4.05e-06, + "cache_read_input_token_cost": 4.05e-07, + "output_cost_per_token": 2.025e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 1024 + }, + "kyma/claude-sonnet-5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.35e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 1024 + }, + "kyma/claude-sonnet-5.5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 1.35e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512 + }, + "kyma/deepseek-r1": { + "max_input_tokens": 64000, + "max_output_tokens": 32768, + "input_cost_per_token": 7.425e-07, + "output_cost_per_token": 2.957e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v3": { + "max_input_tokens": 160000, + "max_output_tokens": 8192, + "input_cost_per_token": 5.18e-07, + "output_cost_per_token": 7.571e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v4-flash": { + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "input_cost_per_token": 1.23e-07, + "cache_read_input_token_cost": 2.43e-08, + "output_cost_per_token": 2.458e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v4-flash-vision-exp": { + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "input_cost_per_token": 3.892e-07, + "output_cost_per_token": 1.168e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v4.1-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 2.7e-07, + "output_cost_per_token": 8.1e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 6.75e-07, + "cache_read_input_token_cost": 6.75e-08, + "output_cost_per_token": 4.05e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3.1-pro": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.62e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemini-3.5-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 2.025e-06, + "cache_read_input_token_cost": 2.025e-07, + "output_cost_per_token": 1.215e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3.5-flash-lite": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 4.05e-07, + "cache_read_input_token_cost": 4.05e-08, + "output_cost_per_token": 3.375e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3.6-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.013e-06, + "cache_read_input_token_cost": 1.0125e-07, + "output_cost_per_token": 5.063e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": false, + "supports_pdf_input": false + }, + "kyma/gemini-3.7-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 2.026e-06, + "cache_read_input_token_cost": 1.0125e-07, + "output_cost_per_token": 1.0126e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemini-3.8-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 2.026e-06, + "output_cost_per_token": 1.0126e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemini-4-argon": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemma-4-31b": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.642e-07, + "cache_read_input_token_cost": 1.642e-07, + "output_cost_per_token": 4.689e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-4.5-air": { + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "input_cost_per_token": 1.797e-07, + "cache_read_input_token_cost": 3.375e-08, + "output_cost_per_token": 1.175e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-4.7-flash": { + "max_input_tokens": 203000, + "max_output_tokens": 65536, + "input_cost_per_token": 8.1e-08, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 5.4e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.1": { + "max_input_tokens": 203000, + "max_output_tokens": 65536, + "input_cost_per_token": 1.89e-06, + "cache_read_input_token_cost": 2.7675e-07, + "output_cost_per_token": 5.94e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.2": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 9.029e-07, + "cache_read_input_token_cost": 2.9835e-07, + "output_cost_per_token": 3.083e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.3": { + "max_input_tokens": 1310720, + "max_output_tokens": 131072, + "input_cost_per_token": 1.178e-06, + "output_cost_per_token": 3.702e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.3-flash": { + "max_input_tokens": 1310720, + "max_output_tokens": 65536, + "input_cost_per_token": 1.72e-07, + "output_cost_per_token": 5.733e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gpt-5.6-luna": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.944e-07, + "cache_read_input_token_cost": 2.7e-08, + "output_cost_per_token": 1.767e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-luna-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.703e-07, + "cache_read_input_token_cost": 2.7e-08, + "output_cost_per_token": 1.621e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-sol": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-sol-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-terra": { + "max_input_tokens": 1050000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.62e-05, + "input_cost_per_token_above_272k_tokens": 5.4e-06, + "output_cost_per_token_above_272k_tokens": 2.43e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gpt-5.6-terra-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.62e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-6-astra": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-05, + "cache_read_input_token_cost": 1.35e-06, + "output_cost_per_token": 6.75e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gpt-6-luna": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.639e-07, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 8.195e-07, + "input_cost_per_token_above_272k_tokens": 2.7e-07, + "output_cost_per_token_above_272k_tokens": 1.0125e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-6.1-sol": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 1.35e-05, + "input_cost_per_token_above_272k_tokens": 5.4e-06, + "output_cost_per_token_above_272k_tokens": 2.025e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-oss-120b": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 4.73e-08, + "output_cost_per_token": 2.303e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/grok-4.20": { + "max_input_tokens": 2000000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.763e-06, + "output_cost_per_token": 3.528e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.20-multi-agent": { + "max_input_tokens": 2000000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.767e-06, + "output_cost_per_token": 3.534e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": false, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.3": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 1.92e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 3.838e-06, + "input_cost_per_token_above_200k_tokens": 3.375e-06, + "output_cost_per_token_above_200k_tokens": 6.75e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/grok-4.5": { + "max_input_tokens": 500000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.73e-06, + "output_cost_per_token": 8.189e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.6": { + "max_input_tokens": 500000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.758e-06, + "output_cost_per_token": 8.274e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.7": { + "max_input_tokens": 500000, + "max_output_tokens": 450000, + "input_cost_per_token": 2.16e-06, + "cache_read_input_token_cost": 5.4e-07, + "output_cost_per_token": 6.48e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-build": { + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "input_cost_per_token": 1.389e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 2.777e-06, + "input_cost_per_token_above_200k_tokens": 2.7e-06, + "output_cost_per_token_above_200k_tokens": 5.4e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/hy3": { + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "input_cost_per_token": 1.594e-07, + "output_cost_per_token": 6.576e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/hy4-preview": { + "max_input_tokens": 1048576, + "max_output_tokens": 64000, + "input_cost_per_token": 1.1259e-06, + "output_cost_per_token": 3.3764e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k2.6": { + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "input_cost_per_token": 5.909e-07, + "output_cost_per_token": 2.758e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k2.7-code": { + "max_input_tokens": 262144, + "max_output_tokens": 16384, + "input_cost_per_token": 1.009e-06, + "output_cost_per_token": 4.774e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k3": { + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "input_cost_per_token": 3.052e-06, + "cache_read_input_token_cost": 4.05e-07, + "output_cost_per_token": 1.5258e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k3.1-pro": { + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.35e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/llama-3.3-70b": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.35e-07, + "output_cost_per_token": 4.32e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/llama-4-maverick": { + "max_input_tokens": 1048576, + "max_output_tokens": 16384, + "input_cost_per_token": 3.344e-07, + "output_cost_per_token": 1.337e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/mimo-v2.5": { + "max_input_tokens": 1050000, + "max_output_tokens": 16384, + "input_cost_per_token": 1.859e-07, + "output_cost_per_token": 3.718e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/mimo-v2.6-flash": { + "max_input_tokens": 1050000, + "max_output_tokens": 131072, + "input_cost_per_token": 1.89e-07, + "output_cost_per_token": 3.78e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/mimo-v2.6-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 131072, + "input_cost_per_token": 5.805e-07, + "output_cost_per_token": 1.1745e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m2.5": { + "max_input_tokens": 196608, + "max_output_tokens": 32768, + "input_cost_per_token": 4.188e-07, + "output_cost_per_token": 1.675e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m2.7": { + "max_input_tokens": 204800, + "max_output_tokens": 32768, + "input_cost_per_token": 4.05e-07, + "output_cost_per_token": 1.62e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m3": { + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "input_cost_per_token": 3.956e-07, + "cache_read_input_token_cost": 7.56e-08, + "output_cost_per_token": 1.582e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m3.1-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 524288, + "input_cost_per_token": 2.025e-07, + "cache_read_input_token_cost": 2.025e-08, + "output_cost_per_token": 8.1e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/muse-glimmer-30b": { + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "input_cost_per_token": 4.554e-07, + "output_cost_per_token": 1.821e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/muse-spark-1.1": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.542e-06, + "output_cost_per_token": 5.243e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/muse-spark-1.2": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.773e-06, + "output_cost_per_token": 6.025e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/muse-spark-1.3": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.688e-06, + "output_cost_per_token": 5.738e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/muse-spark-open": { + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "input_cost_per_token": 1.688e-06, + "output_cost_per_token": 5.738e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/muse-wonder": { + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "input_cost_per_token": 5.4e-06, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/nemotron-3-ultra-550b": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 6.75e-07, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 3.375e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/nemotron-3.5-super-vl": { + "max_input_tokens": 1048576, + "max_output_tokens": 235929, + "input_cost_per_token": 8.1e-08, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 4.725e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3-32b": { + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "input_cost_per_token": 1.08e-07, + "output_cost_per_token": 3.78e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3-coder": { + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "input_cost_per_token": 3.546e-07, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 1.612e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.6-plus": { + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "input_cost_per_token": 4.388e-07, + "output_cost_per_token": 2.633e-06, + "input_cost_per_token_above_256k_tokens": 1.755e-06, + "output_cost_per_token_above_256k_tokens": 5.265e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.7-max": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 2.071e-06, + "cache_read_input_token_cost": 1.6875e-07, + "output_cost_per_token": 6.207e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.7-plus": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 4.325e-07, + "cache_read_input_token_cost": 8.64e-08, + "output_cost_per_token": 1.731e-06, + "input_cost_per_token_above_256k_tokens": 1.296e-06, + "output_cost_per_token_above_256k_tokens": 5.184e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.8-max": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 2.228e-06, + "cache_read_input_token_cost": 2.781e-07, + "output_cost_per_token": 6.684e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen3.7-flash": { + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "input_cost_per_token": 4.35e-08, + "cache_read_input_token_cost": 8.1e-09, + "output_cost_per_token": 1.886e-07, + "input_cost_per_token_above_32k_tokens": 1.35e-07, + "output_cost_per_token_above_32k_tokens": 5.4e-07, + "input_cost_per_token_above_256k_tokens": 2.7e-07, + "output_cost_per_token_above_256k_tokens": 1.08e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen3.8-27b": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 3.826e-07, + "output_cost_per_token": 2.732e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen3.8-flash": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 2.045e-07, + "output_cost_per_token": 6.41e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/sonar": { + "max_input_tokens": 127072, + "max_output_tokens": 4096, + "input_cost_per_token": 1.35e-06, + "output_cost_per_token": 1.35e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": false, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_pdf_input": false + }, + "kyma/sonar-pro": { + "max_input_tokens": 200000, + "max_output_tokens": 8000, + "input_cost_per_token": 4.05e-06, + "output_cost_per_token": 2.025e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": false, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_pdf_input": false + }, + "kyma/step-3.7-flash": { + "max_input_tokens": 256000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.229e-07, + "cache_read_input_token_cost": 5.4e-08, + "output_cost_per_token": 1.282e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/step-5-preview": { + "max_input_tokens": 1048576, + "max_output_tokens": 64000, + "input_cost_per_token": 1.35e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 3.645e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, "databricks/databricks-claude-haiku-5-5": { "cache_creation_input_token_cost": 1.2502e-07, "cache_creation_input_token_cost_above_100k_tokens": 6.2503e-07, @@ -78995,5 +80462,81 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true + }, + "bigmodel/glm-5.3": { + "input_cost_per_token": 1.1428571428571428e-06, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 2.857142857142857e-07, + "litellm_provider": "bigmodel", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "source": "https://docs.bigmodel.cn/cn/guide/start/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "thinking_always_on": true, + "metadata": { + "comment": "Official API list price checked 2026-10-27: input 8, output 28, cached input 2 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded." + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_native_streaming": true + }, + "bigmodel/glm-5.3-flash": { + "input_cost_per_token": 1.142857142857143e-07, + "output_cost_per_token": 4e-07, + "cache_read_input_token_cost": 3.2857142857142855e-08, + "litellm_provider": "bigmodel", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "source": "https://docs.bigmodel.cn/cn/guide/start/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "thinking_always_on": true, + "supports_vision": true, + "supports_video_input": true, + "metadata": { + "comment": "Official API list price checked 2026-10-27: input 0.8, output 2.8, cached input 0.23 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded." + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_native_streaming": true + }, + "bigmodel/glm-5.3-flashx": { + "input_cost_per_token": 2.857142857142857e-07, + "output_cost_per_token": 1e-06, + "cache_read_input_token_cost": 8.142857142857142e-08, + "litellm_provider": "bigmodel", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "source": "https://docs.bigmodel.cn/cn/guide/start/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "thinking_always_on": true, + "supports_vision": true, + "supports_video_input": true, + "metadata": { + "comment": "Official API list price checked 2026-10-27: input 2, output 7, cached input 0.57 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded." + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_native_streaming": true } } diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index e40cd134759..1d487f33835 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -3724,7 +3724,7 @@ "ocr": false, "search": false, "moderations": false, - "realtime": false, + "realtime": true, "video_generations": false, "batches": false, "messages": false, @@ -4910,6 +4910,33 @@ "vector_stores_search": true } }, + "opensearch": { + "display_name": "Amazon OpenSearch (`opensearch`)", + "url": "https://docs.litellm.ai/docs/providers/opensearch_vector_stores", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false, + "vector_stores_search": true + } + }, "helicone": { "display_name": "Helicone (`helicone`)", "url": "https://docs.litellm.ai/docs/providers/helicone", @@ -5326,6 +5353,58 @@ "a2a": false, "generateContent": false } + }, + "kyma": { + "display_name": "Kyma API (`kyma`)", + "url": "https://docs.litellm.ai/docs/providers/kyma", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, + "bigmodel": { + "display_name": "BigModel (Zhipu AI) (`bigmodel`)", + "url": "https://docs.bigmodel.cn/cn/guide/develop/responses/introduction", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } } }, "endpoints": { diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 39adb99a0d0..dd964a74735 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -7816,7 +7816,7 @@ "paths": { "/claude-code/marketplace.json": { "get": { - "description": "Serve marketplace.json for Claude Code plugin discovery.\n\nThis endpoint is accessed by Claude Code CLI when users run:\n- claude plugin marketplace add \n- claude plugin install @\n\nWithout `key` the catalog holds the enabled (public) plugins. With `?key=sk-...`\nthe key is authenticated and the catalog also holds the disabled plugins granted\nto it through `object_permission.skills` or an access group on the key or its team.\n\nReturns:\n Marketplace catalog with list of available plugins and their git sources.\n\nExample:\n ```bash\n claude plugin marketplace add http://localhost:4000/claude-code/marketplace.json\n claude plugin marketplace add \"http://localhost:4000/claude-code/marketplace.json?key=sk-...\"\n claude plugin install my-plugin@litellm\n ```", + "description": "Serve marketplace.json for Claude Code plugin discovery.\n\nThis endpoint is accessed by Claude Code CLI when users run:\n- claude plugin marketplace add \n- claude plugin install @\n\nWithout a key the catalog holds the enabled (public) plugins. With `?key=sk-...`\nthe key is authenticated and the catalog also holds the disabled plugins granted\nto it through `object_permission.skills` or an access group on the key or its team.\n\nWith `general_settings.claude_code_marketplace_auth_required: true` the key must come\nin the `Authorization` or `x-litellm-api-key` header (Claude Code sends them from the\nmarketplace source's `headers`), `?key=` is ignored, and a caller with no valid key\ngets a 401 with `WWW-Authenticate: Bearer`.\n\nReturns:\n Marketplace catalog with list of available plugins and their git sources.\n\nExample:\n ```bash\n claude plugin marketplace add http://localhost:4000/claude-code/marketplace.json\n claude plugin marketplace add \"http://localhost:4000/claude-code/marketplace.json?key=sk-...\"\n claude plugin install my-plugin@litellm\n ```", "operationId": "get_marketplace_claude_code_marketplace_json_get", "parameters": [ { @@ -33639,7 +33639,7 @@ ] }, "get": { - "description": "Get a response by ID.\n\nSupports both:\n- Polling IDs (litellm_poll_*): Returns cumulative cached content from background responses\n- Provider response IDs: Passes through to provider API\n\nFollows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/get\n\n```bash\n# Get polling response\ncurl -X GET http://localhost:4000/v1/responses/litellm_poll_abc123 -H \"Authorization: Bearer $LITELLM_MASTER_KEY\"\n\n# Get provider response\ncurl -X GET http://localhost:4000/v1/responses/resp_abc123 -H \"Authorization: Bearer $LITELLM_MASTER_KEY\"\n```", + "description": "Get a response by ID.\n\nSupports both:\n- Polling IDs (litellm_poll_*): Returns cumulative cached content from background responses\n- Provider response IDs: Passes through to provider API\n\nFollows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/get\n\n```bash\n# Get polling response\ncurl -X GET http://localhost:4000/v1/responses/litellm_poll_abc123 -H \"Authorization: Bearer $LITELLM_MASTER_KEY\"\n\n# Get provider response\ncurl -X GET http://localhost:4000/v1/responses/resp_abc123 -H \"Authorization: Bearer $LITELLM_MASTER_KEY\"\n\n# Resume a background response's event stream after sequence number 7\ncurl -N \"http://localhost:4000/v1/responses/resp_abc123?stream=true&starting_after=7\" -H \"Authorization: Bearer $LITELLM_MASTER_KEY\"\n```", "operationId": "get_response_openai_v1_responses__response_id__get", "parameters": [ { @@ -33650,6 +33650,46 @@ "title": "Response Id", "type": "string" } + }, + { + "description": "Replay the response's events as server-sent events (OpenAI, Azure, and Azure AI Foundry)", + "in": "query", + "name": "stream", + "required": false, + "schema": { + "type": "boolean" + } + }, + { + "description": "With stream=true, replay only the events after this sequence number", + "in": "query", + "name": "starting_after", + "required": false, + "schema": { + "minimum": 0, + "type": "integer" + } + }, + { + "description": "Additional output data to include, forwarded to the provider", + "in": "query", + "name": "include", + "required": false, + "schema": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + { + "description": "With stream=true, whether delta events carry obfuscation padding", + "in": "query", + "name": "include_obfuscation", + "required": false, + "schema": { + "type": "boolean" + } } ], "responses": { @@ -33659,6 +33699,12 @@ "schema": { "$ref": "#/components/schemas/ResponsesAPIResponse" } + }, + "text/event-stream": { + "schema": { + "description": "Server sent events when stream=true", + "type": "string" + } } }, "description": "Successful Response" diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index df139a9b84d..4047fa3f0ad 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2833,6 +2833,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): None, description="If True, stores request messages and responses in spend logs. Default is False.", ) + claude_code_marketplace_auth_required: bool = Field( + False, + description="If True, GET /claude-code/marketplace.json requires a LiteLLM key in the Authorization or x-litellm-api-key header (?key= is not accepted) and answers 401 with WWW-Authenticate: Bearer without one, and GET /public/skill_hub returns an empty list. Default is False, which serves the enabled plugins to anyone.", + ) disable_requester_ip_address_logging: bool | None = Field( None, description="If True, new spend log rows store requester_ip_address as null in the column, the metadata and the stored request. The stored request also drops client-IP forwarding headers such as x-forwarded-for, and IP addresses in its other header values and in stored error messages are masked. Guardrails, metrics and callbacks that read the standard logging payload still receive the IP. Default is False.", @@ -3744,6 +3748,7 @@ class SpendLogsMetadata(TypedDict): internal_call_origin: InternalCallOrigin | None billed_retry_attempt: ReadOnly[NotRequired[bool | None]] litellm_roi_estimator: ReadOnly[NotRequired[bool | None]] + is_silent_experiment: ReadOnly[NotRequired[bool | None]] guardrail_information: list[StandardLoggingGuardrailInformation] | None eval_information: Any | None status: StandardLoggingPayloadStatus diff --git a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py index b0088d93112..fc82686947d 100644 --- a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py +++ b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py @@ -7,7 +7,8 @@ Actual plugin files are hosted on GitHub/GitLab/Bitbucket or as a zip archive on any HTTPS host (S3, Artifactory, a static file server). Endpoints: -/claude-code/marketplace.json - GET - List plugins for Claude Code discovery (unauthenticated; `?key=` adds the key's granted skills) +/claude-code/marketplace.json - GET - List plugins for Claude Code discovery (anonymous unless + general_settings.claude_code_marketplace_auth_required; a key adds its granted skills) /claude-code/plugins - POST - Register a new plugin (create-only, proxy admin only) /claude-code/plugins - GET - List plugins visible to the key (enabled, plus granted disabled ones) /claude-code/plugins/{name} - GET - Get plugin details (403 on a disabled plugin the key is not granted) @@ -31,14 +32,17 @@ from pydantic import TypeAdapter from typing_extensions import ReadOnly from litellm._logging import verbose_proxy_logger -from litellm.proxy._types import CommonProxyErrors, ProxyException, UserAPIKeyAuth +from litellm.proxy._types import CommonProxyErrors, ProxyErrorTypes, ProxyException, UserAPIKeyAuth from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_skill_access import ( SkillVisibility, resolve_skill_visibility, ) +from litellm.proxy.auth.auth_exception_handler import DB_UNAVAILABLE_FALLBACK_USER_ID from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.claude_code_marketplace_auth import marketplace_auth_required from litellm.proxy.common_utils.resource_ownership import is_proxy_admin from litellm.repositories.table_repositories import ClaudeCodePluginRepository +from litellm.types.proxy.auth.special_headers import SpecialHeaders from litellm.types.proxy.claude_code_endpoints import ( InstallationPreference, ListPluginsResponse, @@ -109,6 +113,41 @@ async def get_prisma_client() -> object: _get_prisma_client: Final = get_prisma_client +async def _header_key_caller(request: Request) -> UserAPIKeyAuth: + try: + caller: Final = await user_api_key_auth( + request=request, + api_key=request.headers.get(SpecialHeaders.openai_authorization.value) or "", + custom_litellm_key_header=request.headers.get(SpecialHeaders.custom_litellm_api_key.value), + ) + except ProxyException as e: + if e.code != "401": + raise + raise ProxyException( + message=e.message, + type=e.type, + param=e.param, + code=401, + headers={**e.headers, "WWW-Authenticate": "Bearer"}, + ) from e + if caller.user_id == DB_UNAVAILABLE_FALLBACK_USER_ID: + raise ProxyException( + message="The LiteLLM key could not be verified because the database is unavailable", + type=ProxyErrorTypes.no_db_connection, + param=None, + code=503, + ) + return caller + + +async def _marketplace_caller(request: Request, key: str | None) -> UserAPIKeyAuth | None: + if marketplace_auth_required(): + return await _header_key_caller(request) + if key: + return await user_api_key_auth(request=request, api_key=f"Bearer {key}") + return None + + @router.get( "/claude-code/marketplace.json", tags=["Claude Code Marketplace"], @@ -121,10 +160,15 @@ async def get_marketplace(request: Request, key: str | None = None): - claude plugin marketplace add - claude plugin install @ - Without `key` the catalog holds the enabled (public) plugins. With `?key=sk-...` + Without a key the catalog holds the enabled (public) plugins. With `?key=sk-...` the key is authenticated and the catalog also holds the disabled plugins granted to it through `object_permission.skills` or an access group on the key or its team. + With `general_settings.claude_code_marketplace_auth_required: true` the key must come + in the `Authorization` or `x-litellm-api-key` header (Claude Code sends them from the + marketplace source's `headers`), `?key=` is ignored, and a caller with no valid key + gets a 401 with `WWW-Authenticate: Bearer`. + Returns: Marketplace catalog with list of available plugins and their git sources. @@ -137,10 +181,7 @@ async def get_marketplace(request: Request, key: str | None = None): """ try: prisma_client: Final = await get_prisma_client() - - caller: Final[UserAPIKeyAuth | None] = ( - await user_api_key_auth(request=request, api_key=f"Bearer {key}") if key else None - ) + caller: Final = await _marketplace_caller(request, key) visibility: Final[SkillVisibility] = await resolve_skill_visibility(caller) plugins: Final[Sequence[_PluginRecord]] = await ClaudeCodePluginRepository(prisma_client).table.find_many( where=visibility.where() @@ -201,7 +242,7 @@ async def get_marketplace(request: Request, key: str | None = None): verbose_proxy_logger.exception("Error generating marketplace: %s", e) raise HTTPException( status_code=500, - detail={"error": f"Failed to generate marketplace: {e}"}, + detail={"error": "Failed to generate marketplace"}, ) diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 1d88e133cf9..1a2e610d5c4 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -2,7 +2,7 @@ Unified /v1/messages endpoint - (Anthropic Spec) """ -from collections.abc import Mapping +from collections.abc import AsyncGenerator, Mapping from types import MappingProxyType from typing import Final @@ -19,12 +19,10 @@ from litellm.anthropic_interface.exceptions import ( ) from litellm.integrations.custom_guardrail import ModifyResponseException from litellm.litellm_core_utils.tool_fields import first_validation_error +from litellm.llms.anthropic.chat.guardrail_translation.handler import AnthropicMessagesHandler from litellm.llms.anthropic.messages.context_management import ( AnthropicContextManagementError, ) -from litellm.llms.base_llm.guardrail_translation.utils import ( - blocked_response_usage as _blocked_response_usage, -) from litellm.proxy._types import * from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_request_processing import ( @@ -44,6 +42,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # leg from litellm.proxy.common_utils.openai_error_payload import ( LITELLM_CALL_ID_HEADER, error_status_code, + litellm_call_id_headers, openai_error_param, openai_error_type, with_litellm_call_id, @@ -58,6 +57,8 @@ from litellm.router import Router router: Final = APIRouter() +_NO_HEADERS: Final[Mapping[str, str]] = MappingProxyType({}) + def _with_provider_specific_fields(exc: ProxyException, detail: AnthropicErrorDetail) -> AnthropicErrorDetail: if not exc.provider_specific_fields: @@ -229,44 +230,22 @@ async def anthropic_response( request_data=_data, ) - # Create Anthropic-formatted response with violation message - import uuid + block_headers: Final = litellm_call_id_headers(base_llm_response_processor.litellm_call_id) or _NO_HEADERS + block_handler: Final = AnthropicMessagesHandler() - from litellm.types.utils import AnthropicMessagesResponse + if data.get("stream") is True: + block_chunks: Final = block_handler.build_block_sse_chunks(e) - # Report the blocked LLM response's real token usage (carried on the - # exception) instead of discarding it; zero for pre-call blocks. - _usage: Final = _blocked_response_usage(e.original_response) - - _anthropic_response: Final = AnthropicMessagesResponse( - id=f"msg_{uuid.uuid4()}", - type="message", - role="assistant", - content=[{"type": "text", "text": e.message}], - model=e.model, - stop_reason="end_turn", - usage=_usage, - ) - - if data.get("stream", None) is not None and data["stream"] is True: - # For streaming, use the standard SSE data generator - async def _passthrough_stream_generator(): - yield _anthropic_response - - selected_data_generator: Final = ProxyBaseLLMRequestProcessing.async_sse_data_generator( - response=_passthrough_stream_generator(), - user_api_key_dict=user_api_key_dict, - request_data=_data, - proxy_logging_obj=proxy_logging_obj, - ) + async def _blocked_stream() -> AsyncGenerator[str, None]: + for chunk in block_chunks: + yield chunk.decode() return await create_response( - generator=selected_data_generator, - media_type="text/event-stream", - headers={}, + generator=_blocked_stream(), media_type="text/event-stream", headers=block_headers ) - return _anthropic_response + fastapi_response.headers.update(block_headers) + return block_handler.build_block_response(e) except AnthropicContextManagementError as e: if e.status_code >= 500: # Server-side polyfill failures hit the failure hook for spend/alert diff --git a/litellm/proxy/anthropic_endpoints/gateway_endpoints.py b/litellm/proxy/anthropic_endpoints/gateway_endpoints.py index 1911cb574ef..6c58f2edc02 100644 --- a/litellm/proxy/anthropic_endpoints/gateway_endpoints.py +++ b/litellm/proxy/anthropic_endpoints/gateway_endpoints.py @@ -45,7 +45,11 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # leg safe_set_request_parsed_body, ) from litellm.proxy.management_endpoints.sso_helper_utils import CLI_SSO_SESSIONS_TARGET -from litellm.proxy.management_endpoints.ui_sso import CliSsoTeamDetail +from litellm.proxy.management_endpoints.ui_sso import ( + CLI_SSO_BROWSER_TEAM_CHOICE, + CLI_SSO_CHOSEN_TEAM_ID, + CliSsoTeamDetail, +) from litellm.proxy.mcp_server.gateway_dcr_flow import ( PROXY_API_AUDIENCE, MintedProxyCredential, @@ -235,6 +239,7 @@ async def device_authorization(request: Request) -> JSONResponse: "sso_complete": False, "user_code_verified": False, "session_data": None, + CLI_SSO_BROWSER_TEAM_CHOICE: True, } _set_cli_sso_flow(login_id=login_id, cache=cli_sso_session_cache, flow=flow) @@ -267,7 +272,15 @@ def _validate_login(flow: Mapping[str, object]) -> _GatewayLogin | _OAuthError: status_code=400, error="invalid_grant", description="The login session is malformed; sign in again" ) - team_id: Final = session_data.teams[0] if session_data.teams else None + chosen: Final = flow.get(CLI_SSO_CHOSEN_TEAM_ID) + chosen_team_id: Final = chosen if isinstance(chosen, str) else None + if chosen_team_id is not None and chosen_team_id not in session_data.teams: + return _OAuthError( + status_code=400, + error="invalid_grant", + description="The team picked at sign-in is no longer one of your teams; sign in again", + ) + team_id: Final = chosen_team_id or next(iter(session_data.teams), None) selected_team: Final = selected_cli_sso_team_detail(team_details=session_data.team_details, team_id=team_id) if selected_team is None: return _OAuthError( diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index c4c538083ee..48884d0fedc 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -20,6 +20,8 @@ from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager, get_valid_models +LLM_PROVIDER_PREFIXES: Final = frozenset(llm_provider.value for llm_provider in LlmProviders) + _CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields) @@ -458,23 +460,21 @@ def get_known_models_from_wildcard( # add model prefix to wildcard models wildcard_models = [f"{model_prefix}{model.removeprefix(provider_id_prefix)}" for model in wildcard_models] - known_providers: Final = {provider.value for provider in LlmProviders} - suffix_appended_wildcard_models: Final = [] - for model in wildcard_models: - if not model.startswith(f"{wildcard_provider_prefix}/"): - # `get_provider_models` returns provider-prefixed ids (e.g. "ollama/gemma3:1b"). - # When the wildcard uses a custom prefix (e.g. "ollama_server1/*" to distinguish - # multiple instances), replace that existing provider prefix instead of stacking - # both, which would otherwise yield an uncallable "ollama_server1/ollama/gemma3:1b". - # Only strip the leading segment when it is a known provider, so ids whose first - # segment is an org rather than a provider (e.g. "meta-llama/Llama-3-8B") keep it. - leading, sep, model_suffix = model.partition("/") - if sep and leading in known_providers: - model = f"{wildcard_provider_prefix}/{model_suffix}" - else: - model = f"{wildcard_provider_prefix}/{model}" - suffix_appended_wildcard_models.append(model) - return list(dict.fromkeys(suffix_appended_wildcard_models)) + return list( + dict.fromkeys(_under_wildcard_prefix(model, wildcard_provider_prefix, provider) for model in wildcard_models) + ) + + +def _under_wildcard_prefix(model: str, wildcard_provider_prefix: str, listed_provider: str) -> str: + if model.startswith(f"{wildcard_provider_prefix}/"): + return model + leading, sep, model_suffix = model.partition("/") + leading_is_listed_providers_own_prefix: Final = ( + sep != "" + and leading in LLM_PROVIDER_PREFIXES + and (leading == listed_provider or listed_provider.startswith(f"{leading}_")) + ) + return f"{wildcard_provider_prefix}/{model_suffix if leading_is_listed_providers_own_prefix else model}" def expand_wildcard_deployments_for_model_info( diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 6877d6c0118..fc29a6e3a08 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -3206,7 +3206,7 @@ class ProxyBaseLLMRequestProcessing: request_data=self.data, request=request, ) - if route_type == "aresponses": + if route_type in ("aresponses", "aget_responses"): selected_data_generator = ( ProxyBaseLLMRequestProcessing._wrap_responses_stream_for_container_ownership( original_stream_response=response, diff --git a/litellm/proxy/common_utils/claude_code_marketplace_auth.py b/litellm/proxy/common_utils/claude_code_marketplace_auth.py new file mode 100644 index 00000000000..0cfb3061744 --- /dev/null +++ b/litellm/proxy/common_utils/claude_code_marketplace_auth.py @@ -0,0 +1,17 @@ +from typing import Final + +from pydantic import TypeAdapter, ValidationError + +_AUTH_REQUIRED_SETTING: Final[TypeAdapter[bool | None]] = TypeAdapter(bool | None) + + +def marketplace_auth_required() -> bool: + from litellm.proxy.proxy_server import general_settings_view + + try: + return ( + _AUTH_REQUIRED_SETTING.validate_python(general_settings_view().get("claude_code_marketplace_auth_required")) + is True + ) + except ValidationError: + return True diff --git a/litellm/proxy/common_utils/html_forms/native_client_consent.py b/litellm/proxy/common_utils/html_forms/native_client_consent.py index dac92c4e787..aeadc93221c 100644 --- a/litellm/proxy/common_utils/html_forms/native_client_consent.py +++ b/litellm/proxy/common_utils/html_forms/native_client_consent.py @@ -62,7 +62,7 @@ button {{ flex: 1; padding: 10px; border-radius: 6px; font-size: 15px; cursor: p

Approving issues it a personal credential that expires within {CLI_JWT_EXPIRATION_HOURS} hours. lite logout stops it from being renewed. Only approve if you started this sign-in yourself.

-{_team_field(teams)} +{team_select_field(teams)}
@@ -74,7 +74,7 @@ button {{ flex: 1; padding: 10px; border-radius: 6px; font-size: 15px; cursor: p """ -def _team_field(teams: Sequence[tuple[str, str]]) -> str: +def team_select_field(teams: Sequence[tuple[str, str]], *, require_choice: bool = False) -> str: if not teams: return "" if len(teams) == 1: @@ -83,9 +83,12 @@ def _team_field(teams: Sequence[tuple[str, str]]) -> str: f'' f"

Requests are attributed to team {escape(team_label)}.

" ) + placeholder: Final = '' if require_choice else "" options: Final = "".join( f'' for team_id, team_label in teams ) + required: Final = " required" if require_choice else "" return ( - f'' + '' + f'' ) diff --git a/litellm/proxy/db/check_migration.py b/litellm/proxy/db/check_migration.py index 83cc4501466..d126f112f90 100644 --- a/litellm/proxy/db/check_migration.py +++ b/litellm/proxy/db/check_migration.py @@ -4,7 +4,7 @@ import os import subprocess from typing import Final -from litellm._logging import print_startup_message, verbose_logger +from litellm._logging import print_startup_message, print_startup_warning, verbose_logger def extract_sql_commands(diff_output: str) -> list[str]: @@ -63,7 +63,7 @@ def check_prisma_schema_diff_helper(db_url: str) -> tuple[bool, list[str]]: run_prisma, ) except ImportError as e: - print_startup_message( + print_startup_warning( f"Skipping the migration diff: litellm-proxy-extras has no Prisma runner. Error: {e}", ) return False, [] @@ -96,14 +96,14 @@ def check_prisma_schema_diff_helper(db_url: str) -> tuple[bool, list[str]]: else: return False, [] except subprocess.TimeoutExpired: - print_startup_message( + print_startup_warning( f"Timed out after {timeout}s generating the migration diff. " f"Raise {PRISMA_COMMAND_TIMEOUT_ENV_VAR} if this database needs longer.", ) return False, [] except subprocess.CalledProcessError as e: error_message: Final = f"Failed to generate migration diff. Error: {e.stderr}" - print_startup_message(error_message) + print_startup_warning(error_message) return False, [] diff --git a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py index 7f5efcdaec3..170eb450339 100644 --- a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py +++ b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py @@ -421,7 +421,10 @@ class SpendLogCleanup: general_settings=None, redis_cache: RedisCache | None = None, partition_manager: SpendLogsPartitionManager | None = None, + *, + clock: Callable[[], float] = time.monotonic, ): + self._clock = clock self.retention_seconds: int | None = None self.partition_manager = partition_manager or SpendLogsPartitionManager() from litellm.proxy.proxy_server import general_settings as default_settings @@ -586,7 +589,7 @@ class SpendLogCleanup: async def _probe_storage(self, prisma_client: PrismaClient, deadline: float) -> StorageProbe | None: """Read what the spend-log tables weigh from the catalog, or None when the probe fails or has no budget.""" - if time.monotonic() >= deadline: + if self._clock() >= deadline: return None timeout_ms: Final = self._timeout_ms(deadline) try: @@ -624,7 +627,7 @@ class SpendLogCleanup: stale.allocated_bytes, _ANALYZE_SQL, ) - if time.monotonic() >= deadline: + if self._clock() >= deadline: return None timeout_ms: Final = self._timeout_ms(deadline) try: @@ -653,7 +656,7 @@ class SpendLogCleanup: async def _load_size_state(self, prisma_client: PrismaClient, deadline: float) -> SizeCleanupState | None: """The reclaimable count saved by the last check, or None on the first check or when it cannot be read.""" - if time.monotonic() >= deadline: + if self._clock() >= deadline: return None timeout_ms: Final = self._timeout_ms(deadline) try: @@ -686,7 +689,7 @@ class SpendLogCleanup: the tool-index rows that expire with their spend logs. When fewer rows than that remain, the step is everything up to now. """ - if time.monotonic() >= deadline: + if self._clock() >= deadline: return None timeout_ms: Final = self._timeout_ms(deadline) async with db_span("size_step_cutoff", "LiteLLM_SpendLogs"), bounded_tx(prisma_client, timeout_ms) as tx: @@ -718,11 +721,11 @@ class SpendLogCleanup: Interpolating this into SQL is safe by construction: an int cannot carry SQL, and SET does not accept a bind parameter. """ - remaining_ms: Final = int((deadline - time.monotonic()) * 1000) + remaining_ms: Final = int((deadline - self._clock()) * 1000) return max(1, min(int(self.batch_timeout_seconds * 1000), remaining_ms)) @staticmethod - def _group_deadline(overall_deadline: float, groups_remaining: int) -> float: + def _group_deadline(overall_deadline: float, groups_remaining: int, now: float) -> float: """ Give each pending cleanup group an equal share of the time left. @@ -731,10 +734,9 @@ class SpendLogCleanup: """ if groups_remaining == 1: return overall_deadline - current_time: Final = time.monotonic() - if current_time >= overall_deadline: + if now >= overall_deadline: return overall_deadline - return current_time + (overall_deadline - current_time) / groups_remaining + return now + (overall_deadline - now) / groups_remaining def _remaining_timeout_ms(self, deadline: float) -> RemainingTimeoutMs: """ @@ -748,7 +750,7 @@ class SpendLogCleanup: """ def remaining() -> int | None: - return None if time.monotonic() >= deadline else self._timeout_ms(deadline) + return None if self._clock() >= deadline else self._timeout_ms(deadline) return remaining @@ -841,7 +843,7 @@ class SpendLogCleanup: run_count = 0 consecutive_failures = 0 while True: - if time.monotonic() >= deadline: + if self._clock() >= deadline: verbose_proxy_logger.info( "Run budget exhausted during %s cleanup after %d rows; the next run resumes from here", table_name, @@ -858,13 +860,13 @@ class SpendLogCleanup: prisma_client, cutoff_date, table_name, time_column, total_deleted, "batch_cap_reached", deadline ) # Find rows and delete them in one go without fetching to application - batch_started_at = time.monotonic() + batch_started_at = self._clock() try: batch_result = await self._execute_delete_batch( prisma_client, delete_sql, cutoff_date, table_name, deadline ) except Exception as batch_exc: - if time.monotonic() >= deadline: + if self._clock() >= deadline: # The statement timeout was clamped to the budget that was # left, so this batch was cancelled by the deadline itself. # That is the bound working, not a database fault, and @@ -920,7 +922,7 @@ class SpendLogCleanup: consecutive_failures = 0 deleted_count = batch_result - SpendLogCleanupMetrics.record_batch(table_name, deleted_count, time.monotonic() - batch_started_at) + SpendLogCleanupMetrics.record_batch(table_name, deleted_count, self._clock() - batch_started_at) verbose_proxy_logger.info("Deleted %s %s rows in this batch", deleted_count, table_name) if deleted_count == 0: @@ -957,7 +959,7 @@ class SpendLogCleanup: operator the backlog was not drained; the gauge simply keeps its value from the last run that finished inside its budget. """ - if time.monotonic() >= deadline: + if self._clock() >= deadline: return TableCleanupResult(table_name=table_name, rows_deleted=rows_deleted, stop_reason=stop_reason) remaining: Final = await self._count_remaining(prisma_client, cutoff_date, table_name, time_column, deadline) if remaining is not None: @@ -1070,7 +1072,7 @@ class SpendLogCleanup: # behind any long-running reader for as long as that reader lives, which # is the one way this job could still outlast its budget without bound. remaining_timeout_ms: Final = self._remaining_timeout_ms(deadline) - if time.monotonic() >= deadline: + if self._clock() >= deadline: verbose_proxy_logger.info("Run budget already spent, skipping partition maintenance this run") elif self.general_settings.get( "use_spend_logs_partitioning", False @@ -1082,7 +1084,7 @@ class SpendLogCleanup: verbose_proxy_logger.info("Dropped %d expired spend-log partitions: %s", len(dropped), dropped) logs_result: Final = await self._delete_old_logs( - prisma_client, cutoff_date, self._group_deadline(deadline, groups_remaining=2) + prisma_client, cutoff_date, self._group_deadline(deadline, groups_remaining=2, now=self._clock()) ) verbose_proxy_logger.info("Deleted %s logs", logs_result.rows_deleted) @@ -1210,7 +1212,7 @@ class SpendLogCleanup: self, prisma_client: PrismaClient, deadline: float, still_over_target: Callable[[], Awaitable[bool]] ) -> tuple[str, ...]: remaining_timeout_ms: Final = self._remaining_timeout_ms(deadline) - if time.monotonic() >= deadline or not self._setting("use_spend_logs_partitioning"): + if self._clock() >= deadline or not self._setting("use_spend_logs_partitioning"): return () if not await self.partition_manager.is_partitioned(prisma_client, remaining_timeout_ms): return () @@ -1257,7 +1259,7 @@ class SpendLogCleanup: needed: Final = rows_still_needed(state.logs_deleted + state.index_deleted) if needed <= 0: return state.stopped("target_reached") - if time.monotonic() >= deadline: + if self._clock() >= deadline: return state.stopped("budget_exhausted") if state.batches_left <= 0: return state.stopped("batch_cap_reached") @@ -1318,11 +1320,11 @@ class SpendLogCleanup: except Exception: # noqa: BLE001 # retained observations are retried by the next cleanup job verbose_proxy_logger.warning("Auto-router baseline retention remains pending") sessions_result: Final = await self._delete_old_autorouter_session_rows( - prisma_client, session_cutoff, self._group_deadline(deadline, 3) + prisma_client, session_cutoff, self._group_deadline(deadline, 3, self._clock()) ) verbose_proxy_logger.info("Deleted %s expired auto-router session rollup rows", sessions_result.rows_deleted) user_sessions_result: Final = await self._delete_old_autorouter_user_session_rows( - prisma_client, session_cutoff, self._group_deadline(deadline, 2) + prisma_client, session_cutoff, self._group_deadline(deadline, 2, self._clock()) ) verbose_proxy_logger.info( "Deleted %s expired auto-router user session rollup rows", user_sessions_result.rows_deleted @@ -1443,7 +1445,7 @@ class SpendLogCleanup: async def _run_passes(self, prisma_client: PrismaClient, age_rules: bool, size_rule: bool) -> None: lock_acquired = False - run_started_at: Final = time.monotonic() + run_started_at: Final = self._clock() progress: Final = _RunProgress() progress_token: Final = _run_progress.set(progress) try: @@ -1493,7 +1495,7 @@ class SpendLogCleanup: SpendLogCleanupMetrics.record_run("skipped_locked") return - deadline: Final = time.monotonic() + self.run_budget_seconds + deadline: Final = self._clock() + self.run_budget_seconds size_group: Final = int(active_size_rule is not None) configured_group_count: Final = ( int(delete_spend_logs and self.retention_seconds is not None) @@ -1506,7 +1508,7 @@ class SpendLogCleanup: spend_log_results: Final = ( await self._clean_spend_log_tables( prisma_client, - self._group_deadline(deadline, configured_group_count), + self._group_deadline(deadline, configured_group_count, self._clock()), ) if delete_spend_logs and self.retention_seconds is not None else () @@ -1526,7 +1528,7 @@ class SpendLogCleanup: await self._clean_session_rollup( prisma_client, autorouter_retention_seconds, - self._group_deadline(deadline, remaining_groups_after_spend_logs), + self._group_deadline(deadline, remaining_groups_after_spend_logs, self._clock()), ) if autorouter_retention_seconds is not None else () @@ -1535,7 +1537,7 @@ class SpendLogCleanup: await self._clean_health_checks( prisma_client, health_check_retention_seconds, - self._group_deadline(deadline, remaining_groups_after_sessions), + self._group_deadline(deadline, remaining_groups_after_sessions, self._clock()), ) if health_check_retention_seconds is not None else () @@ -1544,7 +1546,7 @@ class SpendLogCleanup: await self._clean_daily_tag_spend( prisma_client, daily_tag_spend_retention_seconds, - self._group_deadline(deadline, 1 + size_group), + self._group_deadline(deadline, 1 + size_group, self._clock()), ) if daily_tag_spend_retention_seconds is not None else () @@ -1561,12 +1563,12 @@ class SpendLogCleanup: ) outcome: Final = self._run_outcome(results) SpendLogCleanupMetrics.record_run(outcome) - self._log_run_summary(outcome, results, time.monotonic() - run_started_at, size_report) + self._log_run_summary(outcome, results, self._clock() - run_started_at, size_report) except asyncio.CancelledError: verbose_proxy_logger.error( "Spend log cleanup cancelled after %.2fs (rows_deleted=%d, batches=%d); the next run resumes from here", - time.monotonic() - run_started_at, + self._clock() - run_started_at, progress.rows_deleted, progress.batches, ) diff --git a/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py b/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py index 792ef8a3f8c..36f424507d7 100644 --- a/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py +++ b/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py @@ -4,6 +4,7 @@ from typing import Final from fastapi import APIRouter +from litellm.proxy.common_utils.claude_code_marketplace_auth import marketplace_auth_required from litellm.proxy.common_utils.html_forms.default_credentials_hint import should_hide_default_credentials_hint from litellm.proxy.mcp_server.stdio_gate import is_mcp_stdio_enabled from litellm.types.proxy.discovery_endpoints.ui_discovery_endpoints import ( @@ -51,4 +52,5 @@ async def get_ui_config(): workers=proxy_config.worker_registry if is_control_plane else [], mcp_stdio_enabled=is_mcp_stdio_enabled(), use_si_terminology=use_si_terminology, + claude_code_marketplace_auth_required=marketplace_auth_required(), ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py b/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py index 0f66135d038..8e2fea415ef 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py +++ b/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py @@ -647,9 +647,10 @@ class PromptSecurityGuardrail(CustomGuardrail): async def _process_document_item(self, item: dict, user_api_key_alias: str | None) -> dict: """Process and sanitize document/file items.""" doc_data: Final = item.get("document") or item.get("file") or item + url_key: Final = "file_data" if isinstance(doc_data, dict) and not doc_data.get("url") else "url" if isinstance(doc_data, dict): - url = doc_data.get("url", "") + url = doc_data.get(url_key) or "" doc_content = doc_data.get("data", "") else: url = doc_data if isinstance(doc_data, str) else "" @@ -698,7 +699,7 @@ class PromptSecurityGuardrail(CustomGuardrail): if url.startswith("data:") and header: sanitized_url: Final = f"{header},{sanitized_encoded}" if isinstance(doc_data, dict): - doc_data["url"] = sanitized_url + doc_data[url_key] = sanitized_url elif isinstance(doc_data, dict): doc_data["data"] = sanitized_encoded diff --git a/litellm/proxy/health_check_utils/shared_health_check_manager.py b/litellm/proxy/health_check_utils/shared_health_check_manager.py index 62c973196b9..d689ab2a537 100644 --- a/litellm/proxy/health_check_utils/shared_health_check_manager.py +++ b/litellm/proxy/health_check_utils/shared_health_check_manager.py @@ -2,7 +2,10 @@ import asyncio import json import time from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Final + +from pydantic import ConfigDict, TypeAdapter, with_config +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger @@ -19,6 +22,17 @@ if TYPE_CHECKING: from litellm.router import Router +@with_config(ConfigDict(extra="allow", strict=True)) +class _CachedHealthCheckResults(TypedDict): + healthy_endpoints: NotRequired[ReadOnly[list[Mapping[str, object]]]] + unhealthy_endpoints: NotRequired[ReadOnly[list[Mapping[str, object]]]] + timestamp: NotRequired[ReadOnly[int | float]] + checked_by: NotRequired[ReadOnly[object]] + + +_CACHED_HEALTH_CHECK_RESULTS: Final = TypeAdapter(_CachedHealthCheckResults) + + class SharedHealthCheckManager: """ Manager for coordinating health checks across multiple pods using Redis. @@ -109,7 +123,7 @@ class SharedHealthCheckManager: verbose_proxy_logger.error("Error releasing health check lock: %s", str(e)) @with_service_target(HEALTH_CHECKS_TARGET) - async def get_cached_health_check_results(self) -> dict[str, Any] | None: + async def get_cached_health_check_results(self) -> _CachedHealthCheckResults | None: """ Get cached health check results from Redis. @@ -127,10 +141,9 @@ class SharedHealthCheckManager: return None # Parse the cached data - if isinstance(cached_data, str): - cached_results = json.loads(cached_data) - else: - cached_results = cached_data + cached_results: Final = _CACHED_HEALTH_CHECK_RESULTS.validate_python( + json.loads(cached_data) if isinstance(cached_data, str) else cached_data + ) # Check if the cache is still valid cache_timestamp: Final = cached_results.get("timestamp", 0) diff --git a/litellm/proxy/hooks/responses_id_security.py b/litellm/proxy/hooks/responses_id_security.py index 27b5924c540..771d68a0ea0 100644 --- a/litellm/proxy/hooks/responses_id_security.py +++ b/litellm/proxy/hooks/responses_id_security.py @@ -31,7 +31,7 @@ if TYPE_CHECKING: _RESPONSES_API_PROVIDER_PREFIX: Final = "/openai" -_RESPONSES_API_CREATE_ROUTES: Final = frozenset({"/v1/responses", "/responses"}) +_RESPONSES_API_ROUTES: Final = frozenset({"/v1/responses", "/responses"}) _ADDRESSED_RESPONSE_ID_FIELD_BY_CALL_TYPE: Final[Mapping[str, str]] = MappingProxyType( { "aresponses": "previous_response_id", @@ -104,7 +104,7 @@ def _rewrite_advertised_id( return event -def _is_responses_api_create_route(request_route: str | None) -> bool: +def _streams_response_ids(request_route: str | None) -> bool: if request_route is None: return False canonical: Final = ( @@ -112,7 +112,8 @@ def _is_responses_api_create_route(request_route: str | None) -> bool: if request_route.startswith(_RESPONSES_API_PROVIDER_PREFIX + "/") else request_route ) - return canonical in _RESPONSES_API_CREATE_ROUTES + parent, _, response_id = canonical.rpartition("/") + return canonical in _RESPONSES_API_ROUTES or (parent in _RESPONSES_API_ROUTES and response_id != "") class ResponsesIDSecurity(CustomLogger): @@ -339,7 +340,7 @@ class ResponsesIDSecurity(CustomLogger): async for chunk in response: if ( isinstance(chunk, BaseLiteLLMOpenAIResponseObject) - and _is_responses_api_create_route(user_api_key_dict.request_route) + and _streams_response_ids(user_api_key_dict.request_route) and not general_settings.get("disable_responses_id_security", False) ): chunk = self._encrypt_response_id(chunk, user_api_key_dict, request_encryption_cache) diff --git a/litellm/proxy/management_endpoints/fallback_management_endpoints.py b/litellm/proxy/management_endpoints/fallback_management_endpoints.py index 6a9a30a7689..ec8ee5ca213 100644 --- a/litellm/proxy/management_endpoints/fallback_management_endpoints.py +++ b/litellm/proxy/management_endpoints/fallback_management_endpoints.py @@ -20,7 +20,7 @@ from litellm._logging import verbose_proxy_logger from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.model_checks import get_all_fallbacks from litellm.proxy.auth.user_api_key_auth import user_api_key_auth -from litellm.proxy.utils import PrismaClient, evict_config_param, invalidate_config_param +from litellm.proxy.utils import PrismaClient, invalidate_config_param if TYPE_CHECKING: from fastapi import APIRouter, Depends, HTTPException, status @@ -48,22 +48,50 @@ FallbackRule = dict[str, list[str]] StoredFallback = Annotated[FallbackRule | dict[str, object] | str, Field(union_mode="left_to_right")] _STORED_FALLBACKS: Final = TypeAdapter(list[StoredFallback]) _JSON_RULES: Final = TypeAdapter(list[JsonValue]) +_STORED_ROUTER_SETTINGS: Final = TypeAdapter(dict[str, JsonValue]) def _rule_covers(entry: StoredFallback, model: str) -> bool: return isinstance(entry, dict) and model in entry -async def _router_settings_fresh_from_db(proxy_config: "ProxyConfig") -> dict[str, object]: - await evict_config_param(ROUTER_SETTINGS_PARAM) - config: Final = await proxy_config.get_config() - return config.get(ROUTER_SETTINGS_PARAM, {}) +async def _reload_config_file_ownership(proxy_config: "ProxyConfig") -> None: + await proxy_config.get_config() -def _reject_config_owned_rules(proxy_config: "ProxyConfig", fallback_key: str, rules: Sequence[StoredFallback]) -> None: +async def _stored_router_settings(prisma_client: PrismaClient) -> Mapping[str, JsonValue]: + try: + row: Final = await ConfigRepository(prisma_client, use_writer=True).get_param(ROUTER_SETTINGS_PARAM) + except json.JSONDecodeError: + return {} + if row is None or not isinstance(row.param_value, Mapping): + return {} + return _STORED_ROUTER_SETTINGS.validate_python(row.param_value) + + +def _editable_rules( + proxy_config: "ProxyConfig", stored_settings: Mapping[str, JsonValue], fallback_key: str +) -> list[StoredFallback]: + file_settings: Final = proxy_config.router_settings + if not file_settings.owned_by_config(fallback_key): + return _STORED_FALLBACKS.validate_python(stored_settings.get(fallback_key) or []) + file_rules: Final = file_settings.get(fallback_key) + return _STORED_FALLBACKS.validate_python(file_rules if isinstance(file_rules, list) else []) + + +async def _write_rules( + prisma_client: PrismaClient, + proxy_config: "ProxyConfig", + stored_settings: Mapping[str, JsonValue], + fallback_key: str, + rules: Sequence[StoredFallback], +) -> None: proxy_config.reject_config_owned_writes( section_name=ROUTER_SETTINGS_PARAM, changed_keys={fallback_key: _JSON_RULES.validate_python(rules)} ) + if proxy_config.router_settings.owned_by_config(fallback_key): + return + await _persist_router_settings(prisma_client, {**stored_settings, fallback_key: rules}) async def _persist_router_settings(prisma_client: PrismaClient, router_settings: Mapping[str, object]) -> None: @@ -162,7 +190,8 @@ async def create_fallback( }, ) - router_settings: Final = await _router_settings_fresh_from_db(proxy_config) + await _reload_config_file_ownership(proxy_config) + router_settings: Final = await _stored_router_settings(prisma_client) # Get the appropriate fallback list based on type fallback_key = "fallbacks" @@ -172,7 +201,7 @@ async def create_fallback( fallback_key = "content_policy_fallbacks" # Get existing fallbacks - existing_fallbacks: Final = _STORED_FALLBACKS.validate_python(router_settings.get(fallback_key) or []) + existing_fallbacks: Final = _editable_rules(proxy_config, router_settings, fallback_key) # Update or add the fallback configuration fallback_updated = False @@ -187,12 +216,7 @@ async def create_fallback( # Add new fallback existing_fallbacks.append({data.model: data.fallback_models}) - _reject_config_owned_rules(proxy_config, fallback_key, existing_fallbacks) - - # Update router settings - router_settings[fallback_key] = existing_fallbacks - - await _persist_router_settings(prisma_client, router_settings) + await _write_rules(prisma_client, proxy_config, router_settings, fallback_key, existing_fallbacks) # Update the in-memory router configuration setattr(llm_router, fallback_key, existing_fallbacks) @@ -320,7 +344,8 @@ async def delete_fallback( }, ) - router_settings: Final = await _router_settings_fresh_from_db(proxy_config) + await _reload_config_file_ownership(proxy_config) + router_settings: Final = await _stored_router_settings(prisma_client) # Get the appropriate fallback list based on type fallback_key = "fallbacks" @@ -330,7 +355,7 @@ async def delete_fallback( fallback_key = "content_policy_fallbacks" # Get existing fallbacks - existing_fallbacks: Final = _STORED_FALLBACKS.validate_python(router_settings.get(fallback_key) or []) + existing_fallbacks: Final = _editable_rules(proxy_config, router_settings, fallback_key) updated_fallbacks: Final = [rule for rule in existing_fallbacks if not _rule_covers(rule, model)] @@ -340,12 +365,7 @@ async def delete_fallback( detail={"error": f"No {fallback_type} fallbacks configured for model '{model}'"}, ) - _reject_config_owned_rules(proxy_config, fallback_key, updated_fallbacks) - - # Update router settings - router_settings[fallback_key] = updated_fallbacks - - await _persist_router_settings(prisma_client, router_settings) + await _write_rules(prisma_client, proxy_config, router_settings, fallback_key, updated_fallbacks) # Update the in-memory router configuration setattr(llm_router, fallback_key, updated_fallbacks) diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 22c4f27d94a..a980b9c33ff 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -16,6 +16,7 @@ import json import os import re import secrets +from collections import Counter from collections.abc import Awaitable, Callable, Mapping, Sequence from copy import deepcopy from html import escape @@ -42,7 +43,7 @@ if TYPE_CHECKING: import jwt from fastapi import APIRouter, Depends, Header, HTTPException, Request, Response, status from fastapi.responses import RedirectResponse -from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError, field_validator +from pydantic import AliasPath, BaseModel, ConfigDict, Field, TypeAdapter, ValidationError, field_validator import litellm from litellm._internal_context import with_service_target @@ -106,6 +107,7 @@ from litellm.proxy.common_utils.html_forms.default_credentials_hint import shoul from litellm.proxy.common_utils.html_forms.jwt_display_template import ( jwt_display_template, ) +from litellm.proxy.common_utils.html_forms.native_client_consent import team_select_field from litellm.proxy.common_utils.html_forms.ui_login import build_ui_login_form from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET, UserApiKeyCache from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler @@ -184,6 +186,8 @@ _CLI_SSO_START_RATE_LIMIT_WINDOW_SECONDS: Final = 60 _CLI_SSO_START_RATE_LIMIT_MAX_ATTEMPTS: Final = 30 _CLI_SSO_USER_CODE_ALPHABET: Final = "ABCDEFGHJKLMNPQRSTUVWXYZ23456789" _CLI_SSO_LOGIN_ID_RE: Final = re.compile(r"^cli-[A-Za-z0-9_-]{12,124}$") +CLI_SSO_BROWSER_TEAM_CHOICE: Final = "browser_team_choice" +CLI_SSO_CHOSEN_TEAM_ID: Final = "chosen_team_id" _CLI_SSO_USER_CODE_RE = re.compile(rf"^[{_CLI_SSO_USER_CODE_ALPHABET}]{{4}}-[{_CLI_SSO_USER_CODE_ALPHABET}]{{4}}$") _CLI_SSO_SCALAR_TYPES: Final = (str, int, float, bool) _CLI_SSO_DEST_KEY_RE: Final = re.compile(r"^[A-Za-z0-9_.-]+$") @@ -670,6 +674,7 @@ def _render_cli_sso_verification_page( verify_url: str, browser_complete_token: str, prefill_user_code: str | None = None, + team_choices: Sequence[tuple[str, str]] = (), ) -> str: escaped_verify_url: Final = escape(verify_url, quote=True) escaped_browser_complete_token: Final = escape(browser_complete_token, quote=True) @@ -716,6 +721,15 @@ def _render_cli_sso_verification_page( letter-spacing: 0.08em; text-transform: uppercase; }} + select {{ + box-sizing: border-box; + width: 100%; + padding: 12px; + margin-bottom: 18px; + border: 1px solid #cbd5e1; + border-radius: 6px; + font-size: 16px; + }} button {{ margin-top: 16px; width: 100%; @@ -735,6 +749,7 @@ def _render_cli_sso_verification_page(

{instructions}

+ {team_select_field(team_choices, require_choice=True)} @@ -823,13 +838,46 @@ async def cli_sso_complete(request: Request, login_id: str): ): raise HTTPException(status_code=400, detail="Invalid verification code") - flow["user_code_verified"] = True - _set_cli_sso_flow(login_id=login_id, cache=cli_sso_session_cache, flow=flow) + team_choice_flow: Final = _CliSsoTeamChoiceFlow.model_validate(flow) + chosen_team_id: Final = _browser_chosen_team_id(team_choice_flow, form_values) + if chosen_team_id is not None and chosen_team_id not in team_choice_flow.teams: + raise HTTPException(status_code=400, detail="Choose one of the teams you belong to") + + _set_cli_sso_flow( + login_id=login_id, + cache=cli_sso_session_cache, + flow={ + **flow, + "user_code_verified": True, + CLI_SSO_CHOSEN_TEAM_ID: chosen_team_id, + "session_data": ( + flow["session_data"] + if chosen_team_id is None + else { + **flow["session_data"], + "teams": [chosen_team_id, *(team for team in team_choice_flow.teams if team != chosen_team_id)], + } + ), + }, + ) html_content: Final = render_cli_sso_success_page() return HTMLResponse(content=html_content, status_code=200) +class _CliSsoTeamChoiceFlow(BaseModel): + model_config = ConfigDict(frozen=True) + + browser_team_choice: bool = Field(default=False, validation_alias=CLI_SSO_BROWSER_TEAM_CHOICE) + teams: tuple[str, ...] = Field(default=(), validation_alias=AliasPath("session_data", "teams")) + + +def _browser_chosen_team_id(flow: _CliSsoTeamChoiceFlow, form_values: Mapping[str, Sequence[str]]) -> str | None: + if not flow.browser_team_choice: + return None + return (form_values.get("team_id") or [""])[0] or None + + def normalize_email(email: str | None) -> str | None: """ Normalize email address to lowercase for consistent storage and comparison. @@ -2416,6 +2464,18 @@ def _cli_sso_session_teams(team_details: Sequence[CliSsoTeamDetail]) -> list[str return [detail.team_id for detail in team_details if detail.team_id is not None] +def _browser_team_choices( + flow: _CliSsoTeamChoiceFlow, team_details: Sequence[CliSsoTeamDetail] +) -> tuple[tuple[str, str], ...]: + if not flow.browser_team_choice: + return () + labeled: Final = tuple( + (detail.team_id, detail.team_alias or detail.team_id) for detail in team_details if detail.team_id is not None + ) + label_counts: Final = Counter(label for _, label in labeled) + return tuple((team_id, label if label_counts[label] == 1 else f"{label} ({team_id})") for team_id, label in labeled) + + def selected_cli_sso_team_detail(team_details: object, team_id: str | None) -> CliSsoTeamDetail | None: """``None`` means the team's grants are unknown. An empty grant is a real value meaning unrestricted, so an unknown one must not be minted as empty.""" @@ -2484,6 +2544,7 @@ async def _complete_cli_sso_callback_session( request.scope.get("litellm_microsoft_interactive_subject"), resolved_user_id, prisma_client ) resolved_teams: Final = _cli_sso_session_teams(team_details) + team_choices: Final = _browser_team_choices(_CliSsoTeamChoiceFlow.model_validate(flow), team_details) attribution_metadata: Final = build_cli_sso_attribution_metadata(result=result) if attribution_metadata: await _persist_cli_sso_user_metadata( @@ -2521,6 +2582,7 @@ async def _complete_cli_sso_callback_session( verify_url=verify_url, browser_complete_token=browser_complete_token, prefill_user_code=prefill_user_code, + team_choices=team_choices, ), status_code=200, ) diff --git a/litellm/proxy/mcp_server/oauth_issuer_stamp_backfill.py b/litellm/proxy/mcp_server/oauth_issuer_stamp_backfill.py index c417187396f..6c6be219a92 100644 --- a/litellm/proxy/mcp_server/oauth_issuer_stamp_backfill.py +++ b/litellm/proxy/mcp_server/oauth_issuer_stamp_backfill.py @@ -43,6 +43,7 @@ from litellm._logging import verbose_proxy_logger from litellm.proxy.db.db_span import db_span from litellm.proxy.mcp_server.oauth_utils import canonicalize_url_identity from litellm.proxy.utils import PrismaClient +from litellm.repositories.table_repositories import MCPServerRepository # The actor the removed discovery write-back stamped rows with. _DISCOVERY_ACTOR: Final = "mcp_oauth_discovery" @@ -106,7 +107,7 @@ def _is_stamped_issuer_row(row: _MCPServerRow) -> bool: async def backfill_discovery_stamped_issuers(prisma_client: PrismaClient) -> int: """Clear gateway-written issuer stamps, returning the number of rows healed.""" - candidate_rows: Final[list[_MCPServerRow]] = await prisma_client.db.litellm_mcpservertable.find_many( + candidate_rows: Final = await MCPServerRepository(prisma_client).table.find_many( where={ "updated_by": _DISCOVERY_ACTOR, "auth_type": {"in": list(_AUTH_TYPES_WITH_ISSUER_ANCHORING)}, diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 972ec18d487..f96fab75bd1 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -68,6 +68,7 @@ from litellm.llms.deepgram.common_utils import ( from litellm.llms.fal_ai.cost_calculator import fal_ai_passthrough_cost, fal_ai_queue_base from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_model_group_in_path from litellm.llms.openai.common_utils import OpenAIError as LiteLLMOpenAIError +from litellm.llms.openai.common_utils import is_openai_agents_api_path, openai_api_resource from litellm.llms.openai.workload_identity import get_workload_identity_bearer_token_for_api_base from litellm.llms.oss_decision import OssDecisionProvider, oss_connection, validate_oss_request from litellm.llms.vertex_ai.vertex_llm_base import VertexBase @@ -3530,6 +3531,10 @@ async def deepgram_listen_websocket_route( ) +_OPENAI_BETA_HEADER: Final = "openai-beta" +_CLIENT_OPENAI_BETA_RESOURCES: Final = frozenset({"agents", "vaults", "chatkit"}) + + class BaseOpenAIPassThroughHandler: @staticmethod async def base_openai_pass_through_handler( @@ -3540,7 +3545,7 @@ class BaseOpenAIPassThroughHandler: base_target_url: str, api_key: str | None, custom_llm_provider: litellm.LlmProviders, - extra_headers: dict | None = None, + extra_headers: Mapping[str, str] | None = None, ): encoded_endpoint = httpx.URL(endpoint).path # Ensure endpoint starts with '/' for proper URL construction @@ -3565,7 +3570,11 @@ class BaseOpenAIPassThroughHandler: endpoint=endpoint, target=str(updated_url), custom_headers=BaseOpenAIPassThroughHandler._assemble_headers( - api_key=api_key, request=request, extra_headers=extra_headers + api_key=api_key, + endpoint=endpoint, + request=request, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, ), is_streaming_request=is_streaming_request, custom_llm_provider=( @@ -3587,27 +3596,41 @@ class BaseOpenAIPassThroughHandler: _base_openai_pass_through_handler = base_openai_pass_through_handler @staticmethod - def _append_openai_beta_header(headers: dict, request: Request) -> dict: - """ - Appends the OpenAI-Beta header to the headers if the request is an OpenAI Assistants API request - """ - if RouteChecks.is_assistants_api_request(request) is True and "OpenAI-Beta" not in headers: - headers["OpenAI-Beta"] = "assistants=v2" - return headers + def _openai_beta(endpoint: str, request: Request, custom_llm_provider: litellm.LlmProviders) -> str | None: + if ( + custom_llm_provider == litellm.LlmProviders.OPENAI + and openai_api_resource(endpoint) in _CLIENT_OPENAI_BETA_RESOURCES + ): + return request.headers.get(_OPENAI_BETA_HEADER) or ( + "agents=v1" if is_openai_agents_api_path(endpoint) else None + ) + return "assistants=v2" if RouteChecks.is_assistants_api_request(request) else None @staticmethod - def _assemble_headers(api_key: str | None, request: Request, extra_headers: dict | None = None) -> dict: - base_headers = {} - if api_key is not None: - base_headers = { - "authorization": f"Bearer {api_key}", - "api-key": f"{api_key}", - } - if extra_headers is not None: - base_headers.update(extra_headers) + def _append_openai_beta_header( + headers: Mapping[str, str], endpoint: str, request: Request, custom_llm_provider: litellm.LlmProviders + ) -> dict[str, str]: + if any(name.lower() == _OPENAI_BETA_HEADER for name in headers): + return {**headers} + openai_beta: Final = BaseOpenAIPassThroughHandler._openai_beta(endpoint, request, custom_llm_provider) + return {**headers} if openai_beta is None else {**headers, _OPENAI_BETA_HEADER: openai_beta} + + @staticmethod + def _assemble_headers( + api_key: str | None, + endpoint: str, + request: Request, + custom_llm_provider: litellm.LlmProviders, + extra_headers: Mapping[str, str] | None = None, + ) -> dict[str, str]: + api_key_headers: Final[Mapping[str, str]] = ( + {} if api_key is None else {"authorization": f"Bearer {api_key}", "api-key": api_key} + ) return BaseOpenAIPassThroughHandler._append_openai_beta_header( - headers=base_headers, + headers={**api_key_headers, **(extra_headers or {})}, + endpoint=endpoint, request=request, + custom_llm_provider=custom_llm_provider, ) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py index a11b0be3983..4eeb9b26174 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py @@ -1,10 +1,12 @@ from datetime import datetime from typing import Final +from urllib.parse import urlparse import httpx import litellm from litellm import stream_chunk_builder +from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.hidden_params import set_hidden_param from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.litellm_logging import ( @@ -16,7 +18,7 @@ from litellm.llms.cohere.chat.v2_transformation import CohereV2ChatConfig from litellm.llms.cohere.common_utils import ( ModelResponseIterator as CohereModelResponseIterator, ) -from litellm.llms.cohere.embed.v1_transformation import CohereEmbeddingConfig +from litellm.llms.cohere.embed.transformation import CohereEmbeddingConfig, CohereEmbeddingRowShape from litellm.proxy._types import PassThroughEndpointLoggingTypedDict from litellm.types.passthrough_endpoints.pass_through_endpoints import ( PassthroughStandardLoggingPayload, @@ -29,6 +31,12 @@ from litellm.types.utils import ( from .base_passthrough_logging_handler import BasePassthroughLoggingHandler +COHERE_EMBED_PATHS: Final = ("/v1/embed", "/v2/embed") + + +def is_cohere_embed_route(url_route: str) -> bool: + return urlparse(url_route).path.rstrip("/").endswith(COHERE_EMBED_PATHS) + class CoherePassthroughLoggingHandler(BasePassthroughLoggingHandler): @property @@ -82,28 +90,24 @@ class CoherePassthroughLoggingHandler(BasePassthroughLoggingHandler): """ Handle Cohere passthrough logging with route detection and cost tracking. """ - # Check if this is an embed endpoint - if "/v1/embed" in url_route and "/v1/embeddings" not in url_route: + if is_cohere_embed_route(url_route): model: Final = request_body.get("model", response_body.get("model", "")) try: - cohere_embed_config: Final = CohereEmbeddingConfig() - litellm_model_response = litellm.EmbeddingResponse() handler_instance: Final = CoherePassthroughLoggingHandler() input_texts = request_body.get("texts", []) if not input_texts: input_texts = request_body.get("input", []) - # Transform the response - litellm_model_response = cohere_embed_config.transform_response( + litellm_model_response: Final = CohereEmbeddingConfig().transform_native_embed_response( response=httpx_response, api_key="", logging_obj=logging_obj, data=request_body, - model_response=litellm_model_response, model=model, encoding=litellm.encoding, input=input_texts, + row_shape=CohereEmbeddingRowShape.TYPED, ) # Calculate cost using LiteLLM's cost calculator @@ -153,20 +157,11 @@ class CoherePassthroughLoggingHandler(BasePassthroughLoggingHandler): "result": litellm_model_response, "kwargs": kwargs, } - except Exception: - # For other routes (e.g., /v2/chat), fall back to chat handler - return super().passthrough_chat_handler( - httpx_response=httpx_response, - response_body=response_body, - logging_obj=logging_obj, - url_route=url_route, - result=result, - start_time=start_time, - end_time=end_time, - cache_hit=cache_hit, - request_body=request_body, - **kwargs, + except Exception as e: + verbose_proxy_logger.warning( + "Could not price the Cohere passthrough embed response, logging it without a cost: %s", e ) + return {"result": None, "kwargs": kwargs} # For non-embed routes (e.g., /v2/chat), fall back to chat handler return super().passthrough_chat_handler( diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py index c410150df70..b98dd42e3de 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py @@ -10,7 +10,7 @@ from typing import Final, Literal, TypeAlias from urllib.parse import urlparse import httpx -from pydantic import TypeAdapter +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError import litellm from litellm._logging import verbose_proxy_logger @@ -21,6 +21,8 @@ from litellm.litellm_core_utils.litellm_logging import ( get_standard_logging_object_payload, ) from litellm.litellm_core_utils.token_counter import high_detail_image_token_upper_bound +from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator +from litellm.llms.openai.common_utils import is_openai_agents_api_path from litellm.llms.openai.openai import OpenAIConfig from litellm.llms.openai.openai import OpenAIConfig as OpenAIConfigType from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig @@ -38,7 +40,13 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( EndpointType, PassthroughStandardLoggingPayload, ) -from litellm.types.utils import EmbeddingResponse, ImageResponse, LlmProviders, PassthroughCallTypes +from litellm.types.utils import ( + EmbeddingResponse, + ImageResponse, + LlmProviders, + PassthroughCallTypes, + StandardPassThroughResponseObject, +) from litellm.utils import ModelResponse, TextCompletionResponse, convert_to_model_response_object # Hostnames that route to OpenAI-compatible APIs. @@ -121,6 +129,52 @@ def _responses_sub_route(path: str) -> tuple[str, ...] | None: return None if anchor is None else tuple(segments[anchor + 1 :]) +_STR_MAPPING: Final = TypeAdapter(Mapping[str, object]) + + +class _AgentConfig(BaseModel): + model_config = ConfigDict(frozen=True) + + model: str + + +class _AgentsApiBody(BaseModel): + model_config = ConfigDict(frozen=True) + + agent: _AgentConfig + + +class _AgentsApiSessionEvent(BaseModel): + model_config = ConfigDict(frozen=True) + + session: _AgentsApiBody + + +_AGENTS_API_BODY: Final = TypeAdapter(_AgentsApiBody | _AgentConfig) + + +def _agent_model(body: Mapping[str, object]) -> str | None: + try: + parsed: Final = _AGENTS_API_BODY.validate_python(body) + except ValidationError: + return None + return parsed.agent.model if isinstance(parsed, _AgentsApiBody) else parsed.model + + +def _session_event_model(chunk: str) -> str | None: + event: Final = BaseModelResponseIterator.string_to_dict_parser(str_line=chunk) + if event is None: + return None + try: + return _AgentsApiSessionEvent.model_validate(event).session.agent.model + except ValidationError: + return None + + +def _streamed_agent_model(all_chunks: Sequence[str]) -> str | None: + return next((model for chunk in all_chunks if (model := _session_event_model(chunk)) is not None), None) + + def _is_remote_high_detail_image(part: object) -> bool: if not isinstance(part, Mapping) or part.get("type") != "image_url": return False @@ -220,14 +274,44 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): return "billed" @staticmethod - def _label_unbilled_responses_call( + def is_openai_agents_route(url_route: str) -> bool: + return is_openai_compatible_url(url_route) and is_openai_agents_api_path(urlparse(url_route).path) + + @staticmethod + def _unbilled_call_model( + url_route: str, request_body: Mapping[str, object], response_body: Mapping[str, object] + ) -> str | None: + if OpenAIPassthroughLoggingHandler.is_openai_agents_route(url_route): + return _agent_model(response_body) or _agent_model(request_body) + response_model: Final = response_body.get("model") + if OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route) and isinstance(response_model, str): + return response_model + return None + + @staticmethod + def _label_model(logging_obj: LiteLLMLoggingObj, model: str | None) -> None: + if model is not None: + logging_obj.model_call_details["model"] = model # rebind-ok: spend rows read the model from here + + @staticmethod + def _label_unbilled_call( logging_obj: LiteLLMLoggingObj, url_route: str, + request_body: Mapping[str, object], response_body: Mapping[str, object], ) -> None: - model: Final = response_body.get("model") - if OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route) and isinstance(model, str): - logging_obj.model_call_details["model"] = model # rebind-ok: spend rows read the model from here + OpenAIPassthroughLoggingHandler._label_model( + logging_obj, OpenAIPassthroughLoggingHandler._unbilled_call_model(url_route, request_body, response_body) + ) + + @staticmethod + def _log_agents_stream( + litellm_logging_obj: LiteLLMLoggingObj, request_body: Mapping[str, object], all_chunks: Sequence[str] + ) -> PassThroughEndpointLoggingTypedDict: + OpenAIPassthroughLoggingHandler._label_model( + litellm_logging_obj, _streamed_agent_model(all_chunks) or _agent_model(request_body) + ) + return {"result": StandardPassThroughResponseObject(response="\n".join(all_chunks)), "kwargs": {}} @staticmethod def is_openai_embeddings_route(url_route: str) -> bool: @@ -397,7 +481,12 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): is_responses: Final = responses_billing != "unbilled" if not (is_chat_completions or is_embeddings or is_image_generation or is_image_editing or is_responses): - OpenAIPassthroughLoggingHandler._label_unbilled_responses_call(logging_obj, url_route, response_body) + OpenAIPassthroughLoggingHandler._label_unbilled_call( + logging_obj, + url_route, + request_body=_STR_MAPPING.validate_python(request_body), + response_body=_STR_MAPPING.validate_python(response_body), + ) return { "result": None, "kwargs": kwargs, @@ -676,6 +765,10 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): """ Handle logging for collected OpenAI streaming chunks with cost tracking. """ + if OpenAIPassthroughLoggingHandler.is_openai_agents_route(url_route): + return OpenAIPassthroughLoggingHandler._log_agents_stream( + litellm_logging_obj, _STR_MAPPING.validate_python(request_body), all_chunks + ) try: is_responses: Final = OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route) terminal_response: Final = ( diff --git a/litellm/proxy/pass_through_endpoints/success_handler.py b/litellm/proxy/pass_through_endpoints/success_handler.py index 9c6c3111209..27e31c8f74d 100644 --- a/litellm/proxy/pass_through_endpoints/success_handler.py +++ b/litellm/proxy/pass_through_endpoints/success_handler.py @@ -22,6 +22,7 @@ from .llm_provider_handlers.assembly_passthrough_logging_handler import ( ) from .llm_provider_handlers.cohere_passthrough_logging_handler import ( CoherePassthroughLoggingHandler, + is_cohere_embed_route, ) from .llm_provider_handlers.cursor_passthrough_logging_handler import ( CursorPassthroughLoggingHandler, @@ -91,7 +92,7 @@ class PassThroughEndpointLogging: self.TRACKED_ANTHROPIC_ROUTES = ["/messages", "/v1/messages/batches"] # Cohere - self.TRACKED_COHERE_ROUTES = ["/v2/chat", "/v1/embed"] + self.TRACKED_COHERE_ROUTES = ["/v2/chat"] self.assemblyai_passthrough_logging_handler = AssemblyAIPassthroughLoggingHandler() # Langfuse @@ -536,13 +537,7 @@ class PassThroughEndpointLogging: return False def is_cohere_route(self, url_route: str) -> bool: - for route in self.TRACKED_COHERE_ROUTES: - if route not in url_route: - continue - if route == "/v1/embed" and "/v1/embeddings" in url_route: - continue - return True - return False + return is_cohere_embed_route(url_route) or any(route in url_route for route in self.TRACKED_COHERE_ROUTES) def is_assemblyai_route(self, url_route: str): parsed_url: Final = urlparse(url_route) @@ -645,6 +640,7 @@ class PassThroughEndpointLogging: or OpenAIPassthroughLoggingHandler.is_openai_image_generation_route(url_route) or OpenAIPassthroughLoggingHandler.is_openai_image_editing_route(url_route) or OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route) + or OpenAIPassthroughLoggingHandler.is_openai_agents_route(url_route) ) def _set_cost_per_request( diff --git a/litellm/proxy/plugin_routes.py b/litellm/proxy/plugin_routes.py index 1dfe5cbc6bb..cfbcfefe9e8 100644 --- a/litellm/proxy/plugin_routes.py +++ b/litellm/proxy/plugin_routes.py @@ -31,6 +31,8 @@ from typing import Final from cryptography.fernet import Fernet, InvalidToken from fastapi import APIRouter, Depends, HTTPException, Request, Response +from pydantic import ConfigDict, TypeAdapter, with_config +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.proxy._types import PluginConfig, SpecialHeaders, UserAPIKeyAuth @@ -39,6 +41,18 @@ from litellm.types.llms.custom_http import httpxSpecialProvider router: Final = APIRouter() + +@with_config(ConfigDict(extra="allow", strict=True)) +class PluginSessionClaim(TypedDict): + plugin: NotRequired[ReadOnly[object]] + user_id: NotRequired[ReadOnly[str]] + user_role: NotRequired[ReadOnly[str]] + exp: NotRequired[ReadOnly[int | float | str]] + + +_PLUGIN_SESSION_CLAIM: Final = TypeAdapter(PluginSessionClaim) + + # Hop-by-hop headers (RFC 7230) and the litellm session cookie — never forwarded # to a plugin backend. Credential headers are added on top per-request from the # canonical SpecialHeaders set so the plugin only ever authenticates via its own @@ -155,7 +169,7 @@ def issue_plugin_session_claim(plugin_name: str, user_id: str | None, user_role: return _plugin_fernet(plugin_name).encrypt(json.dumps(claim).encode()).decode() -def verify_plugin_session_claim(plugin_name: str, ciphertext: str) -> dict: +def verify_plugin_session_claim(plugin_name: str, ciphertext: str) -> PluginSessionClaim: """Verify and decode a plugin session claim. Raises ValueError if the HMAC is invalid, the audience is wrong, or @@ -163,7 +177,7 @@ def verify_plugin_session_claim(plugin_name: str, ciphertext: str) -> dict: """ try: raw: Final = _plugin_fernet(plugin_name).decrypt(ciphertext.encode(), ttl=_CLAIM_TTL_SECONDS) - claim: Final = json.loads(raw) + claim: Final = _PLUGIN_SESSION_CLAIM.validate_python(json.loads(raw)) except (InvalidToken, Exception) as exc: raise ValueError("Invalid, tampered, or expired plugin session claim") from exc diff --git a/litellm/proxy/policy_engine/init_policies.py b/litellm/proxy/policy_engine/init_policies.py index 59d15b0ba0a..2d0e9d25368 100644 --- a/litellm/proxy/policy_engine/init_policies.py +++ b/litellm/proxy/policy_engine/init_policies.py @@ -11,7 +11,7 @@ from collections.abc import Mapping, Sequence from itertools import chain from typing import TYPE_CHECKING, Any, Final, Optional -from litellm._logging import print_startup_message, verbose_proxy_logger +from litellm._logging import print_startup_message, print_startup_warning, verbose_proxy_logger from litellm.proxy.policy_engine.attachment_registry import get_attachment_registry from litellm.proxy.policy_engine.policy_registry import get_policy_registry from litellm.proxy.policy_engine.policy_validator import PolicyValidator @@ -93,7 +93,7 @@ def _print_policies_on_startup( ) else: print_startup_message("\n".join((header, *policy_lines))) - print_startup_message( + print_startup_warning( f"\n{_yellow_color_code}Warning: No policy_attachments configured. Policies will not be applied to any requests.{_reset_color_code}\n", ) sys.stdout.flush() diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 909f6de40cd..0d8ab98a0b1 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -10,16 +10,17 @@ import sys import urllib.parse as urlparse from collections.abc import Callable, Iterable, Mapping, Sequence from pathlib import Path -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Annotated, Any, Final import click import httpx from click.core import ParameterSource from dotenv import load_dotenv -from pydantic import ConfigDict +from pydantic import BeforeValidator, ConfigDict, TypeAdapter +from typing_extensions import ReadOnly, TypedDict import litellm -from litellm._logging import print_startup_message +from litellm._logging import StartupOutputSettings, print_startup_message, print_startup_warning, terminal_text from litellm.constants import DEFAULT_NUM_WORKERS_LITELLM_PROXY from litellm.proxy.db.pgbouncer import ( PgBouncerError, @@ -205,7 +206,7 @@ def deprecated_v2_flag_passed_on_cli() -> bool: def _print_enforce_prisma_migration_check_notice() -> None: - print_startup_message( + print_startup_warning( "\033[1;33mLiteLLM Proxy: --enforce_prisma_migration_check is " "deprecated and has no effect, because the proxy always exits " "when database setup fails at startup. You can safely remove it.\033[0m", @@ -227,6 +228,36 @@ def _print_or_hold_enforce_prisma_migration_check_notice( return print_once +def _empty_as_none(value: object) -> object: + return value or None + + +class _ConfigStartupOutputSettings(TypedDict, total=False): + litellm_settings: ReadOnly[Annotated[StartupOutputSettings | None, BeforeValidator(_empty_as_none)]] + + +_CONFIG_STARTUP_OUTPUT_SETTINGS: Final = TypeAdapter(_ConfigStartupOutputSettings) + + +def _apply_config_suppress_startup_output(litellm_settings: StartupOutputSettings | None) -> None: + if litellm_settings is None or litellm_settings.get("suppress_startup_output") is not True: + return + litellm.suppress_startup_output = True + os.environ["LITELLM_SUPPRESS_STARTUP_OUTPUT"] = "true" + + +def _apply_config_logging_settings(litellm_settings: StartupOutputSettings | None) -> None: + """Turn on the config's json_logs and suppress_startup_output before any worker starts. Workers that + uvicorn or granian spawn import litellm afresh and read both from the env at import, so they log JSON + and stay quiet from their first line instead of from when load_config reaches the key.""" + _apply_config_suppress_startup_output(litellm_settings) + if litellm_settings is None or litellm_settings.get("json_logs") is not True: + return + litellm.json_logs = True + os.environ["JSON_LOGS"] = "true" + litellm.turn_on_json() + + class ProxyInitializationHelpers: @staticmethod def echo_litellm_version(): @@ -373,7 +404,7 @@ class ProxyInitializationHelpers: if "timeout_worker_healthcheck" in inspect.signature(uvicorn.Config.__init__).parameters: uvicorn_args["timeout_worker_healthcheck"] = timeout_worker_healthcheck else: - print_startup_message( + print_startup_warning( f"\033[1;33mLiteLLM Proxy: --timeout_worker_healthcheck " f"requires uvicorn>=0.37.0, but installed uvicorn=={uvicorn.__version__}. " f"Ignoring the flag.\033[0m", @@ -396,7 +427,7 @@ class ProxyInitializationHelpers: import uvicorn if max_requests_before_restart is None: - print_startup_message( + print_startup_warning( "\033[1;33mLiteLLM Proxy: --max_requests_before_restart_jitter " "has no effect without --max_requests_before_restart\033[0m\n", ) @@ -404,7 +435,7 @@ class ProxyInitializationHelpers: if "limit_max_requests_jitter" in inspect.signature(uvicorn.Config.__init__).parameters: uvicorn_args["limit_max_requests_jitter"] = jitter else: - print_startup_message( + print_startup_warning( f"\033[1;33mLiteLLM Proxy: --max_requests_before_restart_jitter " f"requires uvicorn>=0.41.0, but installed uvicorn=={uvicorn.__version__}. " f"Ignoring the flag.\033[0m", @@ -571,12 +602,12 @@ class ProxyInitializationHelpers: print_startup_message(f"\033[1;32mLiteLLM Proxy: Starting server on {host}:{port} using Granian\033[0m\n") if max_requests_before_restart is not None: - print_startup_message( + print_startup_warning( "\033[1;33mLiteLLM: --max_requests_before_restart is not supported by Granian " "(Granian uses workers_lifetime in seconds, not a per-request limit).\033[0m\n", ) if ciphers is not None: - print_startup_message("\033[1;33mLiteLLM: --ciphers is not applied when using --run_granian.\033[0m\n") + print_startup_warning("\033[1;33mLiteLLM: --ciphers is not applied when using --run_granian.\033[0m\n") kwargs: Final[dict[str, object]] = { "target": "litellm.proxy.proxy_server:app", @@ -698,7 +729,7 @@ class ProxyInitializationHelpers: gunicorn_options["max_requests"] = max_requests_before_restart if max_requests_before_restart_jitter is not None: if max_requests_before_restart is None: - print_startup_message( + print_startup_warning( "\033[1;33mLiteLLM Proxy: --max_requests_before_restart_jitter " "has no effect without --max_requests_before_restart\033[0m\n", ) @@ -739,7 +770,7 @@ class ProxyInitializationHelpers: with open(os.devnull, "w") as devnull: subprocess.Popen(command, stdout=devnull, stderr=devnull) except Exception as e: - print_startup_message(f""" + print_startup_warning(f""" LiteLLM Warning: proxy started with `ollama` model\n`ollama serve` failed with Exception{e}. \nEnsure you run `ollama serve` """) @@ -1325,6 +1356,19 @@ def run_server( os.environ[k] = v litellm_settings = None + env_config_file: Final = os.environ.get("CONFIG_FILE_PATH") + if ( + env_config_file + and os.environ.get("LITELLM_CONFIG_BUCKET_NAME") is None + and ProxyConfig().is_yaml(config_file_path=env_config_file) + ): + import asyncio + + _apply_config_suppress_startup_output( + _CONFIG_STARTUP_OUTPUT_SETTINGS.validate_python( + asyncio.run(ProxyConfig().read_config_file(config_file_path=env_config_file)) + ).get("litellm_settings") + ) if config is not None: """ Allow user to pass in db url via config @@ -1342,17 +1386,9 @@ def run_server( ### LITELLM SETTINGS ### litellm_settings = _config.get("litellm_settings", None) - if ( - litellm_settings is not None - and "json_logs" in litellm_settings - and litellm_settings["json_logs"] is True - ): - import litellm - - litellm.json_logs = True - os.environ["JSON_LOGS"] = "true" - - litellm.turn_on_json() + _apply_config_logging_settings( + _CONFIG_STARTUP_OUTPUT_SETTINGS.validate_python(_config).get("litellm_settings") + ) if held_prisma_notice is not None: held_prisma_notice() ### GENERAL SETTINGS ### @@ -1450,7 +1486,9 @@ def run_server( _bad_scheme = unsupported_db_scheme(_candidate_url) if _bad_scheme is not None: print( - f"\033[1;31mLiteLLM Proxy: {unsupported_db_scheme_message(_db_env, _bad_scheme)}\033[0m", + terminal_text( + f"\033[1;31mLiteLLM Proxy: {unsupported_db_scheme_message(_db_env, _bad_scheme)}\033[0m" + ), file=sys.stderr, flush=True, ) @@ -1553,7 +1591,7 @@ def run_server( env_value=os.getenv("USE_V2_MIGRATION_RESOLVER"), ) if deprecated_v2_flag_passed_on_cli() and use_v2_resolver: - print_startup_message( + print_startup_warning( "\033[1;33mLiteLLM Proxy: --use_v2_migration_resolver is " "deprecated and has no effect, because the v2 migration " "resolver is now the default. You can safely remove it. To " @@ -1561,7 +1599,7 @@ def run_server( "--use_legacy_migration_resolver.\033[0m", ) if not use_v2_resolver: - print_startup_message( + print_startup_warning( "\033[1;33mLiteLLM Proxy: Using the legacy (v1) migration " "resolver. It performs the diff-and-force recovery that can " "cause schema thrashing during rolling deploys where two " @@ -1583,22 +1621,26 @@ def run_server( # issues, and any `prisma db push` against a # partitioned LiteLLM_SpendLogs. print( - f"\033[1;31mLiteLLM Proxy: Database migration cannot proceed. {e}\033[0m", + terminal_text(f"\033[1;31mLiteLLM Proxy: Database migration cannot proceed. {e}\033[0m"), file=sys.stderr, flush=True, ) sys.exit(2) if not setup_ok: print( - "\033[1;31mLiteLLM Proxy: Database setup failed after multiple retries. " - "The proxy cannot start safely. Please check your database connection and migration status.\033[0m" + terminal_text( + "\033[1;31mLiteLLM Proxy: Database setup failed after multiple retries. " + "The proxy cannot start safely. Please check your database connection and migration status.\033[0m" + ) ) sys.exit(1) else: print( - "\033[1;31mLiteLLM Proxy: a database URL is set but the prisma CLI is neither on PATH nor importable " - "as a package, so the database cannot be set up. Install it with `pip install 'litellm[extra_proxy]'` " - "or run a shipped LiteLLM image.\033[0m" + terminal_text( + "\033[1;31mLiteLLM Proxy: a database URL is set but the prisma CLI is neither on PATH nor importable " + "as a package, so the database cannot be set up. Install it with `pip install 'litellm[extra_proxy]'` " + "or run a shipped LiteLLM image.\033[0m" + ) ) sys.exit(1) pgbouncer_settings: Final = PgBouncerSettings() @@ -1609,8 +1651,10 @@ def run_server( ) if isinstance(pooled_database_url, PgBouncerError): print( - f"\033[1;31mLiteLLM Proxy: LITELLM_PGBOUNCER_ENABLED is set but the in-container pgbouncer " - f"could not start: {pooled_database_url.reason}\033[0m", + terminal_text( + f"\033[1;31mLiteLLM Proxy: LITELLM_PGBOUNCER_ENABLED is set but the in-container pgbouncer " + f"could not start: {pooled_database_url.reason}\033[0m" + ), file=sys.stderr, flush=True, ) @@ -1675,7 +1719,7 @@ def run_server( run_gunicorn=run_gunicorn is True, ) if single_worker_warning is not None: - print_startup_message(f"\033[1;33m{single_worker_warning}\033[0m\n") + print_startup_warning(f"\033[1;33m{single_worker_warning}\033[0m\n") if run_gunicorn is False and run_hypercorn is False and run_granian is False: if limit_concurrency is not None: uvicorn_args["limit_concurrency"] = limit_concurrency diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 5e37ff1ba65..091e05610ba 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -310,6 +310,7 @@ import litellm._redis from litellm import Router from litellm._internal_context import service_target, with_service_target from litellm._logging import ( + STARTUP_OUTPUT_SETTINGS, print_startup_message, redact_string, verbose_proxy_logger, @@ -5805,6 +5806,12 @@ class ProxyConfig: # verbose_proxy_logger.debug(f"loaded config={json.dumps(config, indent=4)}") return config + async def read_config_file(self, config_file_path: str) -> Mapping[str, object]: + """The config file with its includes merged, without the bucket, database and secret steps of get_config.""" + return await self._process_includes( + config=self._load_yaml_file(config_file_path), config_file_path=os.path.abspath(config_file_path) + ) + async def _process_includes(self, config: dict, config_file_path: str) -> dict: """ Process includes by appending their contents to the main config @@ -6429,13 +6436,8 @@ class ProxyConfig: search_tools_parsed: Final[list[SearchToolTypedDict]] = [] - print_startup_message("\033[32mLiteLLM: Proxy initialized with Search Tools:\033[0m") - for search_tool in search_tools_raw: - # Display loaded search tool search_tool_name = search_tool.get("search_tool_name", "") - search_provider = search_tool.get("litellm_params", {}).get("search_provider", "") - print_startup_message(f"\033[32m {search_tool_name} ({search_provider})\033[0m") # Handle os.environ/ variables in litellm_params litellm_params = search_tool.get("litellm_params", {}) @@ -6605,13 +6607,15 @@ class ProxyConfig: from litellm.types.integrations.prometheus import validate_caller_identity_settings validate_caller_identity_settings(litellm_settings) + if STARTUP_OUTPUT_SETTINGS.validate_python(litellm_settings).get("suppress_startup_output") is True: + litellm.suppress_startup_output = True # ANSI escape code for blue text blue_color_code: Final = "\033[94m" reset_color_code: Final = "\033[0m" for key, value in litellm_settings.items(): if key == "cache" and value is True: - print_startup_message(f"{blue_color_code}\nSetting Cache on Proxy{reset_color_code}") + print_startup_message(f"\n{blue_color_code}Setting Cache on Proxy{reset_color_code}") from litellm.caching.caching import Cache cache_params = {} @@ -6926,6 +6930,8 @@ class ProxyConfig: setattr(litellm, key, value) elif key == "drop_params": litellm.drop_params = drop_params_flag(value, "litellm_settings.drop_params", verbose_proxy_logger) + elif key == "suppress_startup_output": + continue else: verbose_proxy_logger.debug( "%s setting litellm.%s=%s%s", @@ -6972,6 +6978,7 @@ class ProxyConfig: for name in ( "vector_store_deny_by_default", "search_tool_deny_by_default", + "claude_code_marketplace_auth_required", "vertex_ai_pass_through_drop_anthropic_betas", "request_tag_mappings", ) @@ -7281,6 +7288,13 @@ class ProxyConfig: ## SEARCH TOOLS SETTINGS search_tools: Final[list[SearchToolTypedDict] | None] = self.parse_search_tools(config) + if search_tools: + print_startup_message("\033[32mLiteLLM: Proxy initialized with Search Tools:\033[0m") + for tool in search_tools: + print_startup_message( + f"\033[32m {tool.get('search_tool_name', '')} " + f"({tool.get('litellm_params', {}).get('search_provider', '')})\033[0m" + ) ## SANDBOX TOOLS SETTINGS from litellm.sandbox.sandbox_tools import register_sandbox_tools @@ -9683,12 +9697,9 @@ async def initialize( master_key, \ user_custom_auth, \ prisma_client - from litellm.proxy.common_utils.banner import LITELLM_BANNER, show_banner + from litellm.proxy.common_utils.banner import LITELLM_BANNER - if litellm.json_logs: - print_startup_message(LITELLM_BANNER) - else: - show_banner() + print_startup_message(f"\n{LITELLM_BANNER}\n") if os.getenv("LITELLM_DONT_SHOW_FEEDBACK_BOX", "").lower() != "true": generate_feedback_box() user_model = model diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index 2e72cf3d984..04e6c5d8ba9 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -2023,6 +2023,34 @@ ], "default_model_placeholder": "jina_ai/" }, + { + "provider": "KYMA", + "provider_display_name": "Kyma API", + "litellm_provider": "kyma", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://kymaapi.com/v1", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "kyma/qwen3.7-flash" + }, { "provider": "Laya", "provider_display_name": "Laya", @@ -3584,7 +3612,7 @@ "key": "api_key", "label": "Soniox API Key", "placeholder": null, - "tooltip": "Currently only the async Speech-to-Text REST API (api.soniox.com) is supported. Realtime STT (stt-rt.soniox.com) and TTS (tts-rt.soniox.com) are not yet available.", + "tooltip": "Supports async Speech-to-Text (api.soniox.com) on /v1/audio/transcriptions and realtime Speech-to-Text (stt-rt.soniox.com) on /v1/realtime. TTS (tts-rt.soniox.com) is not yet available.", "required": true, "field_type": "password", "options": null, @@ -4228,5 +4256,33 @@ } ], "default_model_placeholder": "cursor/claude-4-sonnet" + }, + { + "provider": "BIGMODEL", + "provider_display_name": "BigModel (Zhipu AI)", + "litellm_provider": "bigmodel", + "credential_fields": [ + { + "key": "api_key", + "label": "BigModel (Zhipu AI) API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + }, + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://open.bigmodel.cn", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": "https://open.bigmodel.cn" + } + ], + "default_model_placeholder": "bigmodel/glm-5.3" } ] diff --git a/litellm/proxy/public_endpoints/public_endpoints.py b/litellm/proxy/public_endpoints/public_endpoints.py index 9f33e0d2adc..b8cdea0dea4 100644 --- a/litellm/proxy/public_endpoints/public_endpoints.py +++ b/litellm/proxy/public_endpoints/public_endpoints.py @@ -367,19 +367,23 @@ async def get_mcp_servers(): tags=["public", "Claude Code Marketplace"], ) async def public_skill_hub(): - """Return enabled (public) Claude Code skills — no auth required.""" + """Return enabled (public) Claude Code skills, or none when the marketplace requires a key.""" from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_marketplace import ( MANIFEST_FIELDS, get_installation_preference, - get_prisma_client, ) + from litellm.proxy.common_utils.claude_code_marketplace_auth import marketplace_auth_required + from litellm.proxy.proxy_server import prisma_client from litellm.types.proxy.claude_code_endpoints import ( ListPluginsResponse, PluginListItem, ) + if marketplace_auth_required(): + return ListPluginsResponse(plugins=[], count=0) + if prisma_client is None: + raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) try: - prisma_client: Final = await get_prisma_client() plugins: Final = await _plugin_table(prisma_client).find_many(where={"enabled": True}) items: Final = [] for plugin in plugins: @@ -406,7 +410,8 @@ async def public_skill_hub(): ) return ListPluginsResponse(plugins=items, count=len(items)) except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + verbose_logger.exception("Error listing public skills: %s", e) + raise HTTPException(status_code=500, detail="Failed to list skills") @router.get( diff --git a/litellm/proxy/rag_endpoints/endpoints.py b/litellm/proxy/rag_endpoints/endpoints.py index a7146f13035..2d81dc32181 100644 --- a/litellm/proxy/rag_endpoints/endpoints.py +++ b/litellm/proxy/rag_endpoints/endpoints.py @@ -54,6 +54,7 @@ from litellm.proxy.rag_endpoints.upload_security import ( from litellm.proxy.vector_store_endpoints.endpoints import ( build_request_data_from_managed_vector_store, reject_caller_embedding_selection_params, + reject_caller_opensearch_index_name, ) from litellm.proxy.vector_store_endpoints.utils import ( assert_user_can_access_vector_store_id, @@ -913,6 +914,11 @@ async def rag_query( **retrieval_config, **store_data, } + reject_caller_opensearch_index_name( + caller_body=retrieval_config, + request_fields=merged_retrieval_config, + source="retrieval_config", + ) effective_vector_store_config: Final = ( managed_store_params if managed_store is not None diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 5737421d4d4..9e642a02d49 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -86,6 +86,7 @@ from litellm.proxy.route_llm_request import raise_if_required_body_param_missing from litellm.proxy.utils import count_request_input_tokens, handle_exception_on_proxy from litellm.responses.litellm_completion_transformation.compaction_items import messages_since_newest_compaction from litellm.responses.litellm_completion_transformation.count_request import MODEL_FILE_ID_MAPPING_FIELD +from litellm.responses.retrieve_stream_query import RETRIEVE_INCLUDE_QUERY_KEYS, RETRIEVE_STREAM_QUERY_KEYS from litellm.responses.streaming_iterator import GuardedResponsesWsFrame, ResponsesWsFrameGuard, spawn_logging_task from litellm.router import Router from litellm.types.llms.base import LiteLLMBaseModel @@ -106,11 +107,43 @@ router: Final = APIRouter() _RESPONSES_WS_CONFIG_VALUE_ADAPTER: Final[TypeAdapter[object | None]] = TypeAdapter(object | None) _DECODED_JSON: Final = TypeAdapter(object) _PROXY_SERVER_REQUEST_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) +_RETRIEVE_QUERY_PARAMS: Final = tuple(sorted(RETRIEVE_INCLUDE_QUERY_KEYS | RETRIEVE_STREAM_QUERY_KEYS)) +_RETRIEVE_QUERY_OPENAPI: Final = { + "parameters": [ + { + "name": "stream", + "in": "query", + "required": False, + "schema": {"type": "boolean"}, + "description": "Replay the response's events as server-sent events (OpenAI, Azure, and Azure AI Foundry)", + }, + { + "name": "starting_after", + "in": "query", + "required": False, + "schema": {"type": "integer", "minimum": 0}, + "description": "With stream=true, replay only the events after this sequence number", + }, + { + "name": "include", + "in": "query", + "required": False, + "schema": {"type": "array", "items": {"type": "string"}}, + "description": "Additional output data to include, forwarded to the provider", + }, + { + "name": "include_obfuscation", + "in": "query", + "required": False, + "schema": {"type": "boolean"}, + "description": "With stream=true, whether delta events carry obfuscation padding", + }, + ] +} _ResponseDocSchemas: TypeAlias = dict[int | str, dict[str, object]] # fastapi's responses kwarg -RESPONSES_API_RESPONSE_SCHEMAS: Final[_ResponseDocSchemas] = {200: {"model": ResponsesAPIResponse}} -RESPONSES_API_CREATE_RESPONSE_SCHEMAS: Final[_ResponseDocSchemas] = { +RESPONSES_API_RESPONSE_SCHEMAS: Final[_ResponseDocSchemas] = { 200: { "model": ResponsesAPIResponse, "content": { @@ -254,19 +287,19 @@ async def _resolve_cursor_model_variant_before_auth(request: Request) -> None: "/v1/responses", dependencies=[Depends(user_api_key_auth)], tags=["responses"], - responses=RESPONSES_API_CREATE_RESPONSE_SCHEMAS, + responses=RESPONSES_API_RESPONSE_SCHEMAS, ) @router.post( "/responses", dependencies=[Depends(user_api_key_auth)], tags=["responses"], - responses=RESPONSES_API_CREATE_RESPONSE_SCHEMAS, + responses=RESPONSES_API_RESPONSE_SCHEMAS, ) @router.post( "/openai/v1/responses", dependencies=[Depends(user_api_key_auth)], tags=["responses"], - responses=RESPONSES_API_CREATE_RESPONSE_SCHEMAS, + responses=RESPONSES_API_RESPONSE_SCHEMAS, ) async def responses_api( request: Request, @@ -749,18 +782,21 @@ async def cursor_chat_completions( dependencies=[Depends(user_api_key_auth)], tags=["responses"], responses=RESPONSES_API_RESPONSE_SCHEMAS, + openapi_extra=_RETRIEVE_QUERY_OPENAPI, ) @router.get( "/responses/{response_id}", dependencies=[Depends(user_api_key_auth)], tags=["responses"], responses=RESPONSES_API_RESPONSE_SCHEMAS, + openapi_extra=_RETRIEVE_QUERY_OPENAPI, ) @router.get( "/openai/v1/responses/{response_id}", dependencies=[Depends(user_api_key_auth)], tags=["responses"], responses=RESPONSES_API_RESPONSE_SCHEMAS, + openapi_extra=_RETRIEVE_QUERY_OPENAPI, ) async def get_response( response_id: str, @@ -785,6 +821,10 @@ async def get_response( # Get provider response curl -X GET http://localhost:4000/v1/responses/resp_abc123 \ -H "Authorization: Bearer $LITELLM_MASTER_KEY" + + # Resume a background response's event stream after sequence number 7 + curl -N "http://localhost:4000/v1/responses/resp_abc123?stream=true&starting_after=7" \ + -H "Authorization: Bearer $LITELLM_MASTER_KEY" ``` """ from litellm.proxy.proxy_server import ( @@ -832,6 +872,14 @@ async def get_response( data: Final = await read_request_body(request=request) read_settles_billing: Final = not carries_client_side_overrides(data) data["response_id"] = response_id + retrieve_query: Final = { + key: values[0] if len(values) == 1 else values + for key in _RETRIEVE_QUERY_PARAMS + if (values := request.query_params.getlist(key)) + } + if retrieve_query: + body_extra_query: Final[object] = data.get("extra_query") + data["extra_query"] = {**(body_extra_query if isinstance(body_extra_query, dict) else {}), **retrieve_query} processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: response: Final[object] = await processor.base_process_llm_request( @@ -843,7 +891,12 @@ async def get_response( llm_router=llm_router, general_settings=general_settings, proxy_config=proxy_config, - select_data_generator=select_data_generator, + select_data_generator=partial( + _streamed_retrieval_data_generator, + partial(select_data_generator, responses_stream_errors=True), + response_id, + read_settles_billing, + ), model=None, user_model=user_model, user_temperature=user_temperature, @@ -917,12 +970,24 @@ async def _bill_finished_background_response( fastapi_response: Response, read_settles_billing: bool, ) -> None: + _report_charge( + await _settle_finished_background_read(response, response_id, user_api_key_dict, read_settles_billing), + fastapi_response, + ) + + +async def _settle_finished_background_read( + response: object, + response_id: str, + user_api_key_dict: UserAPIKeyAuth, + read_settles_billing: bool, +) -> "_BilledCharge | None": from litellm.proxy.proxy_server import proxy_logging_obj if not isinstance(response, ResponsesAPIResponse) or not is_background_response(response): - return + return None if response.status not in TERMINAL_RESPONSE_STATUSES: - return + return None managed_files_hook: Final = _managed_files_hook(proxy_logging_obj) track_untracked: Final = ( partial( @@ -933,12 +998,11 @@ async def _bill_finished_background_response( if managed_files_hook is not None else None ) - charge: Final = await _settle_background_billing( + return await _settle_background_billing( (lambda cost_check: cost_check.bill_if_finished(response_id, response, track_untracked)) if read_settles_billing else (lambda cost_check: cost_check.bill_from_deployment(response_id, track_untracked)) ) - _report_charge(charge, fastapi_response) class _BilledCharge(Protocol): @@ -1088,27 +1152,53 @@ def _streamed_background_data_generator( user_api_key_dict=user_api_key_dict, request_tags=_background_response_spend_tags(request_data), ) - return _bill_streamed_background_response(response, chunks, track_untracked) + return _bill_streamed_background_response( + response, chunks, partial(_bill_finished_stream, track_untracked=track_untracked) + ) + + +def _streamed_retrieval_data_generator( + select_data_generator: Callable[..., AsyncIterator[str | bytes]], + response_id: str, + read_settles_billing: bool, + response: object, + user_api_key_dict: UserAPIKeyAuth, + request_data: Mapping[str, object], + request: Request | None = None, +) -> AsyncIterator[str | bytes]: + chunks: Final = select_data_generator( + response=response, user_api_key_dict=user_api_key_dict, request_data=request_data, request=request + ) + return _bill_streamed_background_response( + response, + chunks, + partial( + _settle_finished_background_read, + response_id=response_id, + user_api_key_dict=user_api_key_dict, + read_settles_billing=read_settles_billing, + ), + ) async def _bill_streamed_background_response( stream: object, chunks: AsyncIterator[str | bytes], - track_untracked: Callable[[str, ResponsesAPIResponse], Awaitable[bool]], + bill: Callable[[ResponsesAPIResponse], Awaitable[object]], ) -> AsyncIterator[str | bytes]: async for chunk in chunks: completed: object = ProxyBaseLLMRequestProcessing.extract_completed_responses_response(stream) if not isinstance(completed, ResponsesAPIResponse): yield chunk continue - await _bill_finished_stream(completed, track_untracked) + await bill(completed) yield chunk async for remaining_chunk in chunks: yield remaining_chunk return late_completed: Final = ProxyBaseLLMRequestProcessing.extract_completed_responses_response(stream) if isinstance(late_completed, ResponsesAPIResponse): - await _bill_finished_stream(late_completed, track_untracked) + await bill(late_completed) async def _bill_finished_stream( diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 94b079163e9..833ee71ae9c 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -80,7 +80,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( SETTLES_BACKGROUND_RESPONSE_KWARG, BackgroundResponseSettlement, ) -from litellm.types.router import DeploymentTypedDict, LiteLLM_Params +from litellm.types.router import DeploymentTypedDict, LiteLLM_Params, SilentExperimentFlag from litellm.types.utils import ( BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, PROMPT_CARRYING_GUARDRAIL_FIELDS, @@ -194,6 +194,7 @@ _STAMPED_METADATA_KEYS: Final = frozenset( "used_client_oauth_token", KUBERNETES_POD_ROUTING_KEY, "litellm_roi_estimator", + "is_silent_experiment", BILLED_RETRY_ATTEMPT_METADATA_KEY, "user_api_key_user_email", ) @@ -227,6 +228,7 @@ def _get_spend_logs_metadata( azure_spillover: AzureSpillover | None = None, used_client_oauth_token: bool | None = None, kubernetes_pod_routing: StandardLoggingKubernetesPodRouting | None = None, + is_silent_experiment: bool = False, ) -> SpendLogsMetadata: if metadata is None: return SpendLogsMetadata( @@ -294,6 +296,7 @@ def _get_spend_logs_metadata( used_client_oauth_token=used_client_oauth_token, kubernetes_pod_routing=kubernetes_pod_routing, litellm_roi_estimator=metadata.get("litellm_roi_estimator") is True, + **(SilentExperimentFlag(is_silent_experiment=True) if is_silent_experiment else SilentExperimentFlag()), billed_retry_attempt=billed_retry_attempt, user_api_key_user_email=_stored_user_email(metadata), ) @@ -738,6 +741,11 @@ def get_logging_payload( # clean up litellm metadata clean_metadata = _get_spend_logs_metadata( persisted_metadata, + is_silent_experiment=( + standard_logging_payload is not None + and "is_silent_experiment" in standard_logging_payload["metadata"] + and standard_logging_payload["metadata"]["is_silent_experiment"] + ), applied_guardrails=( standard_logging_payload["metadata"].get("applied_guardrails", None) if standard_logging_payload is not None diff --git a/litellm/proxy/vector_store_endpoints/endpoints.py b/litellm/proxy/vector_store_endpoints/endpoints.py index 1b54dddcf4f..b6a0ebe3c58 100644 --- a/litellm/proxy/vector_store_endpoints/endpoints.py +++ b/litellm/proxy/vector_store_endpoints/endpoints.py @@ -17,6 +17,7 @@ from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing from litellm.proxy.common_utils.http_parsing_utils import read_raw_json_body +from litellm.proxy.route_llm_request import JSON_OBJECT_ADAPTER from litellm.proxy.utils import jsonify_object from litellm.proxy.vector_store_endpoints.utils import ( assert_proxy_admin_for_vector_store_index_management, @@ -24,6 +25,7 @@ from litellm.proxy.vector_store_endpoints.utils import ( get_litellm_managed_vector_store, ) from litellm.repositories.table_repositories import ManagedVectorStoreIndexRepository +from litellm.types.utils import LlmProviders from litellm.types.vector_stores import IndexCreateRequest, IndexListResponse from litellm.vector_stores.utils import VectorStoreWriteBody, callers_metadata_snapshot from litellm.vector_stores.vector_store_registry import VectorStoreIndexRegistry @@ -52,6 +54,24 @@ def reject_caller_embedding_selection_params(payload: Mapping[str, object], sour ) +def reject_caller_opensearch_index_name( + caller_body: Mapping[str, object], request_fields: Mapping[str, object], source: str +) -> None: + provider: Final = request_fields.get("custom_llm_provider") + if ( + isinstance(provider, str) + and provider.split("/", 1)[0] == LlmProviders.OPENSEARCH.value + and "opensearch_index_name" in caller_body + ): + raise HTTPException( + status_code=400, + detail={ + "error": f"'opensearch_index_name' cannot be set in {source}. " + "The index comes from the vector store's server-side registration." + }, + ) + + ######################################################## # OpenAI Compatible Endpoints ######################################################## @@ -138,11 +158,17 @@ async def vector_store_search( data = await read_request_body(request=request) # rebind-ok: pre-existing rebinding on a rename-only line reject_caller_embedding_selection_params(payload=data, source="the search request body") data["vector_store_id"] = vector_store_id + caller_body: Final = JSON_OBJECT_ADAPTER.validate_python(data) # Check for legacy vector store registry (non-managed vector stores) data = await _update_request_data_with_litellm_managed_vector_store_registry( data=data, vector_store_id=vector_store_id, user_api_key_dict=user_api_key_dict ) + reject_caller_opensearch_index_name( + caller_body=caller_body, + request_fields=JSON_OBJECT_ADAPTER.validate_python(data), + source="the search request body", + ) # The managed_vector_stores pre-call hook will handle: # 1. Decoding managed vector store IDs diff --git a/litellm/realtime_api/README.md b/litellm/realtime_api/README.md index d810de2f24f..987e8be1ed1 100644 --- a/litellm/realtime_api/README.md +++ b/litellm/realtime_api/README.md @@ -4,6 +4,6 @@ Supported endpoints: - WebSocket: `/v1/realtime` (with `intent=transcription` for transcription-only sessions) - HTTP: `/v1/realtime/client_secrets`, `/v1/realtime/transcription_sessions` -Supported providers: OpenAI, Azure OpenAI, Bedrock, Vertex AI, xAI. +Supported providers: OpenAI, Azure OpenAI, Bedrock, Vertex AI, xAI, Gemini, Meta, Soniox. For user-facing documentation and usage examples, see the litellm-docs repo. \ No newline at end of file diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 7b9068ff95e..919fd54e7d6 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -7,6 +7,7 @@ from dataclasses import dataclass from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, cast +from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict, assert_never import litellm @@ -31,7 +32,12 @@ from litellm.types.realtime import ( ) from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import CallTypes, LlmProviders -from litellm.utils import ProviderConfigManager, resolve_api_key_secret_reference +from litellm.utils import ( + ModelInfoLookupMiss, + ProviderConfigManager, + lookup_model_info_for_registration, + resolve_api_key_secret_reference, +) from ..litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params from ..litellm_core_utils.credential_accessor import CredentialAccessor @@ -42,9 +48,15 @@ from ..llms.azure.common_utils import get_azure_ad_token from ..llms.azure.realtime.handler import AzureOpenAIRealtime, azure_realtime_protocol_for_client from ..llms.bedrock.realtime.handler import BedrockRealtime, raise_if_session_committed from ..llms.custom_httpx.http_handler import realtime_ssl_for_url +from ..llms.openai.common_utils import is_openai_backed_api_base from ..llms.openai.live.handler import OpenAILiveSessions -from ..llms.openai.realtime.handler import OpenAIRealtime +from ..llms.openai.realtime.handler import ( + OpenAIRealtime, + confirm_gateway_session_started, + confirm_session_started, +) from ..llms.openai.workload_identity import openai_static_api_key, resolve_openai_bearer_token_async +from ..llms.soniox.realtime.transformation import SonioxRealtimeOptions, soniox_realtime_health_check from ..llms.vertex_ai.audio_transcription.realtime_transformation import is_vertex_speech_to_text_model from ..llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig, vertex_realtime_config from ..llms.vertex_ai.vertex_llm_base import VertexBase @@ -65,6 +77,7 @@ vertex_llm_base: Final = VertexBase() base_llm_http_handler = BaseLLMHTTPHandler() _EMPTY_MODEL_PARAMS: Final[Mapping[str, object]] = MappingProxyType({}) _EMPTY_AUTH_HEADERS: Final[Mapping[str, str]] = MappingProxyType({}) +_DEPLOYMENT_PARAMS: Final = TypeAdapter(Mapping[str, object]) def _model_params_with_stored_credentials(model_params: Mapping[str, object]) -> Mapping[str, object]: @@ -517,6 +530,7 @@ async def _arealtime( custom_llm_provider=_custom_llm_provider, ) + deployment_params: Final = _DEPLOYMENT_PARAMS.validate_python(kwargs) if live_session_start is not None: await _alive_session( model=model, @@ -527,7 +541,7 @@ async def _arealtime( api_base=dynamic_api_base or litellm_params.api_base, api_key=dynamic_api_key, litellm_metadata=_build_litellm_metadata(kwargs), - litellm_params=kwargs, + litellm_params=deployment_params, ) return @@ -538,6 +552,7 @@ async def _arealtime( model=model, provider=LlmProviders(_custom_llm_provider), drop_params=drop_params, + litellm_params=deployment_params, ) if provider_config is not None: await base_llm_http_handler.async_realtime( @@ -740,6 +755,18 @@ def _is_transcription_only_realtime_model(model: str, custom_llm_provider: str) return "/v1/realtime/transcription_sessions" in (model_info.get("supported_endpoints") or ()) +_REALTIME_SERVED_ENDPOINTS: Final = ("/v1/realtime", "/v1/audio/transcriptions", "/v1/live") +_GATEWAY_FIRST_EVENT_WAIT_SECONDS: Final = 3.0 + + +def _cost_map_lists_outside_realtime(model: str) -> bool: + model_info: Final = lookup_model_info_for_registration(model, "openai") + endpoints: Final = ( + () if isinstance(model_info, ModelInfoLookupMiss) else model_info.get("supported_endpoints") or () + ) + return bool(endpoints) and not any(endpoint.startswith(_REALTIME_SERVED_ENDPOINTS) for endpoint in endpoints) + + _TRANSCRIPTION_QUERY_PARAMS: Final[RealtimeQueryParams] = {"intent": "transcription"} @@ -785,6 +812,7 @@ async def realtime_health_check( api_version: str | None = None, realtime_protocol: str | None = None, model_params: Mapping[str, object] | None = None, + gateway_first_event_wait_seconds: float = _GATEWAY_FIRST_EVENT_WAIT_SECONDS, ) -> bool: """ Health check for realtime API - tries connection to the realtime API websocket @@ -800,9 +828,16 @@ async def realtime_health_check( without the OpenAI-Beta header is bridged to, with transcription-only models adding intent=transcription Returns: - bool - True if connection is successful, False otherwise + bool - True once the connection is open, and on OpenAI's own hosts once the first server event is not an + error, since OpenAI accepts the websocket handshake with missing or invalid credentials and only reports + the failure in its first server event. Behind any other OpenAI api_base the first event is read for + `gateway_first_event_wait_seconds` only, and a gateway that stays silent that long counts as healthy. + An `invalid_model` error also counts as healthy unless litellm's + cost map lists only endpoints realtime never serves for the model: OpenAI checks the key first, and Live + and transcription models are served through other endpoints. The listed `mode` is not read, since a + deployment's own `mode: realtime` replaces it when the router registers the deployment Raises: - Exception - if the connection is not successful + Exception - if the connection is not successful or if OpenAI's first server event is any other error """ import websockets @@ -822,10 +857,11 @@ async def realtime_health_check( str | None, api_version or resolved_params.get("api_version") ) url: str | None = None + openai_api_base: Final = resolved_api_base or litellm.api_base or "https://api.openai.com/" health_check_api_key: Final = ( await resolve_openai_bearer_token_async( static_api_key=openai_static_api_key(resolved_api_key), - api_base=resolved_api_base, + api_base=openai_api_base, litellm_params=resolved_params, ) if custom_llm_provider == LlmProviders.OPENAI.value @@ -850,9 +886,13 @@ async def realtime_health_check( query_params=azure_query_params, ) elif custom_llm_provider == "openai": - url = openai_realtime.construct_url( - api_base=resolved_api_base or "https://api.openai.com/", - query_params={"model": model}, + url = openai_realtime.construct_url(api_base=openai_api_base, query_params={"model": model}) + elif custom_llm_provider == "soniox": + return await soniox_realtime_health_check( + model=model, + api_base=resolved_api_base, + api_key=health_check_api_key, + options=SonioxRealtimeOptions.model_validate(dict(resolved_params)), ) elif custom_llm_provider == "xai": url = xai_realtime.construct_url( @@ -899,8 +939,23 @@ async def realtime_health_check( additional_headers=auth_headers, max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, ssl=realtime_ssl_for_url(url), - ): - return True + ) as connection: + if custom_llm_provider != "openai": + return True + if not is_openai_backed_api_base(url): + return await confirm_gateway_session_started( + connection, + model, + sent_credentials=bool(auth_headers), + model_listed_outside_realtime=_cost_map_lists_outside_realtime(model), + first_event_wait_seconds=gateway_first_event_wait_seconds, + ) + return await confirm_session_started( + connection, + model, + sent_credentials=bool(auth_headers), + model_listed_outside_realtime=_cost_map_lists_outside_realtime(model), + ) _realtime_health_check = realtime_health_check diff --git a/litellm/responses/additional_tools.py b/litellm/responses/additional_tools.py index 4b897ff58c0..ce530447c0d 100644 --- a/litellm/responses/additional_tools.py +++ b/litellm/responses/additional_tools.py @@ -1,5 +1,8 @@ -from collections.abc import Sequence +import json +from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass +from itertools import groupby +from types import MappingProxyType from typing import Final, cast # noqa: TID251 # validating the openai tool union strips vendor keys from raw tools from pydantic import ValidationError @@ -15,8 +18,8 @@ class _InputItemType(LiteLLMBaseModel): type: str = "" -class _AdditionalToolsItem(LiteLLMBaseModel): - tools: tuple[dict[str, object], ...] = () +class _ToolList(LiteLLMBaseModel): + tools: tuple[Mapping[str, object], ...] = () @dataclass(frozen=True, slots=True) @@ -33,12 +36,54 @@ def _is_additional_tools_item(item: object) -> bool: return False -def _tools_of_item(item: object) -> tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]: +def _tool_list_of(value: object) -> tuple[Mapping[str, object], ...]: try: - parsed: Final = _AdditionalToolsItem.model_validate(item) + return _ToolList.model_validate(value).tools except ValidationError: return () - return tuple(cast("ALL_RESPONSES_API_TOOL_PARAMS", tool) for tool in parsed.tools) + + +def _tools_of_item(item: object) -> tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]: + return tuple(cast("ALL_RESPONSES_API_TOOL_PARAMS", tool) for tool in _tool_list_of(item)) + + +def _tool_identity(tool: Mapping[str, object]) -> str: + name: Final = tool.get("name") + if isinstance(name, str) and name: + return f"{tool.get('type')}:{name}" + return json.dumps(tool, sort_keys=True, default=str) + + +def _members_of_copies(copies: Sequence[Mapping[str, object]]) -> Iterator[Mapping[str, object]]: + for namespace in copies: + yield from _tool_list_of(namespace) + + +def _merged(copies: Sequence[Mapping[str, object]]) -> Mapping[str, object]: + newest: Final = copies[-1] + if newest.get("type") != "namespace" or len(copies) == 1: + return newest + members: Final = tuple(_members_of_copies(copies)) + return {**newest, "tools": list(_unique_tools(members))} if members else newest + + +def _identity_of(keyed_tool: tuple[str, Mapping[str, object]]) -> str: + return keyed_tool[0] + + +def _unique_tools(tools: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, object], ...]: + """Codex re-sends its tool list in the input on every turn, so the same tool arrives many times. + Keep each tool once, at its first position, with the newest copy's definition, and merge the + members of a namespace sent more than once.""" + keyed: Final = tuple((_tool_identity(tool), tool) for tool in tools) + copies_by_identity: Final = MappingProxyType( + { + identity: tuple(tool for _, tool in group) + for identity, group in groupby(sorted(keyed, key=_identity_of), key=_identity_of) + } + ) + first_seen_order: Final = tuple(dict.fromkeys(identity for identity, _ in keyed)) + return tuple(_merged(copies_by_identity[identity]) for identity in first_seen_order) def hoist_additional_tools( @@ -58,4 +103,8 @@ def hoist_additional_tools( len(items), ) remaining_input: Final = [item for item in input if not _is_additional_tools_item(item)] - return HoistedAdditionalTools(input=remaining_input, tools=(*existing, *hoisted), hoisted=hoisted) + unique: Final = tuple( + cast("ALL_RESPONSES_API_TOOL_PARAMS", tool) # cast-ok: a merged namespace is rebuilt from the same tool dicts + for tool in _unique_tools((*existing, *hoisted)) + ) + return HoistedAdditionalTools(input=remaining_input, tools=unique, hoisted=hoisted) diff --git a/litellm/responses/litellm_completion_transformation/count_request.py b/litellm/responses/litellm_completion_transformation/count_request.py index 7679b4ede71..8e8e4066bb8 100644 --- a/litellm/responses/litellm_completion_transformation/count_request.py +++ b/litellm/responses/litellm_completion_transformation/count_request.py @@ -20,7 +20,7 @@ from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) from litellm.responses.main import previous_response_client_ids -from litellm.types.llms.openai import ResponseInputParam +from litellm.types.llms.openai import ALL_RESPONSES_API_TOOL_PARAMS, ResponseInputParam from litellm.utils import ProviderConfigManager _COUNTED_CHAT_PARAMS: Final = ( @@ -119,14 +119,13 @@ def _with_hoisted_additional_tools(responses_request: Mapping[str, object]) -> M if not isinstance(responses_input, list): return responses_request input_items: Final = cast(ResponseInputParam, responses_input) # cast-ok: forwarded as sent, as inference does - hoisted: Final = hoist_additional_tools(input_items, None) + top_level_tools: Final = cast( # cast-ok: tools are forwarded as sent, as inference does + "tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]", _ITEMS.validate_python(responses_request.get("tools") or ()) + ) + hoisted: Final = hoist_additional_tools(input_items, top_level_tools) if not hoisted.hoisted: return responses_request - return { - **responses_request, - "input": hoisted.input, - "tools": [*_ITEMS.validate_python(responses_request.get("tools") or ()), *hoisted.hoisted], - } + return {**responses_request, "input": hoisted.input, "tools": list(hoisted.tools)} def _with_emulated_file_search(responses_request: Mapping[str, object]) -> Mapping[str, object]: diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 9f0058a5958..892da541935 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -196,6 +196,9 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._tool_search = tool_search_routing(request_tools) self._tool_search_executions: Mapping[str, ToolSearchExecution] = MappingProxyType({}) + async def aclose(self) -> None: + await self.litellm_custom_stream_wrapper.aclose() + def _tool_search_execution(self, call_id: str, fn_name: str) -> ToolSearchExecution | None: known: Final = self._tool_search_executions.get(call_id) if known is not None: diff --git a/litellm/responses/main.py b/litellm/responses/main.py index d474ed3991f..98356e9a5d4 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -90,9 +90,9 @@ from litellm.types.responses.streaming_websocket import ResponsesWebSocketReques from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( ADDRESSED_RESPONSE_ID_FIELD, + AgenticLoopParams, CallTypes, CostPerToken, - CustomPricingLiteLLMParams, all_litellm_params, ) from litellm.utils import ( @@ -106,7 +106,17 @@ if TYPE_CHECKING: else: MCPTool = Any -from .streaming_iterator import BaseResponsesAPIStreamingIterator +from .retrieve_stream_query import ( + RETRIEVE_INCLUDE_QUERY_KEYS, + InvalidRetrieveStreamQuery, + RetrieveStreamQuery, + parse_retrieve_stream_query, +) +from .streaming_iterator import ( + BaseResponsesAPIStreamingIterator, + ResponsesAPIStreamingIterator, + SyncResponsesAPIStreamingIterator, +) __all__ = ( "acancel_responses", @@ -192,6 +202,7 @@ def mock_responses_api_response( _STR_MAPPING: Final = TypeAdapter(Mapping[str, object]) +_STR_DICT: Final = TypeAdapter(dict[str, object]) _FORWARDED_VALUE: Final = TypeAdapter(object) @@ -554,12 +565,6 @@ async def aresponses_api_with_mcp( return response -def _per_call_custom_pricing(litellm_params: GenericLiteLLMParams) -> Mapping[str, float]: - return MappingProxyType( - litellm_params.model_dump(include=set(CustomPricingLiteLLMParams.model_fields), exclude_none=True) - ) - - class _DeploymentModelInfo(BaseModel): model_config = ConfigDict(frozen=True) id: object = None @@ -1569,6 +1574,7 @@ def responses( local_vars=local_vars, use_chat_completions_api=use_chat_completions_api, ) + litellm_logging_obj.record_agentic_loop_params(AgenticLoopParams(model=model)) from litellm.litellm_core_utils.core_helpers import attribute_or_none @@ -1787,7 +1793,7 @@ def responses( optional_params=dict(responses_api_request_params), litellm_params={ **responses_api_request_params, - **_per_call_custom_pricing(litellm_params), + **litellm_params.custom_pricing_overrides(), "aresponses": _is_async, "litellm_call_id": litellm_call_id, "model_info": kwargs.get("model_info"), @@ -2013,6 +2019,53 @@ def delete_responses( ) +def _requested_retrieve_stream_query( + stream: bool | None, + starting_after: int | None, + extra_query: Mapping[str, object] | None, + custom_llm_provider: str | None, + drop_malformed: bool, +) -> RetrieveStreamQuery: + explicit: Final = { + key: value for key, value in (("stream", stream), ("starting_after", starting_after)) if value is not None + } + requested: Final = {**explicit, **(extra_query or {})} + parsed: Final = parse_retrieve_stream_query( + requested.get("stream"), requested.get("starting_after"), drop_malformed=drop_malformed + ) + if isinstance(parsed, InvalidRetrieveStreamQuery): + raise litellm.BadRequestError( + message=parsed.message, + model=None, + llm_provider=custom_llm_provider, + body={"message": parsed.message, "type": "invalid_request_error", "param": parsed.param}, + ) + return parsed + + +def _supported_retrieve_stream_query( + requested: RetrieveStreamQuery, + responses_api_provider_config: BaseResponsesAPIConfig, + custom_llm_provider: str, + drop_params: object, +) -> RetrieveStreamQuery: + if not requested.stream: + return RetrieveStreamQuery(stream=False, starting_after=None) + if responses_api_provider_config.supports_streamed_retrieval(): + return requested + if drops_unsupported_params(drop_params): + return RetrieveStreamQuery(stream=False, starting_after=None) + raise UnsupportedParamsError( + status_code=400, + llm_provider=custom_llm_provider, + message=( + f"{custom_llm_provider} does not support streaming a stored response (stream=true on " + "GET /v1/responses/{response_id}). To drop it, set `litellm.drop_params=True` or for proxy:\n\n" + "`litellm_settings:\n drop_params: true`\n" + ), + ) + + @client async def aget_responses( response_id: str, @@ -2024,8 +2077,10 @@ async def aget_responses( timeout: float | httpx.Timeout | None = None, # LiteLLM specific params, custom_llm_provider: str | None = None, + stream: bool | None = None, + starting_after: int | None = None, **kwargs, -) -> ResponsesAPIResponse: +) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator: """ Async: Fetch a response by its ID. @@ -2058,6 +2113,8 @@ async def aget_responses( extra_query=extra_query, extra_body=extra_body, timeout=timeout, + stream=stream, + starting_after=starting_after, **kwargs, ) @@ -2071,7 +2128,7 @@ async def aget_responses( response = init_response # Update the responses_api_response_id with the model_id - if not isinstance(response, ResponsesAPIResponse): # pyright: ignore[reportUnnecessaryIsInstance] # handlers can return non-ResponsesAPIResponse objects at runtime + if not isinstance(response, ResponsesAPIResponse): return response return ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id( responses_api_response=response, @@ -2101,8 +2158,14 @@ def get_responses( timeout: float | httpx.Timeout | None = None, # LiteLLM specific params, custom_llm_provider: str | None = None, + stream: bool | None = None, + starting_after: int | None = None, **kwargs, -) -> ResponsesAPIResponse | Coroutine[object, object, ResponsesAPIResponse]: +) -> ( + ResponsesAPIResponse + | SyncResponsesAPIStreamingIterator + | Coroutine[object, object, ResponsesAPIResponse | ResponsesAPIStreamingIterator] +): """ Fetch a response by its ID. @@ -2146,6 +2209,20 @@ def get_responses( if responses_api_provider_config is None: raise ValueError(f"GET responses is not supported for {custom_llm_provider}") + stream_query: Final = _supported_retrieve_stream_query( + requested=_requested_retrieve_stream_query( + stream=stream, + starting_after=starting_after, + extra_query=extra_query, + custom_llm_provider=custom_llm_provider, + drop_malformed=drops_unsupported_params(litellm_params.drop_params), + ), + responses_api_provider_config=responses_api_provider_config, + custom_llm_provider=custom_llm_provider, + drop_params=litellm_params.drop_params, + ) + litellm_metadata: Final = _STR_DICT.validate_python(kwargs.get("litellm_metadata") or {}) + local_vars.update(kwargs) # Pre Call logging @@ -2169,7 +2246,13 @@ def get_responses( litellm_params=litellm_params, logging_obj=litellm_logging_obj, extra_headers=extra_headers, + extra_query={ + key: value for key, value in (extra_query or {}).items() if key in RETRIEVE_INCLUDE_QUERY_KEYS + }, extra_body=extra_body, + stream=stream_query.stream, + starting_after=stream_query.starting_after, + litellm_metadata=litellm_metadata, timeout=timeout or request_timeout, _is_async=_is_async, client=kwargs.get("client"), @@ -2180,7 +2263,7 @@ def get_responses( if isinstance(response, ResponsesAPIResponse): response = ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id( responses_api_response=response, - litellm_metadata=kwargs.get("litellm_metadata", {}), + litellm_metadata=litellm_metadata, custom_llm_provider=custom_llm_provider, ) diff --git a/litellm/responses/retrieve_stream_query.py b/litellm/responses/retrieve_stream_query.py new file mode 100644 index 00000000000..086aa39efc6 --- /dev/null +++ b/litellm/responses/retrieve_stream_query.py @@ -0,0 +1,83 @@ +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Final, Literal + +from pydantic import TypeAdapter + +RETRIEVE_STREAM_QUERY_KEYS: Final = frozenset({"stream", "starting_after"}) +RETRIEVE_INCLUDE_QUERY_KEYS: Final = frozenset({"include", "include[]", "include_obfuscation"}) +_QUERY_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_MAX_STARTING_AFTER_DIGITS: Final = len(str(2**63 - 1)) + + +@dataclass(frozen=True, slots=True) +class RetrieveStreamQuery: + stream: bool + starting_after: int | None + + +@dataclass(frozen=True, slots=True) +class InvalidRetrieveStreamQuery: + param: Literal["stream", "starting_after"] + message: str + + +_NO_STREAM: Final = RetrieveStreamQuery(stream=False, starting_after=None) +_FULL_REPLAY: Final = RetrieveStreamQuery(stream=True, starting_after=None) + + +def _parsed_stream(value: object) -> bool | InvalidRetrieveStreamQuery: + match value: + case None: + return False + case bool(): + return value + case str() if value.lower() in ("true", "false"): + return value.lower() == "true" + case _: + return InvalidRetrieveStreamQuery("stream", f"Invalid 'stream': expected true or false, got {value!r}") + + +def _parsed_starting_after(value: object) -> int | None | InvalidRetrieveStreamQuery: + match value: + case None: + return None + case int() if not isinstance(value, bool) and value >= 0: + return value + case str() if value.isascii() and value.isdigit() and len(value) <= _MAX_STARTING_AFTER_DIGITS: + return int(value) + case _: + return InvalidRetrieveStreamQuery( + "starting_after", f"Invalid 'starting_after': expected an integer >= 0, got {value!r}" + ) + + +def parse_retrieve_stream_query( + stream: object, starting_after: object, drop_malformed: bool +) -> RetrieveStreamQuery | InvalidRetrieveStreamQuery: + parsed_stream: Final = _parsed_stream(stream) + if isinstance(parsed_stream, InvalidRetrieveStreamQuery): + return _NO_STREAM if drop_malformed else parsed_stream + if not parsed_stream: + return _NO_STREAM + parsed_starting_after: Final = _parsed_starting_after(starting_after) + if isinstance(parsed_starting_after, InvalidRetrieveStreamQuery): + return _FULL_REPLAY if drop_malformed else parsed_starting_after + return RetrieveStreamQuery(stream=True, starting_after=parsed_starting_after) + + +def requests_streamed_retrieval(request: Mapping[str, object]) -> bool: + extra_query: Final = request.get("extra_query") + return ( + "response_id" in request + and isinstance(extra_query, Mapping) + and _parsed_stream(_QUERY_ADAPTER.validate_python(extra_query).get("stream")) is True + ) + + +def without_retrieve_stream_defaults( + deployment_params: Mapping[str, object], request: Mapping[str, object] +) -> Mapping[str, object]: + if "response_id" not in request: + return deployment_params + return {key: value for key, value in deployment_params.items() if key not in RETRIEVE_STREAM_QUERY_KEYS} diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 6c57bfd2a3f..8959c32fff2 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -376,6 +376,7 @@ class BaseResponsesAPIStreamingIterator: _pending_logging_tasks: tuple[asyncio.Task[object], ...] = () _started_tool_call: bool = False _earlier_rounds_spend: BilledRoundsSpend | None = None + _replays_stored_stream: bool = False def __init__( self, @@ -574,7 +575,7 @@ class BaseResponsesAPIStreamingIterator: openai_types.ResponsesAPIStreamEvents.RESPONSE_FAILED, ): _response_obj: Final[object] = getattr(openai_responses_api_chunk, "response", None) - _estimate_wanted: Final[bool] = _chunk_type in ( + _estimate_wanted: Final[bool] = not self._replays_stored_stream and _chunk_type in ( openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED, openai_types.ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, ) @@ -995,6 +996,7 @@ class BaseResponsesAPIStreamingIterator: generated_text: Final = self._generated_content + self._generated_tool_arguments if ( not generated_text + or self._replays_stored_stream or requests_background(self.request_data) or is_background_response(self._created_response) or self._completed_response_logged @@ -1110,7 +1112,7 @@ class BaseResponsesAPIStreamingIterator: """ created: Final = self._created_response generated: Final = self._generated_content + self._generated_tool_arguments - if created is None or not generated or self._billed_without_disconnect(): + if self._replays_stored_stream or created is None or not generated or self._billed_without_disconnect(): return None usage: Final = _estimate_usage_safely( self.model or "", @@ -1162,6 +1164,8 @@ class BaseResponsesAPIStreamingIterator: def _raise_if_ended_without_terminal_event(self) -> None: if self.completed_response is not None: return + if self._replays_stored_stream and not self._yielded_first_chunk: + return error: Final = self._stream_ended_early_error() self._handle_failure(error) if self._output_started: @@ -1209,6 +1213,7 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): custom_llm_provider: str | None = None, request_data: dict[str, object] | None = None, call_type: str | None = None, + replays_stored_stream: bool = False, ): super().__init__( response, @@ -1220,6 +1225,7 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): request_data, call_type, ) + self._replays_stored_stream = replays_stored_stream self.stream_iterator = SSEDecoder().aiter_bytes(response.aiter_bytes()) def __aiter__(self): @@ -1232,6 +1238,9 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): await self._await_pending_logging() raise + async def aclose(self) -> None: + await self.response.aclose() + async def _next_event(self) -> ResponsesAPIStreamingResponse: try: self._check_max_streaming_duration() @@ -1304,6 +1313,7 @@ class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): custom_llm_provider: str | None = None, request_data: dict[str, object] | None = None, call_type: str | None = None, + replays_stored_stream: bool = False, ): super().__init__( response, @@ -1315,11 +1325,15 @@ class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): request_data, call_type, ) + self._replays_stored_stream = replays_stored_stream self.stream_iterator = SSEDecoder().iter_bytes(response.iter_bytes()) def __iter__(self): return self + def close(self) -> None: + self.response.close() + def __next__(self): try: self._check_max_streaming_duration() diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 05d6a36c1f6..b0db0516658 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -131,14 +131,14 @@ def _provider_mapped_params( params: ResponsesAPIOptionalRequestParams, model: str, drop_params: bool, - declared_reasoning_efforts: Collection[str], + declared_reasoning_efforts: Collection[str] | None, ) -> Mapping[str, object]: from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig if ( - declared_reasoning_efforts + declared_reasoning_efforts is not None and isinstance(config, OpenAIResponsesAPIConfig) - and type(config).map_openai_params is OpenAIResponsesAPIConfig.map_openai_params + and config.honors_declared_reasoning_efforts() ): return config.map_openai_params_for_deployment( params, model, drop_params, declared_reasoning_efforts=declared_reasoning_efforts @@ -287,7 +287,7 @@ class ResponsesAPIRequestUtils: response_api_optional_params: ResponsesAPIOptionalRequestParams, allowed_openai_params: list[str] | None = None, drop_params: bool | None = None, - declared_reasoning_efforts: Collection[str] = (), + declared_reasoning_efforts: Collection[str] | None = None, ) -> dict: """ Get optional parameters for the responses API. diff --git a/litellm/router.py b/litellm/router.py index 02ad6275b82..ff31f48c49e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -75,7 +75,7 @@ from typing_extensions import assert_never, overload import litellm import litellm.litellm_core_utils.exception_mapping_utils from litellm import get_secret_str -from litellm._internal_context import service_target, with_service_target +from litellm._internal_context import in_mirrored_call, mirrored_call, service_target, with_service_target from litellm._logging import verbose_router_logger from litellm._uuid import uuid from litellm.caching.caching import ( @@ -184,6 +184,7 @@ from litellm.llms.openai_like.model_info import ( get_openai_compatible_model_info, ) from litellm.responses.bridge_check import keep_registered_mode +from litellm.responses.retrieve_stream_query import requests_streamed_retrieval, without_retrieve_stream_defaults from litellm.router_strategy.auto_router.auto_router_model_naming import ( AUTO_ROUTER_MODEL_PREFIX, STRATEGY_ROUTER_PARAM_FIELDS, @@ -753,7 +754,7 @@ def _every_deployment_refused( def _silent_experiment_targets(silent_model: object) -> tuple[str, ...]: - if silent_model is None: + if silent_model is None or in_mirrored_call(): return () try: targets: Final = _SILENT_MODEL_ADAPTER.validate_python(silent_model) @@ -3371,16 +3372,17 @@ class Router: remaining_kwargs: Final = MappingProxyType( {key: value for key, value in silent_kwargs.items() if key != "stream"} ) - response: Final = await self.acompletion( - model=silent_model, - messages=cast(list[AllMessageValues], messages), - stream=bool(silent_kwargs.get("stream", False)), - **remaining_kwargs, - ) - if not isinstance(response, CustomStreamWrapper): - return - async for _ in response: - pass + with mirrored_call(): + response: Final = await self.acompletion( + model=silent_model, + messages=cast(list[AllMessageValues], messages), + stream=bool(silent_kwargs.get("stream", False)), + **remaining_kwargs, + ) + if not isinstance(response, CustomStreamWrapper): + return + async for _ in response: + pass def _silent_experiment_completion(self, silent_model: str, messages: Sequence[Mapping[str, str]], **kwargs): """ @@ -4860,7 +4862,7 @@ class Router: Handles inserting this as either "metadata" or "litellm_metadata" depending on the metadata_variable_name """ # 1) copy your defaults and pull out metadata - defaults: Final = self.default_litellm_params.copy() + defaults: Final = dict(without_retrieve_stream_defaults(self.default_litellm_params, kwargs)) metadata_defaults: Final = defaults.pop("metadata", {}) or {} # 2) add any non-metadata defaults that aren't already in kwargs @@ -5101,12 +5103,13 @@ class Router: router_timeout=_router_timeout, router_stream_timeout=_router_stream_timeout, ) + timeout_request: Final = {**kwargs, "stream": True} if requests_streamed_retrieval(kwargs) else kwargs kwargs.update( CompletionTimeout.stream_request_kwargs( - passthrough_timeout(kwargs={**kwargs, "stream": False}), passthrough_timeout(kwargs=kwargs) + passthrough_timeout(kwargs={**kwargs, "stream": False}), passthrough_timeout(kwargs=timeout_request) ) - if splits_stream_timeout and kwargs.get("stream") - else {"timeout": passthrough_timeout(kwargs=kwargs)} + if splits_stream_timeout and timeout_request.get("stream") + else {"timeout": passthrough_timeout(kwargs=timeout_request)} ) elif splits_stream_timeout: request_kwargs: Final = _REQUEST_KWARGS_ADAPTER.validate_python(kwargs) @@ -6440,7 +6443,7 @@ class Router: generic_function_name: Final[str] = getattr(original_generic_function, "__name__", "") response_kwargs: Final = { - **data, + **without_retrieve_stream_defaults(data, kwargs), "caching": self.cache_responses, **kwargs, "model": model_name, diff --git a/litellm/router_utils/reasoning_effort_capability.py b/litellm/router_utils/reasoning_effort_capability.py index 16b8a22a40f..3c3fad93579 100644 --- a/litellm/router_utils/reasoning_effort_capability.py +++ b/litellm/router_utils/reasoning_effort_capability.py @@ -426,6 +426,27 @@ def _log_remap( logged_deployment_id: Final = _single_line(deployment_id) logged_model_group: Final = _single_line(model_group) logged_deployment_model: Final = _single_line(deployment_model) + extra: Final = MappingProxyType( + { + "reasoning_effort_deployment_id": logged_deployment_id, + "reasoning_effort_model_group": logged_model_group, + "reasoning_effort_deployment_model": logged_deployment_model, + "reasoning_effort_requested": logged_requested_level, + "reasoning_effort_sent": logged_sent_level, + "reasoning_effort_rounding": rounding, + } + ) + if sent_level is None and requested_level != _DEFAULT_EFFORT: + verbose_logger.warning( + "Dropping reasoning level %s (drop_params=True) for deployment %s, model group %s, model %s because its " + "model_info.reasoning_effort_levels does not list that level. List every level the deployment supports.", + logged_requested_level, + logged_deployment_id, + logged_model_group, + logged_deployment_model, + extra=extra, + ) + return verbose_logger.info( "reasoning_effort %s sent as %s (rounding %s) to deployment %s, model group %s, model %s", logged_requested_level, @@ -434,14 +455,7 @@ def _log_remap( logged_deployment_id, logged_model_group, logged_deployment_model, - extra={ - "reasoning_effort_deployment_id": logged_deployment_id, - "reasoning_effort_model_group": logged_model_group, - "reasoning_effort_deployment_model": logged_deployment_model, - "reasoning_effort_requested": logged_requested_level, - "reasoning_effort_sent": logged_sent_level, - "reasoning_effort_rounding": rounding, - }, + extra=extra, ) diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index 7364a1d72f2..62b48621c05 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -14,6 +14,7 @@ class ForkedAfterNativeRuntimeStarted(RuntimeError): ... class ProcessReservedForForking(RuntimeError): ... def trace_encode_error(message: str) -> bytes: ... +def bundled_model_cost_map() -> bytes: ... @final class NativeClickHouseSpendConfig: @@ -220,6 +221,7 @@ __all__ = [ "aocr", "aresponses", "atranscription", + "bundled_model_cost_map", "completion", "embedding", "gil_stats", diff --git a/litellm/rust_bridge/catalog.py b/litellm/rust_bridge/catalog.py index 74cd28161a4..fe52f8416d0 100644 --- a/litellm/rust_bridge/catalog.py +++ b/litellm/rust_bridge/catalog.py @@ -96,6 +96,8 @@ class Feature(str, Enum): """litellm.include_cost_in_streaming_usage, where Python adds cost to streamed usage.""" THINKING_SIGNATURE_RETRY = "thinking_signature_retry" """Replayed thinking blocks, which Python strips and retries once if Anthropic rejects their signature.""" + TOOL_CUSTOM_FIELD = "tool_custom_field" + """Tools carrying a non-spec custom object, which Python drops or hoists for the provider.""" CLIENT_HEADERS = "client_headers" """Proxy-forwarded headers that Python merges into extra_headers.""" DEPLOYMENT_HOOK = "deployment_hook" @@ -176,6 +178,7 @@ MESSAGES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | { Feature.CACHE_CONTROL_INJECTION, Feature.STREAM_COST_INJECTION, Feature.THINKING_SIGNATURE_RETRY, + Feature.TOOL_CUSTOM_FIELD, } RESPONSES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | { diff --git a/litellm/rust_bridge/messages/detectors.py b/litellm/rust_bridge/messages/detectors.py index 5f75858d350..52917126b3b 100644 --- a/litellm/rust_bridge/messages/detectors.py +++ b/litellm/rust_bridge/messages/detectors.py @@ -43,6 +43,11 @@ def thinking_signature_retry(request: NativeCall) -> bool: return any(_replays_thinking(message) for message in optional_sequence(request.resolved.get("messages")) or ()) +def tool_custom_field(request: NativeCall) -> bool: + tools: Final = optional_sequence(request.resolved.get("tools")) or () + return any("custom" in (optional_mapping(tool) or {}) for tool in tools) + + def stream_cost_injection(request: NativeCall) -> bool: """Python adds response cost to streaming usage when the global setting is truthy.""" return streaming(request) and bool(litellm.include_cost_in_streaming_usage) diff --git a/litellm/tool_loop.py b/litellm/tool_loop.py index aaa0d62eea7..d0b2850bb5b 100644 --- a/litellm/tool_loop.py +++ b/litellm/tool_loop.py @@ -63,6 +63,8 @@ def _assistant_tool_call(tool_call: ChatCompletionMessageToolCall) -> ChatComple def _assistant_message( message: Message, tool_calls: tuple[ChatCompletionMessageToolCall, ...], replays_provider_specific_fields: bool ) -> ChatCompletionAssistantMessage: + from litellm.completion_extras.litellm_responses_transformation.history import retained_replay_fields + return cast( # cast-ok: dict literal with the optional message fields spread in only when set "ChatCompletionAssistantMessage", { @@ -78,7 +80,7 @@ def _assistant_message( "provider_specific_fields", getattr(message, "provider_specific_fields", None) if replays_provider_specific_fields - else None, + else retained_replay_fields(getattr(message, "provider_specific_fields", None)), ), ) if value is not None diff --git a/litellm/types/integrations/custom_logger.py b/litellm/types/integrations/custom_logger.py index 13542f22f88..77e75dea030 100644 --- a/litellm/types/integrations/custom_logger.py +++ b/litellm/types/integrations/custom_logger.py @@ -97,9 +97,9 @@ class AgenticLoopRequestPatch(LiteLLMBaseModel): model: str | None = None messages: list[dict[str, Any]] | None = None - tools: list[dict[str, Any]] | None = None + tools: list[dict[str, object]] | None = None max_tokens: int | None = None - optional_params: dict[str, Any] = Field(default_factory=dict) + optional_params: dict[str, object] = Field(default_factory=dict) kwargs: dict[str, Any] = Field(default_factory=dict) diff --git a/litellm/types/integrations/datadog_llm_obs.py b/litellm/types/integrations/datadog_llm_obs.py index f4fcabf53ed..f2042cb7967 100644 --- a/litellm/types/integrations/datadog_llm_obs.py +++ b/litellm/types/integrations/datadog_llm_obs.py @@ -5,7 +5,7 @@ API Reference: https://docs.datadoghq.com/llm_observability/setup/api/?tab=examp """ from collections.abc import Sequence -from typing import Any, Literal +from typing import Literal from typing_extensions import ReadOnly, TypedDict @@ -16,7 +16,7 @@ class ToolCall(TypedDict, total=False): """A tool call on a message, as LLM Obs names its fields.""" name: ReadOnly[str] - arguments: ReadOnly[dict[str, Any] | str] # parsed object, or the raw string when it will not parse to one + arguments: ReadOnly[dict[str, object] | str] # parsed object, or the raw string when it will not parse to one tool_id: ReadOnly[str] type: ReadOnly[str] @@ -35,7 +35,7 @@ class ToolDefinition(TypedDict, total=False): name: ReadOnly[str] description: ReadOnly[str] - schema: ReadOnly[dict[str, Any]] + schema: ReadOnly[dict[str, object]] class Message(TypedDict, total=False): @@ -49,13 +49,15 @@ class Message(TypedDict, total=False): class InputMeta(TypedDict): - messages: Sequence[ - Message | dict[str, Any] # changed to fit with tool calls + messages: ReadOnly[ + Sequence[ + Message | dict[str, object] # changed to fit with tool calls + ] ] # Relevant Issue: https://github.com/BerriAI/litellm/issues/9494 class OutputMeta(TypedDict): - messages: Sequence[Any] + messages: ReadOnly[Sequence[object]] class DDLLMObsError(TypedDict, total=False): @@ -71,7 +73,7 @@ class Meta(TypedDict, total=False): kind: Literal["llm", "tool", "task", "embedding", "retrieval"] input: InputMeta # The span's input information. output: OutputMeta # The span's output information. - metadata: dict[str, Any] + metadata: ReadOnly[dict[str, object]] error: DDLLMObsError | None # Error information on the span tool_definitions: ReadOnly[Sequence[ToolDefinition]] # The tools offered to the model on this request diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index de83e6ea66e..50c4a2340a3 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -832,6 +832,7 @@ ANTHROPIC_EFFORT_BETA_HEADER: Final = "effort-2025-11-24" ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER: Final = "mid-conversation-output-config-2026-07-01" ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER: Final = "thinking-display-updates-2026-08-18" +ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER: Final = "thinking-binding-controls-2026-08-01" ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER: Final = "mid-conversation-tool-changes-2026-07-01" ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER: Final = "fine-grained-tool-streaming-2025-05-14" diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 6ce51abcd8a..d9d86cf54b3 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1600,7 +1600,7 @@ class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject): @field_validator("reasoning", mode="before") @classmethod - def validate_reasoning_to_dict(cls, value: Any) -> dict[str, Any] | None: + def validate_reasoning_to_dict(cls, value: Any) -> builtins.object: """Accept API reasoning dict (including effort 'none'/'xhigh'); always store as dict.""" if value is None: return None diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index f1176f777c7..a938f6a7b1c 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -1,6 +1,6 @@ from collections.abc import Sequence from enum import Enum -from typing import Any, Final, Literal, Protocol +from typing import Final, Literal, Protocol from typing_extensions import ( ReadOnly, @@ -140,7 +140,7 @@ class Schema(TypedDict, total=False): title: str description: str nullable: bool - default: Any + default: ReadOnly[object] items: "Schema" minItems: str maxItems: str @@ -155,7 +155,7 @@ class Schema(TypedDict, total=False): minLength: str maxLength: str pattern: str - example: Any + example: ReadOnly[object] anyOf: list["Schema"] @@ -376,7 +376,7 @@ class RequestBody(TypedDict, total=False): class CachedContentRequestBody(TypedDict, total=False): contents: Required[list[ContentType]] system_instruction: SystemInstructions - tools: Tools + tools: Tools | None toolConfig: ToolConfig model: Required[str] # Format: models/{model} ttl: str # ending in 's' - Example: "3.5s". @@ -794,9 +794,9 @@ class VertexVideoGenerationResponse(TypedDict, total=False): name: str done: bool - response: dict[str, Any] - metadata: dict[str, Any] - error: dict[str, Any] + response: ReadOnly[dict[str, object]] + metadata: ReadOnly[dict[str, object]] + error: ReadOnly[dict[str, object]] VERTEX_CREDENTIALS_TYPES = str | dict[str, str] diff --git a/litellm/types/proxy/discovery_endpoints/ui_discovery_endpoints.py b/litellm/types/proxy/discovery_endpoints/ui_discovery_endpoints.py index d396c79b844..ca6ada0d1ec 100644 --- a/litellm/types/proxy/discovery_endpoints/ui_discovery_endpoints.py +++ b/litellm/types/proxy/discovery_endpoints/ui_discovery_endpoints.py @@ -15,4 +15,5 @@ class UiDiscoveryEndpoints(LiteLLMBaseModel): is_control_plane: bool = False mcp_stdio_enabled: bool = False use_si_terminology: bool = False + claude_code_marketplace_auth_required: bool = False workers: list[WorkerRegistryEntry] = Field(default=[]) diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py b/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py index d38e54986b9..4e4983c53f0 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py @@ -48,7 +48,7 @@ class GenericGuardrailAPIMetadata(TypedDict, total=False): class GenericGuardrailAPIOptionalParams(LiteLLMBaseModel): """Optional parameters for the Generic Guardrail API""" - additional_provider_specific_params: dict[str, Any] | None = Field( + additional_provider_specific_params: dict[str, object] | None = Field( default=None, description="Additional provider-specific parameters to send with the guardrail request", ) @@ -161,7 +161,7 @@ class GenericGuardrailAPIRequest(LiteLLMBaseModel): default=None, description="LiteLLM library version running this proxy.", ) - additional_provider_specific_params: dict[str, Any] | None = None + additional_provider_specific_params: dict[str, object] | None = None tool_calls: list[ChatCompletionToolCallChunk] | list[ChatCompletionMessageToolCall] | None = None model: str | None = None # the model being used for the LLM call diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/ibm/ibm_detector.py b/litellm/types/proxy/guardrails/guardrail_hooks/ibm/ibm_detector.py index b58d1bff8a2..b04be71dc9c 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/ibm/ibm_detector.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/ibm/ibm_detector.py @@ -1,7 +1,5 @@ -from typing import Any - from pydantic import Field -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict from litellm.types.llms.base import LiteLLMBaseModel @@ -14,41 +12,41 @@ from .base import IBMGuardrailsBaseConfigModel class IBMDetectorRequestBodyDetectorServer(TypedDict): """Request body for calling IBM Detector Server directly""" - contents: list[str] - detector_params: dict[str, Any] + contents: ReadOnly[list[str]] + detector_params: ReadOnly[dict[str, object]] class IBMDetectorRequestBodyOrchestrator(TypedDict): """Request body for calling IBM Detector via FMS Guardrails Orchestrator""" - content: str - detectors: dict[str, dict[str, Any]] + content: ReadOnly[str] + detectors: ReadOnly[dict[str, dict[str, object]]] class IBMDetectorDetection(TypedDict, total=False): """Individual detection from IBM Detector""" - start: int - end: int - text: str - detection: str - detection_type: str - score: float - evidences: list[Any] - metadata: dict[str, Any] - detector_id: str | None # Only present in orchestrator response + start: ReadOnly[int] + end: ReadOnly[int] + text: ReadOnly[str] + detection: ReadOnly[str] + detection_type: ReadOnly[str] + score: ReadOnly[float] + evidences: ReadOnly[list[object]] + metadata: ReadOnly[dict[str, object]] + detector_id: ReadOnly[str | None] # Only present in orchestrator response class IBMDetectorResponseDetectorServer(TypedDict): """Response from IBM Detector Server (returns list of lists)""" - detections: list[list[IBMDetectorDetection]] + detections: ReadOnly[list[list[IBMDetectorDetection]]] class IBMDetectorResponseOrchestrator(TypedDict): """Response from IBM FMS Guardrails Orchestrator""" - detections: list[IBMDetectorDetection] + detections: ReadOnly[list[IBMDetectorDetection]] # Pydantic Config Models @@ -57,12 +55,12 @@ class IBMDetectorResponseOrchestrator(TypedDict): class IBMDetectorOptionalParams(LiteLLMBaseModel): """Optional parameters for IBM Detector guardrail""" - detector_params: dict[str, Any] | None = Field( + detector_params: dict[str, object] | None = Field( default_factory=lambda: {}, description="Dictionary of arguments to pass to the detector.", ) - extra_headers: dict[str, Any] | None = Field( + extra_headers: dict[str, object] | None = Field( default_factory=lambda: {}, description="Dictionary of extra headers to pass to the detector.", ) diff --git a/litellm/types/proxy/management_endpoints/key_management_endpoints.py b/litellm/types/proxy/management_endpoints/key_management_endpoints.py index f4348391710..d4f41b538e6 100644 --- a/litellm/types/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/types/proxy/management_endpoints/key_management_endpoints.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import Any, Final, Literal, TypeAlias +from typing import Final, Literal, TypeAlias from pydantic import ConfigDict, model_validator from typing_extensions import ReadOnly, TypedDict @@ -50,14 +50,14 @@ class SuccessfulKeyUpdate(LiteLLMBaseModel): """Successfully updated key with its updated information""" key: str - key_info: dict[str, Any] + key_info: dict[str, object] class FailedKeyUpdate(LiteLLMBaseModel): """Failed key update with reason""" key: str - key_info: dict[str, Any] | None = None + key_info: dict[str, object] | None = None failed_reason: str @@ -78,14 +78,14 @@ class KeyUpdateFields(LiteLLMBaseModel): max_budget: float | None = None budget_id: str | None = None budget_duration: str | None = None - budget_limits: list[Any] | None = None - model_max_budget: dict[str, Any] | None = None + budget_limits: list[object] | None = None + model_max_budget: dict[str, object] | None = None # Rate limits tpm_limit: int | None = None rpm_limit: int | None = None - model_tpm_limit: dict[str, Any] | None = None - model_rpm_limit: dict[str, Any] | None = None + model_tpm_limit: dict[str, object] | None = None + model_rpm_limit: dict[str, object] | None = None max_parallel_requests: int | None = None rpm_limit_type: Literal["guaranteed_throughput", "best_effort_throughput", "dynamic"] | None = None tpm_limit_type: Literal["guaranteed_throughput", "best_effort_throughput", "dynamic"] | None = None @@ -99,7 +99,7 @@ class KeyUpdateFields(LiteLLMBaseModel): # Operational metadata tags: list[str] | None = None - metadata: dict[str, Any] | None = None + metadata: dict[str, object] | None = None @model_validator(mode="after") def validate_temp_budget(self) -> "KeyUpdateFields": diff --git a/litellm/types/router.py b/litellm/types/router.py index 60c9e596588..db8be9e91a3 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -67,6 +67,10 @@ from .utils import ( AllowedModelRegion: TypeAlias = Literal["eu", "us"] +class SilentExperimentFlag(TypedDict, total=False): + is_silent_experiment: ReadOnly[bool] + + class FallbackSkipReason(str, enum.Enum): NOT_AUTHORIZED = "not_authorized" OVER_BUDGET = "over_budget" diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 70ca05904ef..3bdcba75778 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -263,10 +263,10 @@ class AgenticLoopParams(TypedDict, total=False): """ model: str - """The model string with provider prefix (e.g., 'bedrock/invoke/...')""" + """The model as the caller requested it, usually with its provider prefix (e.g., 'bedrock/invoke/...')""" custom_llm_provider: str - """The LLM provider name (e.g., 'bedrock', 'anthropic')""" + """The LLM provider name (e.g., 'bedrock', 'anthropic'), when the recorder resolved one""" class OffPeakWindow(TypedDict, total=False): @@ -3374,6 +3374,7 @@ class StandardLoggingMetadata(StandardLoggingUserAPIKeyMetadata): team_alias: str | None team_id: str | None used_client_oauth_token: ReadOnly[bool | None] + is_silent_experiment: ReadOnly[NotRequired[bool]] class AzureSpillover(TypedDict): @@ -4130,11 +4131,28 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): """ return {k: v for k, v in model_info.items() if k not in cls.model_fields} + def custom_pricing_overrides(self) -> Mapping[str, object]: + return MappingProxyType( + self.model_dump(include=set(CustomPricingLiteLLMParams.model_fields), exclude_none=True) + ) + + def per_second_pricing(self) -> Mapping[str, float]: + return MappingProxyType( + self.model_dump( + include={"cost_per_second", "input_cost_per_second", "output_cost_per_second"}, exclude_none=True + ) + ) + DEPLOYMENT_SCOPED_PRICING_FIELDS: Final[frozenset[str]] = frozenset({"off_peak_pricing"}) DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset( - {"supports_regex_lookaround", "supports_assistant_prefill", "supports_bedrock_native_structured_output"} + { + "supports_regex_lookaround", + "supports_assistant_prefill", + "supports_bedrock_native_structured_output", + "supports_pdf_input", + } ) DEPLOYMENT_SCOPED_TOKEN_COUNT_FIELDS: Final[frozenset[str]] = frozenset({"token_count_framing", "token_count_model"}) @@ -4424,6 +4442,7 @@ class LlmProviders(str, Enum): JINA_AI = "jina_ai" XAI = "xai" ZAI = "zai" + BIGMODEL = "bigmodel" CUSTOM_OPENAI = "custom_openai" TEXT_COMPLETION_OPENAI = "text-completion-openai" COHERE = "cohere" @@ -4545,6 +4564,7 @@ class LlmProviders(str, Enum): S3_VECTORS = "s3_vectors" VALKEY = "valkey" MONGODB = "mongodb" + OPENSEARCH = "opensearch" HELICONE = "helicone" HYPERBOLIC = "hyperbolic" RECRAFT = "recraft" @@ -4598,6 +4618,7 @@ class LlmProviders(str, Enum): CURSOR = "cursor" BEDROCK_MANTLE = "bedrock_mantle" GDC = "gdc" + KYMA = "kyma" # Create a set of all provider values for quick lookup diff --git a/litellm/utils.py b/litellm/utils.py index 72c37c3760e..5fe3e63f31a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -29,8 +29,8 @@ import threading import time import traceback from contextlib import nullcontext -from dataclasses import dataclass, field -from functools import lru_cache, wraps +from dataclasses import dataclass, field, replace +from functools import lru_cache, partial, wraps from importlib import resources from inspect import iscoroutine from io import StringIO @@ -91,7 +91,7 @@ from litellm.litellm_core_utils.fallback_generalizations import ( match_capability_generalizations, match_fill_missing_generalizations, ) -from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry +from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry, GetModelCostMap from litellm.litellm_core_utils.model_param_helper import peek_reasoning_summary_aliases from litellm.litellm_core_utils.offline_mode import is_offline_mode from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload @@ -220,6 +220,7 @@ def _get_cached_audio_utils(): from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper as _TextCompletionStreamWrapper +from litellm.types.llms.base_model import LiteLLMBaseModel from litellm.types.llms.openai import ( AllMessageValues, AllPromptValues, @@ -2032,16 +2033,15 @@ def client(original_function): # MODEL CALL result = original_function(*args, **kwargs) end_time = datetime.datetime.now() - if _is_streaming_request( - kwargs=kwargs, - call_type=call_type, - ): + if _is_streaming_request(kwargs=kwargs, call_type=call_type) or _is_converted_stream_result(result): if "complete_response" in kwargs and kwargs["complete_response"] is True: chunks = [] for idx, chunk in enumerate(result): chunks.append(chunk) return litellm.stream_chunk_builder(chunks, messages=kwargs.get("messages", None)) else: + logging_obj.stream = True + logging_obj.model_call_details["stream"] = True # RETURN RESULT update_response_metadata: _ResponseMetadataUpdater = litellm_utils.update_response_metadata update_response_metadata( @@ -2578,6 +2578,8 @@ _NEVER_STREAMING_CALL_TYPES: Final = frozenset( } ) +_RESULT_DECIDES_STREAMING_CALL_TYPES: Final = frozenset({"get_responses", "aget_responses"}) + def _is_streaming_request( kwargs: dict[str, object], @@ -2589,7 +2591,7 @@ def _is_streaming_request( - if "stream=True" in kwargs (litellm chat completion, litellm text completion, litellm messages) - if call_type is generate_content_stream or agenerate_content_stream (litellm google genai) """ - if call_type in _NEVER_STREAMING_CALL_TYPES: + if call_type in _NEVER_STREAMING_CALL_TYPES or call_type in _RESULT_DECIDES_STREAMING_CALL_TYPES: return False if "stream" in kwargs and kwargs["stream"] is True: return True @@ -2682,9 +2684,99 @@ def huggingface_tokenizer_kind(model: str) -> HuggingFaceTokenizerKind | None: return None +@dataclass(frozen=True, slots=True) +class _HuggingFaceTokenizerLoaded: + tokenizer: HuggingFace + + @dataclass(frozen=True, slots=True) class _HuggingFaceTokenizerLoadFailure: reason: str + retry_at: float + retry_pid: int | None = None + + +_HuggingFaceTokenizerKey: TypeAlias = tuple[HuggingFaceTokenizerKind, Decision] +_HuggingFaceTokenizerLoadOutcome: TypeAlias = _HuggingFaceTokenizerLoaded | _HuggingFaceTokenizerLoadFailure + + +def _start_in_thread_joined_at_exit(work: Callable[[], None]) -> None: + threading.Thread(target=work, name="litellm-huggingface-tokenizer-retry", daemon=False).start() + + +@dataclass(slots=True) +class _HuggingFaceTokenizerLoadOutcomes: + clock: Callable[[], float] = time.monotonic + start_retry: Callable[[Callable[[], None]], None] = _start_in_thread_joined_at_exit + pid: Callable[[], int] = os.getpid + lock: threading.Lock = field(default_factory=threading.Lock) + by_tokenizer: Mapping[_HuggingFaceTokenizerKey, _HuggingFaceTokenizerLoadOutcome] = field( + default_factory=lambda: MappingProxyType({}) + ) + + def known( + self, key: _HuggingFaceTokenizerKey, retry_when_due: Callable[[], None] + ) -> _HuggingFaceTokenizerLoadOutcome | None: + with self.lock: + known: Final = self.by_tokenizer.get(key) + if ( + not isinstance(known, _HuggingFaceTokenizerLoadFailure) + or known.retry_pid == self.pid() + or self.clock() < known.retry_at + ): + return known + retrying: Final = replace(known, retry_pid=self.pid()) + self._remember(key, retrying) + try: + self.start_retry(retry_when_due) + except RuntimeError as error: + return self.remember_failed_retry(key, str(error)) + return retrying + + def remember_loaded(self, key: _HuggingFaceTokenizerKey, tokenizer: HuggingFace) -> _HuggingFaceTokenizerLoaded: + loaded: Final = _HuggingFaceTokenizerLoaded(tokenizer) + with self.lock: + self._remember(key, loaded) + return loaded + + def remember_first_failure(self, key: _HuggingFaceTokenizerKey, reason: str) -> _HuggingFaceTokenizerLoadOutcome: + with self.lock: + known: Final = self.by_tokenizer.get(key) + if known is not None: + return known + failure: Final = self._failure(reason) + self._remember(key, failure) + return failure + + def remember_failed_retry(self, key: _HuggingFaceTokenizerKey, reason: str) -> _HuggingFaceTokenizerLoadOutcome: + with self.lock: + known: Final = self.by_tokenizer.get(key) + if isinstance(known, _HuggingFaceTokenizerLoaded): + return known + failure: Final = self._failure(reason) + self._remember(key, failure) + return failure + + def end_retry(self, key: _HuggingFaceTokenizerKey) -> None: + with self.lock: + known: Final = self.by_tokenizer.get(key) + if isinstance(known, _HuggingFaceTokenizerLoadFailure) and known.retry_pid is not None: + self._remember(key, self._failure(known.reason)) + + def flush_cache(self) -> None: + with self.lock: + self.by_tokenizer = MappingProxyType({}) + + def _failure(self, reason: str) -> _HuggingFaceTokenizerLoadFailure: + return _HuggingFaceTokenizerLoadFailure( + reason=reason, retry_at=self.clock() + HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS + ) + + def _remember(self, key: _HuggingFaceTokenizerKey, outcome: _HuggingFaceTokenizerLoadOutcome) -> None: + self.by_tokenizer = MappingProxyType({**self.by_tokenizer, key: outcome}) + + +_HUGGINGFACE_TOKENIZER_LOAD_OUTCOMES: Final = _HuggingFaceTokenizerLoadOutcomes() def _warn_huggingface_tokenizer_fallback(model: str, reason: str) -> None: @@ -2696,30 +2788,28 @@ def _warn_huggingface_tokenizer_fallback(model: str, reason: str) -> None: ) -def _return_huggingface_tokenizer(model: str) -> SelectTokenizerResponse | _HuggingFaceTokenizerLoadFailure | None: +def _return_huggingface_tokenizer( + model: str, outcomes: _HuggingFaceTokenizerLoadOutcomes = _HUGGINGFACE_TOKENIZER_LOAD_OUTCOMES +) -> SelectTokenizerResponse | _HuggingFaceTokenizerLoadFailure | None: kind: Final = huggingface_tokenizer_kind(model) if kind is None or (kind not in _BUNDLED_TOKENIZER_LOAD_LOCKS and is_offline_mode()): return None - backend: Final = _huggingface_tokenizer_backend() - key: Final = _huggingface_tokenizer_failure_key(kind, backend) + key: Final = (kind, _huggingface_tokenizer_backend()) with _BUNDLED_TOKENIZER_LOAD_LOCKS.get(kind, nullcontext()): - cached_failure: Final = _HUGGINGFACE_TOKENIZER_LOAD_FAILURES.get_cache(key) - if isinstance(cached_failure, _HuggingFaceTokenizerLoadFailure): - return cached_failure - tokenizer: Final = _load_huggingface_tokenizer_recording_failure(kind, backend, key) - if isinstance(tokenizer, _HuggingFaceTokenizerLoadFailure): - _warn_huggingface_tokenizer_fallback(model, tokenizer.reason) - return tokenizer - return {"type": "huggingface_tokenizer", "tokenizer": tokenizer} + known: Final = outcomes.known(key, partial(_retry_huggingface_tokenizer_load, model, key, outcomes)) + outcome: Final = ( + known + if known is not None + else _load_huggingface_tokenizer_remembering_outcome(model, key, outcomes, outcomes.remember_first_failure) + ) + match outcome: + case _HuggingFaceTokenizerLoaded(tokenizer=tokenizer): + return {"type": "huggingface_tokenizer", "tokenizer": tokenizer} + case _HuggingFaceTokenizerLoadFailure(): + return outcome -def _huggingface_tokenizer_failure_key(kind: HuggingFaceTokenizerKind, backend: Decision) -> str: - return f"{kind}:{backend.name}" - - -@lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE) -def _load_huggingface_tokenizer(kind: HuggingFaceTokenizerKind, backend: Decision) -> HuggingFace: - """One tokenizer per kind and backend; `backend` is the cache key, the dispatch re-derives it.""" +def _load_huggingface_tokenizer(kind: HuggingFaceTokenizerKind) -> HuggingFace: match kind: case "cohere": return tokenizer_dispatch.from_pretrained("Xenova/c4ai-command-r-v01-tokenizer") @@ -2739,19 +2829,34 @@ def _load_huggingface_tokenizer(kind: HuggingFaceTokenizerKind, backend: Decisio assert_never(kind) -def _load_huggingface_tokenizer_recording_failure( - kind: HuggingFaceTokenizerKind, backend: Decision, key: str -) -> HuggingFace | _HuggingFaceTokenizerLoadFailure: +def _load_huggingface_tokenizer_remembering_outcome( + model: str, + key: _HuggingFaceTokenizerKey, + outcomes: _HuggingFaceTokenizerLoadOutcomes, + remember_failure: Callable[[_HuggingFaceTokenizerKey, str], _HuggingFaceTokenizerLoadOutcome], +) -> _HuggingFaceTokenizerLoadOutcome: + kind, _ = key try: - return _load_huggingface_tokenizer(kind, backend) + tokenizer: Final = _load_huggingface_tokenizer(kind) except Exception as error: from litellm.rust_bridge.fork_guard import ForkedAfterNativeRuntimeStarted, ProcessReservedForForking if isinstance(error, (ForkedAfterNativeRuntimeStarted, ProcessReservedForForking)): raise - failure: Final = _HuggingFaceTokenizerLoadFailure(reason=str(error)) - _HUGGINGFACE_TOKENIZER_LOAD_FAILURES.set_cache(key, failure, ttl=HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS) - return failure + outcome: Final = remember_failure(key, str(error)) + if isinstance(outcome, _HuggingFaceTokenizerLoadFailure): + _warn_huggingface_tokenizer_fallback(model, outcome.reason) + return outcome + return outcomes.remember_loaded(key, tokenizer) + + +def _retry_huggingface_tokenizer_load( + model: str, key: _HuggingFaceTokenizerKey, outcomes: _HuggingFaceTokenizerLoadOutcomes +) -> None: + try: + _load_huggingface_tokenizer_remembering_outcome(model, key, outcomes, outcomes.remember_failed_retry) + finally: + outcomes.end_retry(key) def encode(model="", text="", custom_tokenizer: dict | None = None): @@ -3397,6 +3502,42 @@ def supports_mid_conversation_system(model: str, custom_llm_provider: str | None ) +class _DeploymentModelInfo(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + + model_info: Mapping[str, object] | None = None + + +def deployment_model_info(litellm_params: object) -> Mapping[str, object] | None: + """The deployment's own ``model_info`` read off its litellm params.""" + return _DeploymentModelInfo.model_validate(litellm_params).model_info + + +def deployment_declared_mid_conversation_system_support(model_info: object) -> bool | None: + match model_info: + case {"supports_mid_conversation_system": bool(declared)}: + return declared + case _: + return None + + +def declared_mid_conversation_system_support( + model: str, custom_llm_provider: str | None, model_info: object +) -> bool | None: + """ + The `supports_mid_conversation_system` the deployment's own model_info declares, else the one the + cost map declares for the model, else None when neither sets it. + """ + deployment_declared: Final = deployment_declared_mid_conversation_system_support(model_info) + if deployment_declared is not None: + return deployment_declared + if is_explicitly_disabled_factory(model, custom_llm_provider, "supports_mid_conversation_system"): + return False + if supports_mid_conversation_system(model, custom_llm_provider): + return True + return None + + def supports_native_structured_output(model: str, custom_llm_provider: str | None = None) -> bool: """ Check if the given model supports native structured outputs and return a boolean value. @@ -3711,7 +3852,7 @@ def _resolve_builtin_model_cost_entry(key: str, provider: str) -> _BuiltinCostMa return None -def is_generalized_model_info(model_info: ModelInfo) -> bool: +def is_generalized_model_info(model_info: ModelInfoBase) -> bool: """Whether ``model_info`` came from a fallback-generalization capability rule. Detected as the resolved key missing ``litellm.model_cost`` while matching a @@ -3727,10 +3868,12 @@ def is_generalized_model_info(model_info: ModelInfo) -> bool: _REGISTRATION_ATTEMPTS: Final = 3 +_ModelInfoT: Final = TypeVar("_ModelInfoT", bound=ModelInfoBase) -def lookup_model_info_for_registration(model: str, custom_llm_provider: str | None) -> ModelInfo | ModelInfoLookupMiss: + +def _lookup_mapped_model_info(lookup: Callable[[], _ModelInfoT]) -> _ModelInfoT | ModelInfoLookupMiss: try: - info: Final = get_model_info(model=model, custom_llm_provider=custom_llm_provider) + info: Final = lookup() except ModelNotMappedError: return ModelInfoLookupMiss(tripped=False) except Exception: @@ -3738,6 +3881,16 @@ def lookup_model_info_for_registration(model: str, custom_llm_provider: str | No return ModelInfoLookupMiss(tripped=False) if is_generalized_model_info(info) else info +def lookup_model_info_for_registration(model: str, custom_llm_provider: str | None) -> ModelInfo | ModelInfoLookupMiss: + return _lookup_mapped_model_info(lambda: get_model_info(model=model, custom_llm_provider=custom_llm_provider)) + + +def lookup_catalog_model_info(model: str, custom_llm_provider: str | None) -> ModelInfoBase | ModelInfoLookupMiss: + return _lookup_mapped_model_info( + lambda: get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider, catalog_only=True) + ) + + def _get_builtin_model_info_for_registration( model: str, custom_llm_provider: str | None ) -> ModelInfo | ModelInfoLookupMiss: @@ -5003,6 +5156,46 @@ def _declared_reasoning_efforts(model_info: Mapping[str, object] | None) -> tupl return deployment_reasoning_effort_levels(model_info) or () +_DROPPED_REASONING_PARAMS: Final = ("reasoning_effort", "thinking") + + +def _turns_reasoning_off(value: object) -> bool: + match value: + case "none" | {"effort": "none"} | {"type": "disabled"}: + return True + case _: + return False + + +def is_known_non_reasoning_model(model: str, custom_llm_provider: str | None) -> bool: + info: Final = lookup_catalog_model_info(model, custom_llm_provider) + if isinstance(info, ModelInfoLookupMiss) or info.get("supports_reasoning") is True: + return False + return info.get("supports_reasoning") is False or info["key"] in GetModelCostMap.loaded_model_cost_map() + + +def has_catalog_entry(model: str, custom_llm_provider: str | None) -> bool: + info: Final = lookup_catalog_model_info(model, custom_llm_provider) + return not isinstance(info, ModelInfoLookupMiss) and info["key"] in GetModelCostMap.loaded_model_cost_map() + + +def _dropped_reasoning_warning(model: str, custom_llm_provider: str | None) -> str: + return "".join( + ( + "Dropping %s=%r for model=%r (drop_params=True) because %s does not support %s for it.", + ( + "" + if has_catalog_entry(model, custom_llm_provider) + else " If the deployment is a reasoning model, set model_info.base_model to its underlying model." + ), + ) + ) + + +def _warns_on_dropped_reasoning(value: object, model: str, custom_llm_provider: str | None) -> bool: + return not (_turns_reasoning_off(value) and is_known_non_reasoning_model(model, custom_llm_provider)) + + def get_optional_params( # use the openai defaults # https://platform.openai.com/docs/api-reference/chat/create @@ -5092,19 +5285,30 @@ def get_optional_params( verbose_logger.debug( "\nLiteLLM: Non-Default params passed to completion() %s", _debug_params_repr(non_default_params) ) - unsupported_params: Final = {} - for k in non_default_params: - if k not in supported_params: - if k in PROVIDER_UNVALIDATED_PARAMS: - continue - if k == "n" and n == 1: # langchain sends n=1 as a default value - continue # skip this param - unsupported_params[k] = non_default_params[k] + unsupported_params: Final[MappingProxyType[str, object]] = MappingProxyType( + { + k: non_default_params[k] + for k in non_default_params + if k not in supported_params and k not in PROVIDER_UNVALIDATED_PARAMS and not (k == "n" and n == 1) + } + ) if unsupported_params: if litellm.drop_params is True or (drop_params is not None and drop_params is True): for k in unsupported_params: non_default_params.pop(k, None) + for reasoning_param in _DROPPED_REASONING_PARAMS: + if reasoning_param in unsupported_params and _warns_on_dropped_reasoning( + unsupported_params[reasoning_param], base_model or model, custom_llm_provider + ): + verbose_logger.warning( + _dropped_reasoning_warning(base_model or model, custom_llm_provider), + reasoning_param, + unsupported_params[reasoning_param], + model, + custom_llm_provider, + reasoning_param, + ) else: raise UnsupportedParamsError( status_code=500, @@ -5674,6 +5878,7 @@ def add_provider_specific_params_to_optional_params( "openai", "azure", "text-completion-openai", + "bigmodel", *litellm.openai_compatible_providers, ] or JSONProviderRegistry.exists(custom_llm_provider) if calls_through_openai_sdk: @@ -6619,6 +6824,8 @@ def get_model_info_helper( custom_llm_provider: str | None = None, api_base: str | None = None, api_key: str | None = None, + *, + catalog_only: bool = False, ) -> ModelInfoBase: """ Helper for 'get_model_info'. Separated out to avoid infinite loop caused by returning 'supported_openai_param's @@ -6652,7 +6859,7 @@ def get_model_info_helper( model_cost_custom_llm_provider: Final = custom_llm_provider ######################### provider_config: BaseLLMModelInfo | None = None - if custom_llm_provider and custom_llm_provider in LlmProvidersSet: + if not catalog_only and custom_llm_provider and custom_llm_provider in LlmProvidersSet: provider_config = ProviderConfigManager.get_provider_model_info( model=model, provider=LlmProviders(custom_llm_provider) ) @@ -6676,7 +6883,7 @@ def get_model_info_helper( e, ) - if custom_llm_provider == "huggingface": + if not catalog_only and custom_llm_provider == "huggingface": max_tokens: Final = _get_max_position_embeddings(model_name=model) return ModelInfoBase( key=model, @@ -8344,26 +8551,6 @@ def trim_messages( from litellm.caching.in_memory_cache import InMemoryCache -class _HuggingFaceTokenizerLoadFailureCache(Protocol): - def get_cache(self, key: str) -> object | None: ... - - def set_cache(self, key: str, value: object, ttl: float | None = None) -> None: ... - - def flush_cache(self) -> None: ... - - -_HUGGINGFACE_TOKENIZER_LOAD_FAILURES: Final[_HuggingFaceTokenizerLoadFailureCache] = ( - cast( # cast-ok: cache methods are untyped - _HuggingFaceTokenizerLoadFailureCache, - InMemoryCache( - max_size_in_memory=16, - default_ttl=HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS, - clock=lambda: time.monotonic(), - ), - ) -) - - class AvailableModelsCache(InMemoryCache): def __init__(self, ttl_seconds: int = 300, max_size: int = 1000): super().__init__(ttl_seconds, max_size) @@ -9183,6 +9370,7 @@ class ProviderConfigManager: LlmProviders.DATABRICKS: (lambda: litellm.DatabricksConfig(), False), LlmProviders.XAI: (lambda: litellm.XAIChatConfig(), False), LlmProviders.ZAI: (lambda: litellm.ZAIChatConfig(), False), + LlmProviders.BIGMODEL: (ProviderConfigManager._get_bigmodel_chat_config, False), LlmProviders.OPENCODE_GO: (ProviderConfigManager.get_opencode_go_config, True), LlmProviders.LAMBDA_AI: (lambda: litellm.LambdaAIChatConfig(), False), LlmProviders.INCEPTION: (lambda: litellm.InceptionChatConfig(), False), @@ -9375,6 +9563,12 @@ class ProviderConfigManager: return get_bedrock_chat_config(model=model) + @staticmethod + def _get_bigmodel_chat_config() -> BaseConfig: + from litellm.llms.bigmodel.chat.transformation import BigModelChatConfig + + return BigModelChatConfig() + @staticmethod def get_opencode_go_config(model: str) -> BaseConfig: from litellm.llms.opencode.chat.transformation import go_chat_config @@ -9696,6 +9890,10 @@ class ProviderConfigManager: model: str, provider: LlmProviders, ) -> BaseAnthropicMessagesConfig | None: + if provider == LlmProviders.BIGMODEL: + from litellm.llms.bigmodel.messages.transformation import BigModelMessagesConfig + + return BigModelMessagesConfig() if provider == LlmProviders.OPENCODE_GO: from litellm.llms.opencode.common_utils import go_endpoint from litellm.llms.opencode.messages.transformation import OpenCodeGoMessagesConfig @@ -9927,6 +10125,10 @@ class ProviderConfigManager: if provider is None: return None + if provider == LlmProviders.BIGMODEL: + from litellm.llms.bigmodel.responses.transformation import BigModelResponsesAPIConfig + + return BigModelResponsesAPIConfig() if provider == LlmProviders.OPENCODE_GO: from litellm.llms.opencode.common_utils import go_endpoint from litellm.llms.opencode.responses.transformation import OpenCodeGoResponsesConfig @@ -10258,6 +10460,22 @@ class ProviderConfigManager: return BedrockVectorStore.get_initialized_custom_logger() return None + @staticmethod + def _get_vertex_ai_vector_stores_config(api_type: str | None) -> BaseVectorStoreConfig | None: + if api_type == "rag_api" or api_type is None: + from litellm.llms.vertex_ai.vector_stores.rag_api.transformation import ( + VertexVectorStoreConfig, + ) + + return VertexVectorStoreConfig() + if api_type == "search_api": + from litellm.llms.vertex_ai.vector_stores.search_api.transformation import ( + VertexSearchAPIVectorStoreConfig, + ) + + return VertexSearchAPIVectorStoreConfig() + return None + @staticmethod def get_provider_vector_stores_config( provider: LlmProviders, @@ -10279,18 +10497,7 @@ class ProviderConfigManager: return AzureOpenAIVectorStoreConfig() elif litellm.LlmProviders.VERTEX_AI == provider: - if api_type == "rag_api" or api_type is None: # default to rag_api - from litellm.llms.vertex_ai.vector_stores.rag_api.transformation import ( - VertexVectorStoreConfig, - ) - - return VertexVectorStoreConfig() - elif api_type == "search_api": - from litellm.llms.vertex_ai.vector_stores.search_api.transformation import ( - VertexSearchAPIVectorStoreConfig, - ) - - return VertexSearchAPIVectorStoreConfig() + return ProviderConfigManager._get_vertex_ai_vector_stores_config(api_type) elif litellm.LlmProviders.BEDROCK == provider: from litellm.llms.bedrock.vector_stores.transformation import ( BedrockVectorStoreConfig, @@ -10345,6 +10552,12 @@ class ProviderConfigManager: ) return MongoDBVectorStoreConfig() + elif litellm.LlmProviders.OPENSEARCH == provider: + from litellm.llms.opensearch.vector_stores.transformation import ( + OpenSearchVectorStoreConfig, + ) + + return OpenSearchVectorStoreConfig() return None @staticmethod @@ -10557,6 +10770,7 @@ class ProviderConfigManager: model: str, provider: LlmProviders, drop_params: bool = False, + litellm_params: Mapping[str, object] = MappingProxyType({}), ) -> BaseRealtimeConfig | None: if LlmProviders.GEMINI == provider: from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig @@ -10566,6 +10780,10 @@ class ProviderConfigManager: from litellm.llms.meta.realtime.transformation import MetaRealtimeConfig return MetaRealtimeConfig(drop_params=drop_params) + if LlmProviders.SONIOX == provider: + from litellm.llms.soniox.realtime.transformation import SonioxRealtimeConfig + + return SonioxRealtimeConfig.from_litellm_params(litellm_params, drop_params=drop_params) return None @staticmethod diff --git a/migrations/Dockerfile b/migrations/Dockerfile index 2b1fd029143..3ffa9140543 100644 --- a/migrations/Dockerfile +++ b/migrations/Dockerfile @@ -43,6 +43,7 @@ ENV UV_PYTHON_DOWNLOADS=0 COPY pyproject.toml README.md rust-toolchain.toml ./ COPY .cargo ./.cargo COPY litellm-rust ./litellm-rust +COPY model_prices_and_context_window.json ./ RUN --mount=type=cache,target=/root/.cache/uv \ mkdir -p litellm/rust_bridge /wheels && \ diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3838f7510f0..c6bbb63b6fc 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -4049,6 +4049,7 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_pdf_input": false, "supports_response_schema": true, "supports_tool_choice": true }, @@ -12437,7 +12438,8 @@ "supported_output_modalities": [ "text" ], - "supports_native_streaming": true + "supports_native_streaming": true, + "supports_pdf_input": false }, "azure_ai/FLUX-1.1-pro": { "deprecation_date": "2027-01-05", @@ -12502,6 +12504,7 @@ "output_cost_per_token": 1.85e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12516,6 +12519,7 @@ "output_cost_per_token": 3.828e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12531,6 +12535,7 @@ "output_cost_per_token": 3.52e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12546,6 +12551,7 @@ "output_cost_per_token": 4.84e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12560,6 +12566,7 @@ "output_cost_per_token": 4.84e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12574,6 +12581,7 @@ "output_cost_per_token": 7.26e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12594,6 +12602,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12616,6 +12625,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -12638,6 +12648,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -12660,6 +12671,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -12687,6 +12699,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -12703,6 +12716,7 @@ "output_cost_per_token": 1.32e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12724,6 +12738,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -12744,6 +12759,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -12765,6 +12781,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12833,6 +12850,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12847,6 +12865,7 @@ "output_cost_per_token": 7.1e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/Llama-4-Maverick-17B-128E-Instruct-FP8": { @@ -12859,6 +12878,7 @@ "output_cost_per_token": 1e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": true }, @@ -12872,6 +12892,7 @@ "output_cost_per_token": 7.8e-07, "source": "https://azure.microsoft.com/en-us/blog/introducing-the-llama-4-herd-in-azure-ai-foundry-and-azure-databricks/", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": true }, @@ -12884,6 +12905,7 @@ "mode": "chat", "output_cost_per_token": 3.7e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/Meta-Llama-3.1-70B-Instruct": { @@ -12895,6 +12917,7 @@ "mode": "chat", "output_cost_per_token": 3.54e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/Phi-3-medium-128k-instruct": { @@ -12906,6 +12929,7 @@ "mode": "chat", "output_cost_per_token": 6.8e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -12918,6 +12942,7 @@ "mode": "chat", "output_cost_per_token": 6.8e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -12930,6 +12955,7 @@ "mode": "chat", "output_cost_per_token": 5.2e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -12942,6 +12968,7 @@ "mode": "chat", "output_cost_per_token": 5.2e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -12954,6 +12981,7 @@ "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -12966,6 +12994,7 @@ "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -12978,6 +13007,7 @@ "mode": "chat", "output_cost_per_token": 6.4e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -12990,6 +13020,7 @@ "mode": "chat", "output_cost_per_token": 5.2e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -13002,6 +13033,7 @@ "mode": "chat", "output_cost_per_token": 5.2e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": true }, @@ -13015,6 +13047,7 @@ "output_cost_per_token": 5e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -13027,7 +13060,8 @@ "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true + "supports_function_calling": true, + "supports_pdf_input": false }, "azure_ai/Phi-4-multimodal-instruct": { "deprecation_date": "2027-01-17", @@ -13041,6 +13075,7 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_audio_input": true, "supports_function_calling": true, + "supports_pdf_input": false, "supports_vision": true }, "azure_ai/Phi-4-mini-reasoning": { @@ -13052,7 +13087,8 @@ "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true + "supports_function_calling": true, + "supports_pdf_input": false }, "azure_ai/Phi-4-reasoning": { "deprecation_date": "2027-01-17", @@ -13064,6 +13100,7 @@ "output_cost_per_token": 5e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true, "supports_reasoning": true }, @@ -13106,6 +13143,7 @@ "output_cost_per_token": 1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/doc-intelligence/prebuilt-read": { @@ -13144,6 +13182,7 @@ "mode": "chat", "output_cost_per_token": 5.4e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_reasoning": true, "supports_tool_choice": true }, @@ -13202,6 +13241,7 @@ "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -13217,6 +13257,7 @@ "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -13230,6 +13271,7 @@ "mode": "chat", "output_cost_per_token": 4.56e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/deepseek-v4-pro": { @@ -13242,6 +13284,7 @@ "output_cost_per_token": 3.48e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_reasoning": true, "supports_tool_choice": true, "cache_read_input_token_cost": 1.45e-07, @@ -13257,6 +13300,7 @@ "output_cost_per_token": 5.1e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_reasoning": true, "supports_tool_choice": true, "cache_read_input_token_cost": 2.8e-08, @@ -13273,6 +13317,7 @@ "output_cost_per_token": 1.32e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -13306,6 +13351,7 @@ "output_cost_per_token": 1.5e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", "supports_function_calling": true, + "supports_pdf_input": false, "supports_response_schema": true, "supports_tool_choice": true, "supports_web_search": true @@ -13323,6 +13369,7 @@ "output_cost_per_token_above_200k_tokens": 5e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -13343,6 +13390,7 @@ "output_cost_per_token_above_200k_tokens": 1.2e-05, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/grok/", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -13361,6 +13409,7 @@ "output_cost_per_token": 2.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -13379,6 +13428,7 @@ "output_cost_per_token": 2.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -13395,6 +13445,7 @@ "mode": "chat", "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_response_schema": true, "supports_tool_choice": true, "supports_web_search": true @@ -13409,6 +13460,7 @@ "mode": "chat", "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -13424,6 +13476,7 @@ "output_cost_per_token": 1.5e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", "supports_function_calling": true, + "supports_pdf_input": false, "supports_response_schema": true, "supports_tool_choice": true, "supports_web_search": true @@ -13436,7 +13489,8 @@ "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.00971, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models" + "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false }, "azure_ai/jamba-instruct": { "deprecation_date": "2025-03-01", @@ -13447,6 +13501,7 @@ "mode": "chat", "output_cost_per_token": 7e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/kimi-k2.5": { @@ -13459,6 +13514,7 @@ "output_cost_per_token": 3e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true, @@ -13482,6 +13538,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_reasoning": true, "supports_tool_choice": true, "supports_vision": true, @@ -13498,6 +13555,7 @@ "output_cost_per_token": 4e-08, "source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.ministral-3b-2410-offer?tab=Overview", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/mistral-large": { @@ -13510,6 +13568,7 @@ "output_cost_per_token": 1.2e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/mistral-large-2407": { @@ -13522,6 +13581,7 @@ "output_cost_per_token": 6e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/mistral-large-latest": { @@ -13533,6 +13593,7 @@ "output_cost_per_token": 6e-06, "source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.mistral-ai-large-2407-offer?tab=Overview", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/mistral-large-3": { @@ -13544,6 +13605,7 @@ "output_cost_per_token": 1.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": true }, @@ -13557,6 +13619,7 @@ "output_cost_per_token": 2e-06, "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/mistral-nemo": { @@ -13568,7 +13631,8 @@ "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_function_calling": true + "supports_function_calling": true, + "supports_pdf_input": false }, "azure_ai/mistral-small": { "deprecation_date": "2025-07-31", @@ -13580,6 +13644,7 @@ "output_cost_per_token": 3e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/mistral-small-2503": { @@ -13591,6 +13656,7 @@ "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": true }, @@ -52006,6 +52072,7 @@ ], "supports_assistant_prefill": false, "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -52026,6 +52093,7 @@ ], "supports_assistant_prefill": false, "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -52777,6 +52845,7 @@ "output_cost_per_token": 2.5e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true, "supports_web_search": true }, @@ -52794,6 +52863,7 @@ "global" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_reasoning": true, "supports_tool_choice": true }, @@ -52811,6 +52881,7 @@ "global" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -52989,6 +53060,7 @@ "output_cost_per_token": 3.6e-07, "output_cost_per_token_batches": 1.8e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supports_pdf_input": false, "supports_reasoning": true }, "vertex_ai/openai/gpt-oss-20b-maas": { @@ -53000,6 +53072,7 @@ "mode": "chat", "output_cost_per_token": 2.5e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supports_pdf_input": false, "supports_reasoning": true, "cache_read_input_token_cost": 7e-09, "input_cost_per_token_batches": 3.5e-08, @@ -53131,6 +53204,7 @@ "us-south1" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas": { @@ -53149,6 +53223,7 @@ "global" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "vertex_ai/qwen/qwen3-next-80b-a3b-instruct-maas": { @@ -53164,6 +53239,7 @@ "global" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "vertex_ai/qwen/qwen3-next-80b-a3b-thinking-maas": { @@ -53179,6 +53255,7 @@ "global" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "vertex_ai/veo-3.1-generate-preview": { @@ -61177,6 +61254,126 @@ ], "supports_audio_input": true }, + "soniox/stt-rt-v5": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "soniox/stt-rt-preview": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "soniox/stt-rt-preview-v2": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "soniox/stt-rt-v3": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "soniox/stt-rt-v3-preview": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "soniox/stt-rt-v4": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, "tensormesh/Qwen/Qwen3.5-397B-A17B-FP8": { "litellm_provider": "tensormesh", "mode": "chat", @@ -63655,6 +63852,7 @@ "output_cost_per_token_priority": 9e-06, "source": "https://api.fireworks.ai/v1/serverless/models", "supports_function_calling": true, + "supports_mid_conversation_system": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -63801,6 +63999,7 @@ "output_cost_per_token_priority": 9e-06, "source": "https://api.fireworks.ai/v1/serverless/models", "supports_function_calling": true, + "supports_mid_conversation_system": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -67200,6 +67399,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -67214,6 +67414,7 @@ "mode": "chat", "output_cost_per_token": 1.2e-06, "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -67256,6 +67457,7 @@ "output_cost_per_token": 1.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -67270,6 +67472,7 @@ "output_cost_per_token": 3.1e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -67284,6 +67487,7 @@ "output_cost_per_token": 5.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -67298,6 +67502,7 @@ "output_cost_per_token": 6.25e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -67314,6 +67519,7 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -67329,6 +67535,7 @@ "output_cost_per_token": 3.2e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_reasoning": true, "supports_tool_choice": true }, @@ -67347,6 +67554,7 @@ "supported_output_modalities": [ "text" ], + "supports_pdf_input": false, "supports_response_schema": true, "supports_vision": true }, @@ -67356,7 +67564,8 @@ "output_cost_per_token": 5.4e-06, "litellm_provider": "azure_ai", "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_pdf_input": false }, "azure_ai/deepseek-v3-0324": { "deprecation_date": "2026-07-13", @@ -67364,7 +67573,8 @@ "output_cost_per_token": 4.56e-06, "litellm_provider": "azure_ai", "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_pdf_input": false }, "azure_ai/deepseek-v3.1": { "deprecation_date": "2026-07-13", @@ -67372,7 +67582,8 @@ "output_cost_per_token": 4.94e-06, "litellm_provider": "azure_ai", "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_pdf_input": false }, "azure_ai/grok-3": { "deprecation_date": "2026-05-01", @@ -67380,7 +67591,8 @@ "output_cost_per_token": 1.5e-05, "litellm_provider": "azure_ai", "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_pdf_input": false }, "azure_ai/grok-3-mini": { "deprecation_date": "2026-05-01", @@ -67388,7 +67600,8 @@ "output_cost_per_token": 1.27e-06, "litellm_provider": "azure_ai", "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_pdf_input": false }, "azure_ai/grok-4-fast-non-reasoning": { "deprecation_date": "2026-05-01", @@ -67396,7 +67609,8 @@ "output_cost_per_token": 5e-07, "litellm_provider": "azure_ai", "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_pdf_input": false }, "azure_ai/grok-4-fast-reasoning": { "deprecation_date": "2026-05-01", @@ -67404,7 +67618,8 @@ "output_cost_per_token": 5e-07, "litellm_provider": "azure_ai", "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_pdf_input": false }, "bedrock/us-gov-west-1/nvidia.nemotron-nano-3-30b": { "input_cost_per_token": 7.2e-08, @@ -73074,6 +73289,7 @@ "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_regions": ["global"], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -73689,6 +73905,7 @@ "output_cost_per_token": 3.5e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/microsoft/", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -74743,6 +74960,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": false, "supports_reasoning": true, "supports_tool_choice": true, @@ -74991,6 +75209,7 @@ "output_cost_per_token_above_200k_tokens": 1.2e-05, "source": "https://docs.x.ai/developers/models/grok-4.7", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -78339,6 +78558,1254 @@ "supports_xhigh_reasoning_effort": true, "use_openai_responses_path": true }, + "kyma/claude-fable-5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-05, + "output_cost_per_token": 6.75e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true + }, + "kyma/claude-fable-5.1": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-05, + "output_cost_per_token": 6.75e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true + }, + "kyma/claude-haiku-4-5": { + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "input_cost_per_token": 1.35e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 6.75e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false, + "prompt_cache_min_tokens": 4096 + }, + "kyma/claude-haiku-5.5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-07, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 6.75e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": false, + "prompt_cache_min_tokens": 512 + }, + "kyma/claude-opus-4-7": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 6.75e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 3.375e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 2048 + }, + "kyma/claude-opus-5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 6.75e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 3.375e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 512 + }, + "kyma/claude-opus-5.5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512 + }, + "kyma/claude-sonnet-4-6": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 4.05e-06, + "cache_read_input_token_cost": 4.05e-07, + "output_cost_per_token": 2.025e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 1024 + }, + "kyma/claude-sonnet-5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.35e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 1024 + }, + "kyma/claude-sonnet-5.5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 1.35e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512 + }, + "kyma/deepseek-r1": { + "max_input_tokens": 64000, + "max_output_tokens": 32768, + "input_cost_per_token": 7.425e-07, + "output_cost_per_token": 2.957e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v3": { + "max_input_tokens": 160000, + "max_output_tokens": 8192, + "input_cost_per_token": 5.18e-07, + "output_cost_per_token": 7.571e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v4-flash": { + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "input_cost_per_token": 1.23e-07, + "cache_read_input_token_cost": 2.43e-08, + "output_cost_per_token": 2.458e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v4-flash-vision-exp": { + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "input_cost_per_token": 3.892e-07, + "output_cost_per_token": 1.168e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v4.1-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 2.7e-07, + "output_cost_per_token": 8.1e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 6.75e-07, + "cache_read_input_token_cost": 6.75e-08, + "output_cost_per_token": 4.05e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3.1-pro": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.62e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemini-3.5-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 2.025e-06, + "cache_read_input_token_cost": 2.025e-07, + "output_cost_per_token": 1.215e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3.5-flash-lite": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 4.05e-07, + "cache_read_input_token_cost": 4.05e-08, + "output_cost_per_token": 3.375e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3.6-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.013e-06, + "cache_read_input_token_cost": 1.0125e-07, + "output_cost_per_token": 5.063e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": false, + "supports_pdf_input": false + }, + "kyma/gemini-3.7-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 2.026e-06, + "cache_read_input_token_cost": 1.0125e-07, + "output_cost_per_token": 1.0126e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemini-3.8-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 2.026e-06, + "output_cost_per_token": 1.0126e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemini-4-argon": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemma-4-31b": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.642e-07, + "cache_read_input_token_cost": 1.642e-07, + "output_cost_per_token": 4.689e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-4.5-air": { + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "input_cost_per_token": 1.797e-07, + "cache_read_input_token_cost": 3.375e-08, + "output_cost_per_token": 1.175e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-4.7-flash": { + "max_input_tokens": 203000, + "max_output_tokens": 65536, + "input_cost_per_token": 8.1e-08, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 5.4e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.1": { + "max_input_tokens": 203000, + "max_output_tokens": 65536, + "input_cost_per_token": 1.89e-06, + "cache_read_input_token_cost": 2.7675e-07, + "output_cost_per_token": 5.94e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.2": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 9.029e-07, + "cache_read_input_token_cost": 2.9835e-07, + "output_cost_per_token": 3.083e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.3": { + "max_input_tokens": 1310720, + "max_output_tokens": 131072, + "input_cost_per_token": 1.178e-06, + "output_cost_per_token": 3.702e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.3-flash": { + "max_input_tokens": 1310720, + "max_output_tokens": 65536, + "input_cost_per_token": 1.72e-07, + "output_cost_per_token": 5.733e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gpt-5.6-luna": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.944e-07, + "cache_read_input_token_cost": 2.7e-08, + "output_cost_per_token": 1.767e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-luna-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.703e-07, + "cache_read_input_token_cost": 2.7e-08, + "output_cost_per_token": 1.621e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-sol": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-sol-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-terra": { + "max_input_tokens": 1050000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.62e-05, + "input_cost_per_token_above_272k_tokens": 5.4e-06, + "output_cost_per_token_above_272k_tokens": 2.43e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gpt-5.6-terra-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.62e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-6-astra": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-05, + "cache_read_input_token_cost": 1.35e-06, + "output_cost_per_token": 6.75e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gpt-6-luna": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.639e-07, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 8.195e-07, + "input_cost_per_token_above_272k_tokens": 2.7e-07, + "output_cost_per_token_above_272k_tokens": 1.0125e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-6.1-sol": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 1.35e-05, + "input_cost_per_token_above_272k_tokens": 5.4e-06, + "output_cost_per_token_above_272k_tokens": 2.025e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-oss-120b": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 4.73e-08, + "output_cost_per_token": 2.303e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/grok-4.20": { + "max_input_tokens": 2000000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.763e-06, + "output_cost_per_token": 3.528e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.20-multi-agent": { + "max_input_tokens": 2000000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.767e-06, + "output_cost_per_token": 3.534e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": false, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.3": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 1.92e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 3.838e-06, + "input_cost_per_token_above_200k_tokens": 3.375e-06, + "output_cost_per_token_above_200k_tokens": 6.75e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/grok-4.5": { + "max_input_tokens": 500000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.73e-06, + "output_cost_per_token": 8.189e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.6": { + "max_input_tokens": 500000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.758e-06, + "output_cost_per_token": 8.274e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.7": { + "max_input_tokens": 500000, + "max_output_tokens": 450000, + "input_cost_per_token": 2.16e-06, + "cache_read_input_token_cost": 5.4e-07, + "output_cost_per_token": 6.48e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-build": { + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "input_cost_per_token": 1.389e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 2.777e-06, + "input_cost_per_token_above_200k_tokens": 2.7e-06, + "output_cost_per_token_above_200k_tokens": 5.4e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/hy3": { + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "input_cost_per_token": 1.594e-07, + "output_cost_per_token": 6.576e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/hy4-preview": { + "max_input_tokens": 1048576, + "max_output_tokens": 64000, + "input_cost_per_token": 1.1259e-06, + "output_cost_per_token": 3.3764e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k2.6": { + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "input_cost_per_token": 5.909e-07, + "output_cost_per_token": 2.758e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k2.7-code": { + "max_input_tokens": 262144, + "max_output_tokens": 16384, + "input_cost_per_token": 1.009e-06, + "output_cost_per_token": 4.774e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k3": { + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "input_cost_per_token": 3.052e-06, + "cache_read_input_token_cost": 4.05e-07, + "output_cost_per_token": 1.5258e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k3.1-pro": { + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.35e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/llama-3.3-70b": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.35e-07, + "output_cost_per_token": 4.32e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/llama-4-maverick": { + "max_input_tokens": 1048576, + "max_output_tokens": 16384, + "input_cost_per_token": 3.344e-07, + "output_cost_per_token": 1.337e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/mimo-v2.5": { + "max_input_tokens": 1050000, + "max_output_tokens": 16384, + "input_cost_per_token": 1.859e-07, + "output_cost_per_token": 3.718e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/mimo-v2.6-flash": { + "max_input_tokens": 1050000, + "max_output_tokens": 131072, + "input_cost_per_token": 1.89e-07, + "output_cost_per_token": 3.78e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/mimo-v2.6-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 131072, + "input_cost_per_token": 5.805e-07, + "output_cost_per_token": 1.1745e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m2.5": { + "max_input_tokens": 196608, + "max_output_tokens": 32768, + "input_cost_per_token": 4.188e-07, + "output_cost_per_token": 1.675e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m2.7": { + "max_input_tokens": 204800, + "max_output_tokens": 32768, + "input_cost_per_token": 4.05e-07, + "output_cost_per_token": 1.62e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m3": { + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "input_cost_per_token": 3.956e-07, + "cache_read_input_token_cost": 7.56e-08, + "output_cost_per_token": 1.582e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m3.1-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 524288, + "input_cost_per_token": 2.025e-07, + "cache_read_input_token_cost": 2.025e-08, + "output_cost_per_token": 8.1e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/muse-glimmer-30b": { + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "input_cost_per_token": 4.554e-07, + "output_cost_per_token": 1.821e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/muse-spark-1.1": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.542e-06, + "output_cost_per_token": 5.243e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/muse-spark-1.2": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.773e-06, + "output_cost_per_token": 6.025e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/muse-spark-1.3": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.688e-06, + "output_cost_per_token": 5.738e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/muse-spark-open": { + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "input_cost_per_token": 1.688e-06, + "output_cost_per_token": 5.738e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/muse-wonder": { + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "input_cost_per_token": 5.4e-06, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/nemotron-3-ultra-550b": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 6.75e-07, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 3.375e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/nemotron-3.5-super-vl": { + "max_input_tokens": 1048576, + "max_output_tokens": 235929, + "input_cost_per_token": 8.1e-08, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 4.725e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3-32b": { + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "input_cost_per_token": 1.08e-07, + "output_cost_per_token": 3.78e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3-coder": { + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "input_cost_per_token": 3.546e-07, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 1.612e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.6-plus": { + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "input_cost_per_token": 4.388e-07, + "output_cost_per_token": 2.633e-06, + "input_cost_per_token_above_256k_tokens": 1.755e-06, + "output_cost_per_token_above_256k_tokens": 5.265e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.7-max": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 2.071e-06, + "cache_read_input_token_cost": 1.6875e-07, + "output_cost_per_token": 6.207e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.7-plus": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 4.325e-07, + "cache_read_input_token_cost": 8.64e-08, + "output_cost_per_token": 1.731e-06, + "input_cost_per_token_above_256k_tokens": 1.296e-06, + "output_cost_per_token_above_256k_tokens": 5.184e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.8-max": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 2.228e-06, + "cache_read_input_token_cost": 2.781e-07, + "output_cost_per_token": 6.684e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen3.7-flash": { + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "input_cost_per_token": 4.35e-08, + "cache_read_input_token_cost": 8.1e-09, + "output_cost_per_token": 1.886e-07, + "input_cost_per_token_above_32k_tokens": 1.35e-07, + "output_cost_per_token_above_32k_tokens": 5.4e-07, + "input_cost_per_token_above_256k_tokens": 2.7e-07, + "output_cost_per_token_above_256k_tokens": 1.08e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen3.8-27b": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 3.826e-07, + "output_cost_per_token": 2.732e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen3.8-flash": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 2.045e-07, + "output_cost_per_token": 6.41e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/sonar": { + "max_input_tokens": 127072, + "max_output_tokens": 4096, + "input_cost_per_token": 1.35e-06, + "output_cost_per_token": 1.35e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": false, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_pdf_input": false + }, + "kyma/sonar-pro": { + "max_input_tokens": 200000, + "max_output_tokens": 8000, + "input_cost_per_token": 4.05e-06, + "output_cost_per_token": 2.025e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": false, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_pdf_input": false + }, + "kyma/step-3.7-flash": { + "max_input_tokens": 256000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.229e-07, + "cache_read_input_token_cost": 5.4e-08, + "output_cost_per_token": 1.282e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/step-5-preview": { + "max_input_tokens": 1048576, + "max_output_tokens": 64000, + "input_cost_per_token": 1.35e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 3.645e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, "databricks/databricks-claude-haiku-5-5": { "cache_creation_input_token_cost": 1.2502e-07, "cache_creation_input_token_cost_above_100k_tokens": 6.2503e-07, @@ -78995,5 +80462,81 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true + }, + "bigmodel/glm-5.3": { + "input_cost_per_token": 1.1428571428571428e-06, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 2.857142857142857e-07, + "litellm_provider": "bigmodel", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "source": "https://docs.bigmodel.cn/cn/guide/start/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "thinking_always_on": true, + "metadata": { + "comment": "Official API list price checked 2026-10-27: input 8, output 28, cached input 2 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded." + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_native_streaming": true + }, + "bigmodel/glm-5.3-flash": { + "input_cost_per_token": 1.142857142857143e-07, + "output_cost_per_token": 4e-07, + "cache_read_input_token_cost": 3.2857142857142855e-08, + "litellm_provider": "bigmodel", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "source": "https://docs.bigmodel.cn/cn/guide/start/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "thinking_always_on": true, + "supports_vision": true, + "supports_video_input": true, + "metadata": { + "comment": "Official API list price checked 2026-10-27: input 0.8, output 2.8, cached input 0.23 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded." + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_native_streaming": true + }, + "bigmodel/glm-5.3-flashx": { + "input_cost_per_token": 2.857142857142857e-07, + "output_cost_per_token": 1e-06, + "cache_read_input_token_cost": 8.142857142857142e-08, + "litellm_provider": "bigmodel", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "source": "https://docs.bigmodel.cn/cn/guide/start/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "thinking_always_on": true, + "supports_vision": true, + "supports_video_input": true, + "metadata": { + "comment": "Official API list price checked 2026-10-27: input 2, output 7, cached input 0.57 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded." + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_native_streaming": true } } diff --git a/packaging/litellm-core/pyproject.toml b/packaging/litellm-core/pyproject.toml index 27681d9116c..db50589a675 100644 --- a/packaging/litellm-core/pyproject.toml +++ b/packaging/litellm-core/pyproject.toml @@ -56,6 +56,7 @@ editable-profile = "dev" include = [ { path = "rust-toolchain.toml", format = "sdist" }, { path = ".cargo/config.toml", format = "sdist" }, + { path = "model_prices_and_context_window.json", format = "sdist" }, "litellm/router_strategy/complexity_router/artifacts/*.json", "litellm/router_strategy/complexity_router/fuse_presets.json", "litellm/proxy/model_insights_tasks.json", diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index e40cd134759..1d487f33835 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -3724,7 +3724,7 @@ "ocr": false, "search": false, "moderations": false, - "realtime": false, + "realtime": true, "video_generations": false, "batches": false, "messages": false, @@ -4910,6 +4910,33 @@ "vector_stores_search": true } }, + "opensearch": { + "display_name": "Amazon OpenSearch (`opensearch`)", + "url": "https://docs.litellm.ai/docs/providers/opensearch_vector_stores", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false, + "vector_stores_search": true + } + }, "helicone": { "display_name": "Helicone (`helicone`)", "url": "https://docs.litellm.ai/docs/providers/helicone", @@ -5326,6 +5353,58 @@ "a2a": false, "generateContent": false } + }, + "kyma": { + "display_name": "Kyma API (`kyma`)", + "url": "https://docs.litellm.ai/docs/providers/kyma", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, + "bigmodel": { + "display_name": "BigModel (Zhipu AI) (`bigmodel`)", + "url": "https://docs.bigmodel.cn/cn/guide/develop/responses/introduction", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } } }, "endpoints": { diff --git a/pyproject.toml b/pyproject.toml index 200bb463313..e6d34a3d90c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -335,6 +335,7 @@ include = [ "litellm/proxy/public_endpoints/whats_new.json", "litellm/proxy/common_utils/codex_base_instructions.md", "litellm/proxy/common_utils/codex_bundled_models_0.159.3.json", + { path = "model_prices_and_context_window.json", format = "sdist" }, ] exclude = [ "litellm/proxy/enterprise", @@ -357,6 +358,7 @@ cache-keys = [ { file = "litellm-rust/Cargo.lock" }, { file = "litellm-rust/Cargo.toml" }, { file = "litellm-rust/crates/**/*" }, + { file = "model_prices_and_context_window.json" }, ] constraint-dependencies = [ "tornado>=6.5.8", diff --git a/scripts/build_core_distribution.py b/scripts/build_core_distribution.py index 36ec20a9031..41d7208d2c3 100644 --- a/scripts/build_core_distribution.py +++ b/scripts/build_core_distribution.py @@ -9,7 +9,15 @@ from pathlib import Path from typing import Final ROOT: Final = Path(__file__).resolve().parents[1] -SOURCES: Final = ("litellm", "litellm-rust", ".cargo", "rust-toolchain.toml", "README.md", "LICENSE") +SOURCES: Final = ( + "litellm", + "litellm-rust", + ".cargo", + "rust-toolchain.toml", + "model_prices_and_context_window.json", + "README.md", + "LICENSE", +) def stage_core_distribution(source: Path, destination: Path) -> None: diff --git a/scripts/pre_commit_lint.sh b/scripts/pre_commit_lint.sh index 371884fb791..95b5942fa8b 100755 --- a/scripts/pre_commit_lint.sh +++ b/scripts/pre_commit_lint.sh @@ -18,7 +18,8 @@ # -> ruff over ruff-tests.toml + `make lint-test-quality` (test-linting.yml's # test-tree ruff and test-quality gate steps) # - dashboard -> prettier + eslint + lint budgets (test-linting.yml's ui job) -# - proxy/types -> regenerate the lazy OpenAPI snapshot and dashboard API types, fail on drift (test-linting.yml's ui-api-types job) +# - spec_pattern -> regenerate the lazy OpenAPI snapshot and dashboard API types, fail on drift +# (test-linting.yml's ui-api-types job) # # Each block is skipped when no matching files are in scope, so unrelated commits # stay fast. This is intentionally not auto-installed as a git hook (see @@ -98,7 +99,7 @@ existing_files() { litellm_py_pattern='^litellm/.*\.py$' e2e_py_pattern='^tests/e2e(_harness)?/.*\.py$' test_tree_pattern='^(tests/.*\.py|ruff-tests\.toml|scripts/(check_test_quality|test_quality_gate)\.py)$' -spec_pattern='^(litellm/(proxy|types)/.*|ui/litellm-dashboard/(scripts/gen-api-types\.mjs|package\.json|package-lock\.json|src/lib/http/schema\.d\.ts))$' +spec_pattern='^(litellm/.*|enterprise/.*|pyproject\.toml|uv\.lock|ui/litellm-dashboard/(scripts/gen-api-types\.mjs|package\.json|package-lock\.json|src/lib/http/schema\.d\.ts))$' ui_prettier_pattern='^ui/litellm-dashboard/.*\.(js|jsx|ts|tsx|mjs|cjs|json|css|scss|md|mdx|yml|yaml|html)$' ui_eslint_pattern='^ui/litellm-dashboard/.*\.(js|jsx|ts|tsx|mjs|cjs)$' @@ -110,9 +111,6 @@ e2e_py_files=$(scope_match "$e2e_py_pattern") test_tree_files=$(scope_match "$test_tree_pattern") # ruff format (and CI's format step) skip enterprise; the rest of make lint covers it. fmt_files=$(printf '%s\n' "$litellm_py_files" | grep -v '^litellm/enterprise/' | existing_files) -# test-linting.yml triggers on any file under litellm/proxy or litellm/types -# (Prisma schema and configs included, not just Python) plus the generator and its -# lockfiles, so match that whole trigger set rather than a Python subset. spec_files=$(scope_match "$spec_pattern") # CI's ui job runs prettier over a wider extension set than eslint; keep that # split so this flags exactly what the job would. @@ -337,7 +335,7 @@ summary_item "tests/e2e checks (basedpyright + raw HTTP client ban)" "$e2e_py_fi summary_item "test-tree lint (ruff-tests.toml + test-quality gate)" "$test_tree_files" \ "no tests/ Python files or test-tree lint inputs in scope" summary_item "dashboard lint (prettier + eslint + lint budgets)" "$ui_prettier_changed$ui_eslint_changed" "no dashboard files in scope" -summary_item "dashboard API-type sync (npm run gen:api)" "$spec_files" "no litellm/proxy, litellm/types, or generator files in scope" +summary_item "dashboard API-type sync (npm run gen:api)" "$spec_files" "no litellm/, enterprise/, dependency, or generator files in scope" if [ -z "$litellm_py_files$e2e_py_files$test_tree_files$ui_prettier_changed$ui_eslint_changed$spec_files" ]; then echo "check: NOTE - no gating lint check matches the files in scope, so nothing ran:" >&2 diff --git a/tests/base_sdk_tests/test_core_distribution.py b/tests/base_sdk_tests/test_core_distribution.py index ba68d08e4d1..c22fa4e54fc 100644 --- a/tests/base_sdk_tests/test_core_distribution.py +++ b/tests/base_sdk_tests/test_core_distribution.py @@ -132,7 +132,9 @@ def test_core_artifacts_exclude_local_configuration(distributions: tuple[Path, P assert not any(member.name.endswith(f"/{name}") for member in archive.getmembers()) -@pytest.mark.parametrize("relative_path", ["rust-toolchain.toml", ".cargo/config.toml"]) +@pytest.mark.parametrize( + "relative_path", ["rust-toolchain.toml", ".cargo/config.toml", "model_prices_and_context_window.json"] +) def test_core_sdist_preserves_native_build_configuration(distribution_directory: Path, relative_path: str) -> None: sdists: Final = tuple(distribution_directory.glob("litellm_core-*.tar.gz")) assert len(sdists) == 1 diff --git a/tests/integration/AGENTS.md b/tests/integration/AGENTS.md index a727b20596c..5bdc538b2d9 100644 --- a/tests/integration/AGENTS.md +++ b/tests/integration/AGENTS.md @@ -37,3 +37,14 @@ together with its dedicated job, and list a new dedicated GitHub workflow with t `DEDICATED_INTEGRATION_WORKFLOWS` in `.github/scripts/assert_ci_coverage.py`. The `ci-coverage` check fails a file here that a GitHub workflow runs unless it is in `GITHUB_FILES` and that workflow owns its directory (#46780) + +## Fixtures that boot a proxy + +Scope them to the module or class, or pin a session one to a single worker with an `xdist_group` mark +as `_llm_gateway` in `mcp/test_mcp_llm_endpoints.py` does, and request them as arguments, never through +`request.getfixturevalue`. The harness puts tests that request the same set of module or class fixtures +on one xdist worker, so the proxy they share boots there once and stops when that worker moves to another +file. A test that requests one more of those fixtures than its neighbors gets its own group and can boot +the proxy again on another worker, so keep that set the same across the tests that share a proxy. An +unpinned session fixture boots a copy on every worker that runs one of its tests and keeps them all until +the node ends, and those copies ran CircleCI integration-extensions nodes out of memory diff --git a/tests/integration/_support/client.py b/tests/integration/_support/client.py index b8ea11f3d2f..a759126fcd4 100644 --- a/tests/integration/_support/client.py +++ b/tests/integration/_support/client.py @@ -280,12 +280,14 @@ class Scenario: self.gateway.rows('SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = %s', (identity,)) == [] ) - def model(self, *, model_info: Mapping[str, JsonValue] | None = None, **parameters: JsonValue) -> str: - name: Final = f"integration-{uuid.uuid4().hex}" + def model( + self, *, name: str | None = None, model_info: Mapping[str, JsonValue] | None = None, **parameters: JsonValue + ) -> str: + model_name: Final = name if name is not None else f"integration-{uuid.uuid4().hex}" created: Final = self.gateway.post( "/model/new", { - "model_name": name, + "model_name": model_name, "litellm_params": { "model": "openai/gpt-4o-mini", "api_key": "integration-provider-key", @@ -298,8 +300,8 @@ class Scenario: identity: Final = string_value(object_value(created["model_info"])["id"]) self.cleanups.callback(self.delete_model, identity) if self.gateway.workers > 1: - self._await_every_worker(name, identity) - return name + self._await_every_worker(model_name, identity) + return model_name def _await_every_worker(self, name: str, identity: str) -> None: from litellm.constants import PROXY_CONFIG_RELOAD_INTERVAL_SECONDS diff --git a/tests/integration/_support/device_login.py b/tests/integration/_support/device_login.py index 9b990fd2167..55519678f40 100644 --- a/tests/integration/_support/device_login.py +++ b/tests/integration/_support/device_login.py @@ -73,6 +73,7 @@ COPILOT_USER_CODE: Final = "COPI-LEAK" COPILOT_ACCESS: Final = "device-login-github-access" COPILOT_REJECTED_ACCESS: Final = "device-login-github-access-rejected" COPILOT_FIRST_LOGIN_ACCESS: Final = "device-login-github-access-first-login" +COPILOT_UNSAVED_KEY_ACCESS: Final = "device-login-github-access-unsaved-key" COPILOT_STORED_KEY: Final = "tid=device-login;stored" COPILOT_MINTED_KEY: Final = "tid=device-login;minted" COPILOT_REJECTED_KEY: Final = "tid=device-login;rejected" @@ -132,7 +133,11 @@ def _oauth_token(request: Request, grant: threading.Event) -> Reply: def _copilot_api_key(request: Request, api_url: str) -> Reply: - if request.headers.get("authorization") not in (f"token {COPILOT_ACCESS}", f"token {COPILOT_FIRST_LOGIN_ACCESS}"): + if request.headers.get("authorization") not in ( + f"token {COPILOT_ACCESS}", + f"token {COPILOT_FIRST_LOGIN_ACCESS}", + f"token {COPILOT_UNSAVED_KEY_ACCESS}", + ): return _json(401, {"message": "Bad credentials"}) return _json( 200, @@ -380,4 +385,7 @@ def write_copilot_access(directory: Path, access_token: str) -> None: def clear_tokens(*directories: Path) -> None: for directory in directories: for path in directory.iterdir(): - path.unlink() + if path.is_dir(): + path.rmdir() + else: + path.unlink() diff --git a/tests/integration/_support/responses_replay.py b/tests/integration/_support/responses_replay.py new file mode 100644 index 00000000000..c4e73ffbf2b --- /dev/null +++ b/tests/integration/_support/responses_replay.py @@ -0,0 +1,214 @@ +import json +import uuid +from collections.abc import Callable, Sequence +from itertools import chain +from typing import Final + +from integration._support.client import list_value, object_value +from integration._support.wire import Reply, Request, Wire +from pydantic import JsonValue + +MODEL: Final = "gpt-6.1-sol" +PLAN: Final = "I am starting by choosing which city to check." +PICK: Final = "My pick is Tokyo, so I will check its weather." +PATCH_NOTE: Final = "I will create hello.txt containing hi." +ANSWER: Final = "It is raining in Tokyo." + +WEATHER_TOOL: Final[dict[str, JsonValue]] = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the weather for a city.", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + }, +} +PATCH_TOOL: Final[dict[str, JsonValue]] = { + "type": "custom", + "custom": {"name": "apply_patch", "description": "Apply a V4A patch to the workspace"}, +} + + +def _message(item_id: str, text: str, phase: str) -> dict[str, JsonValue]: + return { + "type": "message", + "id": item_id, + "status": "completed", + "role": "assistant", + "phase": phase, + "content": [{"type": "output_text", "text": text, "annotations": []}], + } + + +def _reasoning(item_id: str, summary: str) -> dict[str, JsonValue]: + return { + "type": "reasoning", + "id": item_id, + "summary": [{"type": "summary_text", "text": summary}], + "encrypted_content": f"enc_{item_id}", + } + + +_WEATHER_CALL: Final[dict[str, JsonValue]] = { + "type": "function_call", + "id": "fc_weather", + "call_id": "call_weather", + "name": "get_weather", + "arguments": '{"city":"Tokyo"}', + "status": "completed", +} +WEATHER_TURN: Final[tuple[dict[str, JsonValue], ...]] = ( + _message("msg_plan", PLAN, "commentary"), + _reasoning("rs_pick", "Tokyo is often rainy this time of year."), + _message("msg_pick", PICK, "commentary"), + _WEATHER_CALL, +) +TWO_THOUGHTS_TURN: Final[tuple[dict[str, JsonValue], ...]] = ( + _reasoning("rs_scan", "Paris and Tokyo are the two candidates."), + _reasoning("rs_pick", "Tokyo is often rainy this time of year."), + _message("msg_pick", PICK, "commentary"), + _WEATHER_CALL, +) +CALL_ONLY_TURN: Final[tuple[dict[str, JsonValue], ...]] = (_WEATHER_CALL,) +PATCH_TURN: Final[tuple[dict[str, JsonValue], ...]] = ( + _message("msg_patch", PATCH_NOTE, "commentary"), + { + "type": "custom_tool_call", + "id": "ctc_patch", + "call_id": "call_patch", + "name": "apply_patch", + "input": "*** Begin Patch\n*** Add File: hello.txt\n+hi\n*** End Patch", + "status": "completed", + }, +) +WEATHER_REPLAY: Final = ( + ("message", "assistant", "commentary", PLAN), + ("reasoning", "rs_pick"), + ("message", "assistant", "commentary", PICK), + ("function_call", "call_weather", "get_weather"), + ("function_call_output", "call_weather"), +) +TWO_THOUGHTS_REPLAY: Final = ( + ("reasoning", "rs_scan"), + ("reasoning", "rs_pick"), + ("message", "assistant", "commentary", PICK), + ("function_call", "call_weather", "get_weather"), + ("function_call_output", "call_weather"), +) + + +def _response(identity: str, output: Sequence[dict[str, JsonValue]]) -> dict[str, JsonValue]: + return { + "id": f"resp_{identity}", + "object": "response", + "created_at": 1789788253, + "status": "completed", + "model": MODEL, + "output": list(output), + "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15}, + } + + +def _event(payload: dict[str, JsonValue]) -> bytes: + return f"event: {payload['type']}\ndata: {json.dumps(payload)}\n\n".encode() + + +def _opened(item: dict[str, JsonValue]) -> dict[str, JsonValue]: + match item["type"]: + case "message": + return {**item, "status": "in_progress", "content": []} + case "function_call": + return {**item, "status": "in_progress", "arguments": ""} + case _: + return item + + +def _deltas(index: int, item: dict[str, JsonValue]) -> tuple[dict[str, JsonValue], ...]: + match item["type"]: + case "message": + text: Final = object_value(list_value(item["content"])[0])["text"] + return ( + { + "type": "response.output_text.delta", + "item_id": item["id"], + "output_index": index, + "content_index": 0, + "delta": text, + }, + ) + case "function_call": + return ( + { + "type": "response.function_call_arguments.delta", + "item_id": item["id"], + "output_index": index, + "delta": item["arguments"], + }, + ) + case _: + return () + + +def _item_events(index: int, item: dict[str, JsonValue]) -> tuple[bytes, ...]: + return ( + _event({"type": "response.output_item.added", "output_index": index, "item": _opened(item)}), + *(_event(delta) for delta in _deltas(index, item)), + _event({"type": "response.output_item.done", "output_index": index, "item": item}), + ) + + +def _stream(identity: str, output: Sequence[dict[str, JsonValue]]) -> tuple[bytes, ...]: + response: Final = _response(identity, output) + return ( + _event({"type": "response.created", "response": {**response, "status": "in_progress", "output": []}}), + *chain.from_iterable(_item_events(index, item) for index, item in enumerate(output)), + _event({"type": "response.completed", "response": response}), + ) + + +def _answered(body: dict[str, JsonValue]) -> bool: + return any( + object_value(item).get("type") in {"function_call_output", "custom_tool_call_output"} + for item in list_value(body["input"]) + ) + + +def responses_upstream(turn_one: Sequence[dict[str, JsonValue]]) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + if request.method == "GET": + return Reply(body=b'{"object":"list","data":[]}') + assert request.target == "/responses", request.target + body: Final = object_value(json.loads(request.body)) + identity: Final = uuid.uuid4().hex + output: Final = (_message("msg_answer", ANSWER, "final_answer"),) if _answered(body) else turn_one + if body.get("stream"): + return Reply(content_type="text/event-stream", chunks=_stream(identity, output)) + return Reply(body=json.dumps(_response(identity, output)).encode()) + + return respond + + +def _text(content: JsonValue) -> JsonValue: + if isinstance(content, str): + return content + return "".join(str(object_value(part).get("text")) for part in list_value(content)) + + +def _shape(item: JsonValue) -> tuple[JsonValue, ...]: + fields: Final = object_value(item) + match fields.get("type"): + case "message": + return ("message", fields.get("role"), fields.get("phase"), _text(fields.get("content"))) + case "reasoning": + return ("reasoning", fields.get("id")) + case "function_call" | "custom_tool_call" as kind: + return (kind, fields.get("call_id"), fields.get("name")) + case kind: + return (kind, fields.get("call_id")) + + +def posted_inputs(wire: Wire) -> tuple[tuple[tuple[JsonValue, ...], ...], ...]: + return tuple( + tuple(_shape(item) for item in list_value(object_value(json.loads(request.body))["input"])) + for request in wire.drain() + if request.method == "POST" + ) diff --git a/tests/integration/authorization/test_claude_code_gateway_oauth.py b/tests/integration/authorization/test_claude_code_gateway_oauth.py index a80eeacad46..4dedbdcd8e4 100644 --- a/tests/integration/authorization/test_claude_code_gateway_oauth.py +++ b/tests/integration/authorization/test_claude_code_gateway_oauth.py @@ -545,3 +545,94 @@ def test_device_authorization_polls_complete_sso_and_mints_a_model_restricted_be assert set(token_form) == {"code", "code_verifier", "grant_type", "redirect_uri"}, token_form assert userinfo_request.method == "GET" and urlsplit(userinfo_request.target).path == "/userinfo" assert user_id == _USER_ID + + +_TEAM_OPTION: Final = re.compile(r'