diff --git a/.greptile/files.json b/.greptile/files.json index cb54d7f3c5e..99ef105499e 100644 --- a/.greptile/files.json +++ b/.greptile/files.json @@ -196,6 +196,13 @@ "litellm-rust/crates/llms/src/anthropic/batches/**" ] }, + { + "path": "litellm-rust/crates/llms/src/anthropic/chat/AGENTS.md", + "description": "Conventions for code under litellm-rust/crates/llms/src/anthropic/chat/", + "scope": [ + "litellm-rust/crates/llms/src/anthropic/chat/**" + ] + }, { "path": "litellm-rust/crates/llms/src/anthropic/count_tokens/AGENTS.md", "description": "Conventions for code under litellm-rust/crates/llms/src/anthropic/count_tokens/", @@ -224,6 +231,13 @@ "litellm-rust/crates/llms/src/azure_ai/messages/**" ] }, + { + "path": "litellm-rust/crates/llms/src/azure_ai/ocr/AGENTS.md", + "description": "Conventions for code under litellm-rust/crates/llms/src/azure_ai/ocr/", + "scope": [ + "litellm-rust/crates/llms/src/azure_ai/ocr/**" + ] + }, { "path": "litellm-rust/crates/llms/src/base_llm/messages/AGENTS.md", "description": "Conventions for code under litellm-rust/crates/llms/src/base_llm/messages/", @@ -231,6 +245,27 @@ "litellm-rust/crates/llms/src/base_llm/messages/**" ] }, + { + "path": "litellm-rust/crates/llms/src/bedrock/AGENTS.md", + "description": "Conventions for code under litellm-rust/crates/llms/src/bedrock/", + "scope": [ + "litellm-rust/crates/llms/src/bedrock/**" + ] + }, + { + "path": "litellm-rust/crates/llms/src/bedrock/audio_transcription/AGENTS.md", + "description": "Conventions for code under litellm-rust/crates/llms/src/bedrock/audio_transcription/", + "scope": [ + "litellm-rust/crates/llms/src/bedrock/audio_transcription/**" + ] + }, + { + "path": "litellm-rust/crates/llms/src/bedrock/chat/AGENTS.md", + "description": "Conventions for code under litellm-rust/crates/llms/src/bedrock/chat/", + "scope": [ + "litellm-rust/crates/llms/src/bedrock/chat/**" + ] + }, { "path": "litellm-rust/crates/llms/src/bedrock/messages/AGENTS.md", "description": "Conventions for code under litellm-rust/crates/llms/src/bedrock/messages/", @@ -238,6 +273,62 @@ "litellm-rust/crates/llms/src/bedrock/messages/**" ] }, + { + "path": "litellm-rust/crates/llms/src/cohere/ocr/AGENTS.md", + "description": "Conventions for code under litellm-rust/crates/llms/src/cohere/ocr/", + "scope": [ + "litellm-rust/crates/llms/src/cohere/ocr/**" + ] + }, + { + "path": "litellm-rust/crates/llms/src/deepseek/messages/AGENTS.md", + "description": "Conventions for code under litellm-rust/crates/llms/src/deepseek/messages/", + "scope": [ + "litellm-rust/crates/llms/src/deepseek/messages/**" + ] + }, + { + "path": "litellm-rust/crates/llms/src/mistral/ocr/AGENTS.md", + "description": "Conventions for code under litellm-rust/crates/llms/src/mistral/ocr/", + "scope": [ + "litellm-rust/crates/llms/src/mistral/ocr/**" + ] + }, + { + "path": "litellm-rust/crates/llms/src/openai/responses/AGENTS.md", + "description": "Conventions for code under litellm-rust/crates/llms/src/openai/responses/", + "scope": [ + "litellm-rust/crates/llms/src/openai/responses/**" + ] + }, + { + "path": "litellm-rust/crates/llms/src/openai_like/chat/AGENTS.md", + "description": "Conventions for code under litellm-rust/crates/llms/src/openai_like/chat/", + "scope": [ + "litellm-rust/crates/llms/src/openai_like/chat/**" + ] + }, + { + "path": "litellm-rust/crates/llms/src/reducto/ocr/AGENTS.md", + "description": "Conventions for code under litellm-rust/crates/llms/src/reducto/ocr/", + "scope": [ + "litellm-rust/crates/llms/src/reducto/ocr/**" + ] + }, + { + "path": "litellm-rust/crates/llms/src/vertex_ai/messages/AGENTS.md", + "description": "Conventions for code under litellm-rust/crates/llms/src/vertex_ai/messages/", + "scope": [ + "litellm-rust/crates/llms/src/vertex_ai/messages/**" + ] + }, + { + "path": "litellm-rust/crates/llms/src/vertex_ai/ocr/AGENTS.md", + "description": "Conventions for code under litellm-rust/crates/llms/src/vertex_ai/ocr/", + "scope": [ + "litellm-rust/crates/llms/src/vertex_ai/ocr/**" + ] + }, { "path": "litellm-rust/crates/llms-types/AGENTS.md", "description": "Conventions for code under litellm-rust/crates/llms-types/", @@ -246,10 +337,24 @@ ] }, { - "path": "litellm-rust/crates/llms-types/src/formats/AGENTS.md", - "description": "Conventions for code under litellm-rust/crates/llms-types/src/formats/", + "path": "litellm-rust/crates/llms-types/src/formats/audio_transcription/AGENTS.md", + "description": "Conventions for code under litellm-rust/crates/llms-types/src/formats/audio_transcription/", "scope": [ - "litellm-rust/crates/llms-types/src/formats/**" + "litellm-rust/crates/llms-types/src/formats/audio_transcription/**" + ] + }, + { + "path": "litellm-rust/crates/llms-types/src/formats/batches/AGENTS.md", + "description": "Conventions for code under litellm-rust/crates/llms-types/src/formats/batches/", + "scope": [ + "litellm-rust/crates/llms-types/src/formats/batches/**" + ] + }, + { + "path": "litellm-rust/crates/llms-types/src/formats/chat_completions/AGENTS.md", + "description": "Conventions for code under litellm-rust/crates/llms-types/src/formats/chat_completions/", + "scope": [ + "litellm-rust/crates/llms-types/src/formats/chat_completions/**" ] }, { @@ -260,10 +365,45 @@ ] }, { - "path": "litellm-rust/crates/llms-types/src/providers/AGENTS.md", - "description": "Conventions for code under litellm-rust/crates/llms-types/src/providers/", + "path": "litellm-rust/crates/llms-types/src/formats/ocr/AGENTS.md", + "description": "Conventions for code under litellm-rust/crates/llms-types/src/formats/ocr/", "scope": [ - "litellm-rust/crates/llms-types/src/providers/**" + "litellm-rust/crates/llms-types/src/formats/ocr/**" + ] + }, + { + "path": "litellm-rust/crates/llms-types/src/formats/responses/AGENTS.md", + "description": "Conventions for code under litellm-rust/crates/llms-types/src/formats/responses/", + "scope": [ + "litellm-rust/crates/llms-types/src/formats/responses/**" + ] + }, + { + "path": "litellm-rust/crates/llms-types/src/providers/anthropic/AGENTS.md", + "description": "Conventions for code under litellm-rust/crates/llms-types/src/providers/anthropic/", + "scope": [ + "litellm-rust/crates/llms-types/src/providers/anthropic/**" + ] + }, + { + "path": "litellm-rust/crates/llms-types/src/providers/bedrock/AGENTS.md", + "description": "Conventions for code under litellm-rust/crates/llms-types/src/providers/bedrock/", + "scope": [ + "litellm-rust/crates/llms-types/src/providers/bedrock/**" + ] + }, + { + "path": "litellm-rust/crates/llms-types/src/providers/minimax/AGENTS.md", + "description": "Conventions for code under litellm-rust/crates/llms-types/src/providers/minimax/", + "scope": [ + "litellm-rust/crates/llms-types/src/providers/minimax/**" + ] + }, + { + "path": "litellm-rust/crates/llms-types/src/providers/vertex_ai/AGENTS.md", + "description": "Conventions for code under litellm-rust/crates/llms-types/src/providers/vertex_ai/", + "scope": [ + "litellm-rust/crates/llms-types/src/providers/vertex_ai/**" ] }, { diff --git a/litellm-rust/AGENTS.md b/litellm-rust/AGENTS.md index b028425c7c1..069d9fd99ea 100644 --- a/litellm-rust/AGENTS.md +++ b/litellm-rust/AGENTS.md @@ -11,9 +11,9 @@ Use the derived conversions directly (`<&'static str>::from(x)` / `.into()`, `st ## Test placement - Never create a `tests.rs` (or `test.rs`) file under `src/`, and never `#[path = "tests.rs"] mod tests;` -- A test that reaches private items lives inline, in a `#[cfg(test)] mod tests { ... }` at the bottom of the file that owns those items -- A test that only uses the crate's public API lives in `crates//tests/.rs`, next to `src/` -- Split a mixed test file along that line instead of widening visibility to move it +- Tests live inline by default, in a `#[cfg(test)] mod tests { ... }` at the bottom of the file that owns the code under test, whether or not they touch private items +- `crates//tests/` is only for tests that drive several modules together end to end, such as a full request through a route. A test of one type or function never goes there +- A helper shared by several inline test modules goes in a `#[cfg(test)] mod test_support;` at the crate root, and only once a second module needs it - A test for another crate's item belongs in that crate, not in a downstream one - Never set `autotests = false` or hand-list `[[test]]` targets. Every file directly under `tests/` is discovered by cargo, and a shared helper goes in `tests//mod.rs` or `tests//support.rs` so it is not picked up as a test crate of its own diff --git a/litellm-rust/crates/inference-messages/src/common_utils.rs b/litellm-rust/crates/inference-messages/src/common_utils.rs index ea8c7cca4f7..5644d54dda0 100644 --- a/litellm-rust/crates/inference-messages/src/common_utils.rs +++ b/litellm-rust/crates/inference-messages/src/common_utils.rs @@ -15,51 +15,36 @@ use super::Error; const HEADER_CONTEXT: &str = "messages"; -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum MessagesProvider { - Anthropic, - AzureAi, - Bedrock, - Deepseek, - VertexAi, +#[derive(Clone, Copy)] +pub(crate) struct MessagesProvider { + provider: LlmProviders, + config: &'static dyn BaseMessagesConfig, } impl MessagesProvider { pub(crate) fn as_str(self) -> &'static str { - match self { - Self::Anthropic => LlmProviders::Anthropic, - Self::AzureAi => LlmProviders::AzureAi, - Self::Bedrock => LlmProviders::Bedrock, - Self::Deepseek => LlmProviders::Deepseek, - Self::VertexAi => LlmProviders::VertexAi, - } - .into() + self.provider.into() } pub(crate) fn config(self) -> &'static dyn BaseMessagesConfig { - match self { - Self::Anthropic => &ANTHROPIC_MESSAGES_CONFIG, - Self::AzureAi => &AZURE_ANTHROPIC_MESSAGES_CONFIG, - Self::Bedrock => &BEDROCK_ANTHROPIC_MESSAGES_CONFIG, - Self::Deepseek => &DEEPSEEK_ANTHROPIC_MESSAGES_CONFIG, - Self::VertexAi => &VERTEX_ANTHROPIC_MESSAGES_CONFIG, - } + self.config } } /// Python's `get_provider_anthropic_messages_config`: Vertex AI serves only its Claude /// partner models on this route. pub(crate) fn messages_provider(provider: LlmProviders, model: &str) -> Option { - match provider { - LlmProviders::Anthropic => Some(MessagesProvider::Anthropic), - LlmProviders::AzureAi => Some(MessagesProvider::AzureAi), - LlmProviders::Bedrock => Some(MessagesProvider::Bedrock), - LlmProviders::Deepseek => Some(MessagesProvider::Deepseek), + let config: &'static dyn BaseMessagesConfig = match provider { + LlmProviders::Anthropic => &ANTHROPIC_MESSAGES_CONFIG, + LlmProviders::AzureAi => &AZURE_ANTHROPIC_MESSAGES_CONFIG, + LlmProviders::Bedrock => &BEDROCK_ANTHROPIC_MESSAGES_CONFIG, + LlmProviders::Deepseek => &DEEPSEEK_ANTHROPIC_MESSAGES_CONFIG, LlmProviders::VertexAi if model.to_ascii_lowercase().contains("claude") => { - Some(MessagesProvider::VertexAi) + &VERTEX_ANTHROPIC_MESSAGES_CONFIG } - _ => None, - } + _ => return None, + }; + Some(MessagesProvider { provider, config }) } pub(super) fn string_headers( @@ -74,24 +59,19 @@ mod tests { use rstest::rstest; - use super::{MessagesProvider, messages_provider, string_headers, truncate_error_body}; + use super::{messages_provider, string_headers, truncate_error_body}; use crate::Error; use litellm_core_utils::get_llm_provider_logic::LlmProviders; #[rstest] - #[case::anthropic("anthropic", MessagesProvider::Anthropic)] - #[case::azure_ai("azure_ai", MessagesProvider::AzureAi)] - #[case::bedrock("bedrock", MessagesProvider::Bedrock)] - #[case::deepseek("deepseek", MessagesProvider::Deepseek)] - #[case::vertex_ai("vertex_ai", MessagesProvider::VertexAi)] - fn provider_round_trips_through_its_python_name( - #[case] name: &str, - #[case] provider: MessagesProvider, - ) { - assert_eq!( - messages_provider(name.parse::().unwrap(), "claude-sonnet-4-5"), - Some(provider) - ); + #[case::anthropic("anthropic")] + #[case::azure_ai("azure_ai")] + #[case::bedrock("bedrock")] + #[case::deepseek("deepseek")] + #[case::vertex_ai("vertex_ai")] + fn provider_keeps_its_python_name(#[case] name: &str) { + let provider = + messages_provider(name.parse::().unwrap(), "claude-sonnet-4-5").unwrap(); assert_eq!(provider.as_str(), name); } @@ -102,7 +82,7 @@ mod tests { #[case] provider: LlmProviders, #[case] model: &str, ) { - assert_eq!(messages_provider(provider, model), None); + assert!(messages_provider(provider, model).is_none()); } #[test] diff --git a/litellm-rust/crates/llms-types/AGENTS.md b/litellm-rust/crates/llms-types/AGENTS.md index c8e590845d4..aa216d7bdab 100644 --- a/litellm-rust/crates/llms-types/AGENTS.md +++ b/litellm-rust/crates/llms-types/AGENTS.md @@ -1,75 +1,63 @@ -The same ownership rule applies to Messages, Responses, Chat Completions, OCR, and other API formats. This crate owns their shared API data contracts. Adapter contracts and shared transformation machinery belong in `llms/src/base_llm//`, provider policy in `llms/src///`, and call orchestration in `inference-`. A provider originating a format, or several providers using a type, does not change these responsibilities. Existing model locations outside this crate are not exceptions to this rule for new shared API contracts +# rules -- `litellm-llms-types` owns shared API data contracts and their serialization - - A type belongs here when it describes a request, response, event, or value that consumers must agree on independently of how a call executes - - Being public, serializable, or used by several crates is not sufficient - - These are intended boundaries, not a claim that every existing item follows them +## Scope -- Organize public API contracts under `formats`: `messages`, `chat_completions`, `responses`, `ocr`, `audio_transcription`, and `batches` - - Use names such as `litellm_llms_types::formats::messages::MessagesRequest`, without an Anthropic prefix solely because Anthropic designed Messages - - Keep one canonical definition and import path when moving a contract, updating consumers together instead of adding duplicate models or compatibility re-exports +- This crate owns shared API data contracts and their serialization, nothing else +- A type belongs here when consumers must agree on a request, response, event or value independently of how a call executes +- Being public, serializable or used by several crates is not enough on its own -- Keep shared provider-specific wire types and extensions under `providers` - - Provider types may reuse format types. Format types must not depend on provider types - - A field belonging to an API format stays under `formats` even when provider support varies. Including it in a type does not promise provider support - - Add a typed provider extension when a consumer needs to interpret or construct it. Keep adapter-only projections in `llms` until a shared public data contract is needed - - Keep one authoritative representation of each field, preserving unknown fields without duplicating typed values in an extension map - - Provider capability checks, defaults, authentication, header selection, and transformations remain in `llms` +## Layout -- Keep format-independent data helpers such as `headers`, `recognized`, and `serde_compat` at the crate root +- `formats//` holds a provider-neutral API format, `providers//` holds one provider's shared pieces +- Every format and provider is a folder with a `mod.rs` entrypoint and its own `AGENTS.md`. No standalone `.rs` next to a folder +- A `mod.rs` only declares modules and glob re-exports them (`pub use request::*`). Constants go in `constants.rs`, types in a file named for what they describe (`request.rs`, `response.rs`, `streaming.rs`) +- A submodule stays namespaced (`pub mod streaming`) only when its names would clash with the parent's +- Format-independent helpers (`headers`, `recognized`, `serde_compat`, `json_schema`) stay at the crate root +- AGENTS.md files follow the convention in `../llms/AGENTS.md`. Nested files: `src/formats//` for every format, `src/providers//` for every provider -- Shared request/response bodies, message and content-block enums, usage records, tool-call chunks, stream-event payloads, and protocol error bodies belong here - - This includes LiteLLM's normalized response contracts and extensions, not just exact upstream schemas - - `ChatCompletionsResponse` currently represents the response handed to the host, so replacing it with a supposedly more complete upstream schema must not silently change that contract - - Messages web-search result/error schemas and encrypted-content fields belong to the Messages format regardless of which providers implement them +## Formats -- Shared LiteLLM input data such as `ProviderSpecificHeader` and `ProviderSpecificHeaders` also belongs here - - Selecting entries for a provider belongs in `core-utils`, and applying headers belongs in `http` - - A genuinely provider-specific wire value may retain its provider name: `AnthropicBeta` and `BetaSet` describe the `anthropic-beta` header - - Parsing, formatting, deduplication, and value equality belong with those types - - Choosing required betas, OAuth companions, header precedence, or credentials belongs in `llms` and the auth crates +- Name types after the format, never the originator (`formats::messages::MessagesRequest`, not `AnthropicMessagesRequest`) +- A field defined by the format stays in the format type even when provider support varies +- Keep one canonical definition and import path. Move consumers with the type instead of adding re-exports +- LiteLLM's normalized contracts live here too. `ChatCompletionsResponse` is the response handed to the host, so do not swap it for a fuller upstream schema -- Allow deterministic constructors, accessors, serialization, schema generation, and validation of the represented data shape - - `ContentBlock::text`, `ResponsesWsEvent::model`, and `Recognized::known` are examples - - Exact value conversions such as `EffortLevel` to the matching `ReasoningEffort` are acceptable - - Clamping effort, choosing a thinking budget, rewriting content, mapping finish reasons, computing normalized usage, and translating between API formats are policy or transformations and belong outside this crate, even when they are pure functions +## Providers -- Keep call envelopes and execution state in their owning crates - - `MessagesCall`, `MessagesShaping`, prepared provider requests, and the response wrapper containing a live stream belong in `inference-messages` - - Provider config traits, `MessagesTransformContext`, `MessagesModelCapabilities`, `ThinkingBudgets`, `StreamShape`, and transformer state belong in `llms` - - Catalog records and pricing belong in `model-catalog`, which may reuse wire enums such as `ReasoningEffort` - - Host hooks, Python objects, credentials, clients, timeouts, and routing decisions do not become API payload types merely because they cross a crate boundary - - Legacy logging operation selection belongs in `callbacks-legacy-python`, not this crate +- A provider with more than one endpoint format puts shared endpoint paths, default headers and wire types used by several of its formats in `providers//` +- Projections that only one adapter decodes (`InvokeChunkPayload`, the count-tokens and batches adapter structs, `ReplayedWebSearchResult`) stay in `llms/src///` +- A typed provider extension of a format (MiniMax media blocks) belongs here once a consumer must construct or interpret it +- Provider types may use format types. Format types never depend on provider types -- Stream-event data belongs here, but live streams, decoders, framing, buffering, and stream lifecycle decisions do not - - Keep SSE and AWS framing in `framer`, provider decoding and conversion in `llms`, and call orchestration in the `inference-` crates - - `ResponsesWsEvent` belongs here - - `ResponsesWsTransformResult` wraps the output of a provider transformation rather than a wire event and lives in `llms::base_llm::responses::transformation` - - Protocol error payloads may live here, while operational errors remain in the crate that raises them +## Behavior on types -- Keep provider-only wire envelopes and transformation-specific projections local until there is a shared API contract to expose - - `InvokeChunkPayload` and the permissive `ReplayedWebSearchResult` projection serve decoding or rendering and should not be promoted unchanged into public schemas - - Reuse existing shared message contracts inside batch and token-counting adapters without moving every adapter struct into this crate - - Shared normalization intermediates such as the prompt factory's `Conversation` stay with their algorithms in `core-utils` +- Allowed: deterministic constructors, accessors, serialization, schema generation, shape validation and exact value conversions +- Not allowed: clamping, budget selection, content rewriting, finish-reason mapping, usage normalization or cross-format translation. These belong in `llms`, even when pure +- Provider capability checks, defaults, auth and header selection belong in `llms` -- Keep this crate at the bottom of the dependency graph - - Use data, serialization, and optional schema libraries without depending on workspace execution crates, async runtimes, transport clients, or Python bindings - - No network or filesystem I/O, environment lookup, clock access, model catalog lookup, or global configuration reads belong here - - Defaults must describe the data contract, not select runtime policy +## What lives elsewhere -- Represent known discriminators such as `tool_use` and `server_tool_use` with enum variants - - Preserve each contract's existing treatment of unknown variants, extra fields, missing fields, and explicit nulls - - `Recognized` deliberately retains values that fail typed parsing, including wrong-shaped values, so use it only where permissive passthrough is already part of the contract - - Typing an opaque field must neither reject previously accepted inputs nor accept malformed inputs previously rejected - - Do not silently discard unknown data or tighten a partial projection into a stricter public schema +- Call envelopes and execution state (`MessagesCall`, prepared requests, live streams) belong in `inference-` +- Adapter contracts and transform state (`MessagesTransformContext`, `StreamShape`) belong in `llms` +- SSE and AWS framing belong in `framer`. Catalog records and pricing belong in `model-catalog` +- Host hooks, Python objects, credentials, clients and routing decisions are not payload types -- Test observable serialization, malformed-input rejection, unknown-value preservation, and value semantics in this crate - - Follow the workspace test-placement and `rstest` rules - - Test provider transformations, header policy, and stream execution in their owning crates - - Do not test import locations or Rust source structure as substitutes for behavior +## Dependencies + +- Data, serde and optional schema libraries only. No workspace execution crates, async runtimes, transport clients or Python bindings +- No I/O, env lookup, clock access or global config reads. Defaults describe the data contract, not runtime policy + +## Unknown data + +- Model known discriminators as enum variants and keep each contract's existing handling of unknown variants, extra fields, missing fields and explicit nulls +- Use `Recognized` only where permissive passthrough is already part of the contract +- Typing an opaque field must neither reject previously accepted input nor accept previously rejected input + +## Tests + +- Cover serialization, malformed-input rejection, unknown-value preservation and value semantics here +- Transformations and header policy are tested in their owning crates # references -## json_schema.rs - - https://json-schema.org/draft/2020-12/json-schema-core diff --git a/litellm-rust/crates/llms-types/src/formats/AGENTS.md b/litellm-rust/crates/llms-types/src/formats/AGENTS.md deleted file mode 100644 index e49c286f058..00000000000 --- a/litellm-rust/crates/llms-types/src/formats/AGENTS.md +++ /dev/null @@ -1,36 +0,0 @@ -# references - -These links describe shared format fields and discriminators. The official API reference is authoritative, and SDK sources only clarify shapes the reference leaves implicit. These contracts are partial typed projections, not exhaustive upstream schemas - -## chat_completions.rs and chat_completions/content.rs - -- https://developers.openai.com/api/reference/resources/chat - -SDK wire definitions: - -- https://github.com/openai/openai-python/blob/main/src/openai/types/chat/chat_completion_content_part_param.py - -## responses/output.rs and responses/streaming_websocket.rs - -- https://developers.openai.com/api/reference/resources/responses -- https://developers.openai.com/api/reference/resources/responses/streaming-events - -SDK wire definitions: - -- https://github.com/openai/openai-python/blob/main/src/openai/types/responses/response.py -- https://github.com/openai/openai-python/blob/main/src/openai/types/responses/response_output_item.py -- https://github.com/openai/openai-python/blob/main/src/openai/types/responses/mcp_tool_call_error.py - -## batches.rs - -- https://developers.openai.com/api/reference/resources/batches - -## audio_transcription.rs - -- https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions - -## ocr.rs - -- https://docs.mistral.ai/api/endpoint/ocr - -Messages references live in `messages/AGENTS.md`. `LiteLLMOcrResponse` follows the Mistral OCR response shape, and `OcrBoundingBox` describes the corner coordinates providers copy into the normalized `bbox`. Normalized `tables` and `keyValuePairs` stay open because providers pass through different native shapes diff --git a/litellm-rust/crates/llms-types/src/formats/audio_transcription/AGENTS.md b/litellm-rust/crates/llms-types/src/formats/audio_transcription/AGENTS.md new file mode 100644 index 00000000000..22608b1b3e8 --- /dev/null +++ b/litellm-rust/crates/llms-types/src/formats/audio_transcription/AGENTS.md @@ -0,0 +1,9 @@ +# rules + +- The transcription contract follows OpenAI's `audio/transcriptions` shape, which is LiteLLM's normalized input and output for every transcription provider +- Providers without that endpoint translate in `llms/src//audio_transcription/` (Bedrock builds a Converse request) +- Parameter support differs per provider and is filtered in the provider config, not by trimming the type + +# references + +- https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions diff --git a/litellm-rust/crates/llms-types/src/formats/audio_transcription/mod.rs b/litellm-rust/crates/llms-types/src/formats/audio_transcription/mod.rs new file mode 100644 index 00000000000..1d28ef927d6 --- /dev/null +++ b/litellm-rust/crates/llms-types/src/formats/audio_transcription/mod.rs @@ -0,0 +1,3 @@ +mod response; + +pub use response::*; diff --git a/litellm-rust/crates/llms-types/src/formats/audio_transcription.rs b/litellm-rust/crates/llms-types/src/formats/audio_transcription/response.rs similarity index 100% rename from litellm-rust/crates/llms-types/src/formats/audio_transcription.rs rename to litellm-rust/crates/llms-types/src/formats/audio_transcription/response.rs diff --git a/litellm-rust/crates/llms-types/src/formats/batches/AGENTS.md b/litellm-rust/crates/llms-types/src/formats/batches/AGENTS.md new file mode 100644 index 00000000000..a28e9b05f24 --- /dev/null +++ b/litellm-rust/crates/llms-types/src/formats/batches/AGENTS.md @@ -0,0 +1,9 @@ +# rules + +- The batch contract follows OpenAI's Batch object, which is LiteLLM's normalized batch shape for every provider +- Providers with their own batch API map into it in `llms/src//batches/` (Anthropic Message Batches) +- Statuses and counts a provider lacks are derived or defaulted in the provider transformation, not added here + +# references + +- https://developers.openai.com/api/reference/resources/batches diff --git a/litellm-rust/crates/llms-types/src/formats/batches/mod.rs b/litellm-rust/crates/llms-types/src/formats/batches/mod.rs new file mode 100644 index 00000000000..1d28ef927d6 --- /dev/null +++ b/litellm-rust/crates/llms-types/src/formats/batches/mod.rs @@ -0,0 +1,3 @@ +mod response; + +pub use response::*; diff --git a/litellm-rust/crates/llms-types/src/formats/batches.rs b/litellm-rust/crates/llms-types/src/formats/batches/response.rs similarity index 100% rename from litellm-rust/crates/llms-types/src/formats/batches.rs rename to litellm-rust/crates/llms-types/src/formats/batches/response.rs diff --git a/litellm-rust/crates/llms-types/src/formats/chat_completions.rs b/litellm-rust/crates/llms-types/src/formats/chat_completions.rs deleted file mode 100644 index 8f154dd7fb5..00000000000 --- a/litellm-rust/crates/llms-types/src/formats/chat_completions.rs +++ /dev/null @@ -1,214 +0,0 @@ -use serde_json::{Map, Value}; -use strum::IntoStaticStr; - -mod content; -pub use content::{ - ChatContentPart, ChatFile, ChatInputAudio, ChatLogprobs, ChatMediaUrl, ChatMediaUrlParameters, - ChatTokenLogprob, ChatTopLogprob, ChatVideoMetadata, PromptCacheBreakpoint, PromptCacheMode, -}; - -/// Reasoning effort level accepted or applied by the model. -#[macro_rules_attribute::apply(crate::wire_type)] -#[derive(Copy, Eq, IntoStaticStr, strum::EnumString, strum::VariantArray)] -#[serde(rename_all = "snake_case")] -pub enum ReasoningEffort { - #[strum(serialize = "none")] - None, - #[strum(serialize = "minimal")] - Minimal, - #[strum(serialize = "low")] - Low, - #[strum(serialize = "medium")] - Medium, - #[strum(serialize = "high")] - High, - #[strum(serialize = "xhigh")] - Xhigh, - #[strum(serialize = "max")] - Max, -} - -#[macro_rules_attribute::apply(crate::wire_type)] -#[serde(untagged)] -pub enum ChatMessageContent { - Text(String), - Parts(Vec), -} - -#[macro_rules_attribute::apply(crate::wire_type)] -pub struct ChatMessage { - pub role: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub content: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub name: Option, - #[serde(flatten)] - pub extra: Map, -} - -#[macro_rules_attribute::apply(crate::wire_type)] -pub struct ChatCompletionToolCallFunctionChunk { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub name: Option, - pub arguments: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub provider_specific_fields: Option>, -} - -#[macro_rules_attribute::apply(crate::wire_type)] -pub struct ChatCompletionToolCallChunk { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub id: Option, - #[serde(rename = "type")] - pub tool_type: String, - pub function: ChatCompletionToolCallFunctionChunk, - pub index: i64, -} - -#[macro_rules_attribute::apply(crate::wire_type)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum ChatCompletionThinkingBlock { - Thinking { - #[serde(default, skip_serializing_if = "Option::is_none")] - thinking: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - signature: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - cache_control: Option, - }, - RedactedThinking { - #[serde(default, skip_serializing_if = "Option::is_none")] - data: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - cache_control: Option, - }, -} - -/// OpenAI `usage`, including the `prompt_tokens_details` split LiteLLM's Python -/// path reports so cost tracking sees the same numbers on either path. -#[macro_rules_attribute::apply(crate::wire_type)] -#[derive(Default)] -pub struct PromptTokensDetails { - pub cached_tokens: u64, - pub cache_creation_tokens: u64, - pub text_tokens: u64, -} - -#[macro_rules_attribute::apply(crate::wire_type)] -#[derive(Default)] -pub struct ChatCompletionsUsage { - pub prompt_tokens: u64, - pub completion_tokens: u64, - pub total_tokens: u64, - pub prompt_tokens_details: PromptTokensDetails, -} - -#[macro_rules_attribute::apply(crate::wire_type)] -pub struct ChatCompletionsChoiceMessage { - pub role: String, - // Whether an empty turn is `None` or `""` is the provider's choice, not a - // shared invariant: Anthropic's transform ends on `merged_text or None` - // while Converse assigns the joined string unconditionally. Each config - // mirrors its own, so keep this optional and serialize it even when None. - pub content: Option, -} - -#[macro_rules_attribute::apply(crate::wire_type)] -pub struct ChatCompletionsChoice { - pub index: u64, - pub message: ChatCompletionsChoiceMessage, - pub finish_reason: String, -} - -/// The normalized response handed back to the host. -/// -/// There is deliberately no `id`: Python mints the `chatcmpl-…` id on the -/// `ModelResponse` it already created, and echoing the provider's own id here -/// would change it. Pinned by `response_carries_no_id` in the Anthropic chat transformation tests. -#[macro_rules_attribute::apply(crate::wire_type)] -pub struct ChatCompletionsResponse { - pub created: u64, - pub model: String, - pub choices: Vec, - pub usage: ChatCompletionsUsage, -} - -#[macro_rules_attribute::apply(crate::wire_type)] -#[derive(Default)] -pub struct ChatCompletionDelta { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub content: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub role: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub tool_calls: Option>, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub reasoning_content: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub thinking_blocks: Option>, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub provider_specific_fields: Option>, - #[serde(flatten)] - pub extra: Map, -} - -#[macro_rules_attribute::apply(crate::wire_type)] -pub struct ChatCompletionStreamingChoice { - pub index: u64, - pub delta: ChatCompletionDelta, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub finish_reason: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub logprobs: Option, -} - -#[macro_rules_attribute::apply(crate::wire_type)] -pub struct ChatCompletionChunk { - pub id: String, - pub created: u64, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub model: Option, - pub object: String, - pub choices: Vec, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub usage: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub provider_specific_fields: Option>, -} - -#[cfg(test)] -mod tests { - use rstest::rstest; - use strum::VariantArray; - - use super::*; - - #[rstest] - fn reasoning_effort_names_match_the_wire_and_parse_back( - #[values( - ReasoningEffort::None, - ReasoningEffort::Minimal, - ReasoningEffort::Low, - ReasoningEffort::Medium, - ReasoningEffort::High, - ReasoningEffort::Xhigh, - ReasoningEffort::Max - )] - effort: ReasoningEffort, - ) { - assert_eq!( - serde_json::to_value(effort).unwrap(), - Value::String(<&'static str>::from(effort).to_string()) - ); - assert_eq!(<&'static str>::from(effort).parse(), Ok(effort)); - assert!(ReasoningEffort::VARIANTS.contains(&effort)); - } - - #[rstest] - #[case::unknown("ultra")] - #[case::uppercase("HIGH")] - #[case::empty("")] - fn reasoning_effort_parse_rejects(#[case] value: &str) { - assert!(value.parse::().is_err()); - } -} diff --git a/litellm-rust/crates/llms-types/src/formats/chat_completions/AGENTS.md b/litellm-rust/crates/llms-types/src/formats/chat_completions/AGENTS.md new file mode 100644 index 00000000000..34c9b02c012 --- /dev/null +++ b/litellm-rust/crates/llms-types/src/formats/chat_completions/AGENTS.md @@ -0,0 +1,15 @@ +# rules + +- Chat Completions originated at OpenAI and is the most widely cloned LLM API. Types here describe the format, not OpenAI's deployment of it +- Spoken natively by OpenAI and OpenAI-compatible hosts, served by `llms/src/openai_like/chat/` +- Other providers translate into it from their own API (Anthropic Messages, Bedrock Converse) in `llms/src//chat/` +- Compatible hosts commonly + - accept a subset of parameters, which is recorded per provider in `supported_openai_param_mappings`, not by trimming the type + - add top-level request or response fields, preserved through the existing extra-field maps instead of new typed fields + - return slightly different usage or finish reasons, normalized in the provider transformation +- `ChatCompletionsResponse` is LiteLLM's response contract to the host, a typed projection rather than the full upstream schema + +# references + +- https://developers.openai.com/api/reference/resources/chat +- https://github.com/openai/openai-python/blob/main/src/openai/types/chat/chat_completion_content_part_param.py diff --git a/litellm-rust/crates/llms-types/src/formats/chat_completions/content.rs b/litellm-rust/crates/llms-types/src/formats/chat_completions/content.rs index d508bb672ab..7298145533c 100644 --- a/litellm-rust/crates/llms-types/src/formats/chat_completions/content.rs +++ b/litellm-rust/crates/llms-types/src/formats/chat_completions/content.rs @@ -153,3 +153,169 @@ pub struct ChatTopLogprob { #[serde(flatten)] pub extra: Map, } + +#[cfg(test)] +mod tests { + use crate::formats::{ + chat_completions::{ChatContentPart, ChatLogprobs, ChatMediaUrl, PromptCacheMode}, + messages::ContentSource, + }; + + use rstest::rstest; + use serde_json::{Value, json}; + + #[rstest] + #[case::parts(json!([ + {"type":"text","text":"hello","cache_control":{"type":"ephemeral"},"prompt_cache_breakpoint":{"mode":"explicit"}}, + {"type":"image_url","image_url":{"url":"https://example.test/image","detail":"high","format":"image/png"}}, + {"type":"video_url","video_url":"https://example.test/video"}, + {"type":"input_audio","input_audio":{"data":"AA==","format":"wav"}}, + {"type":"file","file":{"file_id":"file_1","file_data":"JVBE","filename":"a.mp4","format":"video/mp4","detail":"low","video_metadata":{"fps":1,"start_offset":"1s","end_offset":"2s"}}}, + {"type":"document","source":{"type":"text","media_type":"text/plain","data":"doc"},"title":"T","context":"C","citations":{"enabled":true}}, + {"type":"refusal","refusal":"refused"} + ]))] + fn content_parts_round_trip(#[case] wire: Value) { + let parts: Vec = serde_json::from_value(wire.clone()).unwrap(); + let [ + ChatContentPart::Text { + text, + cache_control, + prompt_cache_breakpoint: Some(breakpoint), + extra: text_extra, + }, + ChatContentPart::ImageUrl { + image_url: ChatMediaUrl::Parameters(image), + prompt_cache_breakpoint: None, + .. + }, + ChatContentPart::VideoUrl { + video_url: ChatMediaUrl::Url(video), + .. + }, + ChatContentPart::InputAudio { input_audio, .. }, + ChatContentPart::File { file, .. }, + ChatContentPart::Document { + source, + title: Some(title), + context: Some(context), + citations: Some(citations), + .. + }, + ChatContentPart::Refusal { refusal, .. }, + ] = parts.as_slice() + else { + panic!("expected typed content parts"); + }; + assert_eq!(text, "hello"); + assert_eq!( + cache_control.as_ref().unwrap().cache_type.as_deref(), + Some("ephemeral") + ); + assert_eq!(breakpoint.mode, PromptCacheMode::Explicit); + assert!(breakpoint.extra.is_empty()); + assert!(text_extra.is_empty()); + assert_eq!(image.url, "https://example.test/image"); + assert_eq!(image.detail.as_deref(), Some("high")); + assert_eq!(image.format.as_deref(), Some("image/png")); + assert!(image.extra.is_empty()); + assert_eq!(video, "https://example.test/video"); + assert_eq!(input_audio.data, "AA=="); + assert_eq!(input_audio.format, "wav"); + assert_eq!(file.file_id.as_deref(), Some("file_1")); + assert_eq!(file.file_data.as_deref(), Some("JVBE")); + assert_eq!(file.filename.as_deref(), Some("a.mp4")); + assert_eq!(file.format.as_deref(), Some("video/mp4")); + assert_eq!(file.detail.as_deref(), Some("low")); + assert!(file.extra.is_empty()); + let metadata = file.video_metadata.as_ref().unwrap(); + assert_eq!(metadata.fps, Some(1.into())); + assert_eq!(metadata.start_offset.as_deref(), Some("1s")); + assert_eq!(metadata.end_offset.as_deref(), Some("2s")); + assert!(metadata.extra.is_empty()); + let ContentSource::Text { + data, media_type, .. + } = source.as_ref() + else { + panic!("expected document text source"); + }; + assert_eq!(data, "doc"); + assert_eq!(media_type, "text/plain"); + assert_eq!(title, "T"); + assert_eq!(context, "C"); + assert_eq!(citations.enabled, Some(true)); + assert_eq!(refusal, "refused"); + assert_eq!(serde_json::to_value(parts).unwrap(), wire); + } + + #[rstest] + fn logprobs_round_trip_with_tokens_and_alternatives() { + let wire = json!({ + "content":[{"token":"hi","logprob":-1,"bytes":[104,105],"top_logprobs":[{"token":"hey","logprob":-2.5,"bytes":[104]}]}], + "refusal":[{"token":"refused","logprob":-3}] + }); + let logprobs: ChatLogprobs = serde_json::from_value(wire.clone()).unwrap(); + let token = &logprobs.content.as_ref().unwrap()[0]; + assert_eq!(token.token, "hi"); + assert_eq!(token.logprob, (-1).into()); + assert_eq!(token.bytes.as_deref(), Some([104, 105].as_slice())); + let alternative = &token.top_logprobs.as_ref().unwrap()[0]; + assert_eq!(alternative.token, "hey"); + assert_eq!(alternative.bytes.as_deref(), Some([104].as_slice())); + assert_eq!(logprobs.refusal.as_ref().unwrap()[0].token, "refused"); + assert!(logprobs.refusal.as_ref().unwrap()[0].top_logprobs.is_none()); + assert_eq!(serde_json::to_value(logprobs).unwrap(), wire); + } + + #[rstest] + #[case::text(json!({"type":"text","text":false}))] + #[case::missing_image(json!({"type":"image_url"}))] + #[case::audio_shape(json!({"type":"input_audio","input_audio":{"data":7,"format":"wav"}}))] + #[case::audio_missing_format(json!({"type":"input_audio","input_audio":{"data":"AA=="}}))] + #[case::image_missing_url(json!({"type":"image_url","image_url":{"detail":"high"}}))] + #[case::file_metadata(json!({"type":"file","file":{"video_metadata":{"fps":"fast"}}}))] + #[case::document_source(json!({"type":"document","source":{"type":"url","url":false}}))] + #[case::refusal_missing_text(json!({"type":"refusal"}))] + #[case::file_missing_file(json!({"type":"file"}))] + #[case::document_missing_source(json!({"type":"document"}))] + #[case::video_missing_url(json!({"type":"video_url"}))] + #[case::cache_control_shape(json!({"type":"text","text":"t","cache_control":"ephemeral"}))] + #[case::breakpoint_missing_mode(json!({"type":"text","text":"t","prompt_cache_breakpoint":{}}))] + #[case::breakpoint_unknown_mode(json!({"type":"text","text":"t","prompt_cache_breakpoint":{"mode":"auto"}}))] + #[case::unknown_tag(json!({"type":"future"}))] + fn content_parts_reject_malformed_typed_fields(#[case] wire: Value) { + assert!(serde_json::from_value::(wire).is_err()); + } + + #[rstest] + fn partial_file_preserves_extensions_and_omits_null_optionals() { + let wire = json!({"type":"file","file":{"file_id":null,"filename":"a.pdf","extension":[1,null]},"future":true}); + let part: ChatContentPart = serde_json::from_value(wire).unwrap(); + let ChatContentPart::File { + file, + prompt_cache_breakpoint: None, + extra, + } = &part + else { + panic!("expected file") + }; + assert!(file.file_id.is_none()); + assert!(file.file_data.is_none()); + assert_eq!(file.filename.as_deref(), Some("a.pdf")); + assert_eq!(extra["future"], json!(true)); + assert_eq!( + serde_json::to_value(part).unwrap(), + json!({"type":"file","file":{"filename":"a.pdf","extension":[1,null]},"future":true}) + ); + } + + #[rstest] + #[case::missing_token(json!({"logprob":-1}))] + #[case::missing_logprob(json!({"token":"hi"}))] + #[case::wrong_bytes(json!({"token":"hi","logprob":-1,"bytes":[256]}))] + fn token_logprobs_reject_malformed_fields(#[case] wire: Value) { + assert!( + serde_json::from_value::(wire) + .is_err() + ); + } +} diff --git a/litellm-rust/crates/llms-types/src/formats/chat_completions/mod.rs b/litellm-rust/crates/llms-types/src/formats/chat_completions/mod.rs new file mode 100644 index 00000000000..5db2d7f2530 --- /dev/null +++ b/litellm-rust/crates/llms-types/src/formats/chat_completions/mod.rs @@ -0,0 +1,9 @@ +mod content; +mod request; +mod response; +mod streaming; + +pub use content::*; +pub use request::*; +pub use response::*; +pub use streaming::*; diff --git a/litellm-rust/crates/llms-types/src/formats/chat_completions/request.rs b/litellm-rust/crates/llms-types/src/formats/chat_completions/request.rs new file mode 100644 index 00000000000..35d8d3a0110 --- /dev/null +++ b/litellm-rust/crates/llms-types/src/formats/chat_completions/request.rs @@ -0,0 +1,77 @@ +use serde_json::{Map, Value}; +use strum::IntoStaticStr; + +/// Reasoning effort level accepted or applied by the model. +#[macro_rules_attribute::apply(crate::wire_type)] +#[derive(Copy, Eq, IntoStaticStr, strum::EnumString, strum::VariantArray)] +#[serde(rename_all = "snake_case")] +pub enum ReasoningEffort { + #[strum(serialize = "none")] + None, + #[strum(serialize = "minimal")] + Minimal, + #[strum(serialize = "low")] + Low, + #[strum(serialize = "medium")] + Medium, + #[strum(serialize = "high")] + High, + #[strum(serialize = "xhigh")] + Xhigh, + #[strum(serialize = "max")] + Max, +} + +#[macro_rules_attribute::apply(crate::wire_type)] +#[serde(untagged)] +pub enum ChatMessageContent { + Text(String), + Parts(Vec), +} + +#[macro_rules_attribute::apply(crate::wire_type)] +pub struct ChatMessage { + pub role: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub content: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[cfg(test)] +mod tests { + use super::*; + use rstest::rstest; + use strum::VariantArray; + + #[rstest] + fn reasoning_effort_names_match_the_wire_and_parse_back( + #[values( + ReasoningEffort::None, + ReasoningEffort::Minimal, + ReasoningEffort::Low, + ReasoningEffort::Medium, + ReasoningEffort::High, + ReasoningEffort::Xhigh, + ReasoningEffort::Max + )] + effort: ReasoningEffort, + ) { + assert_eq!( + serde_json::to_value(effort).unwrap(), + Value::String(<&'static str>::from(effort).to_string()) + ); + assert_eq!(<&'static str>::from(effort).parse(), Ok(effort)); + assert!(ReasoningEffort::VARIANTS.contains(&effort)); + } + + #[rstest] + #[case::unknown("ultra")] + #[case::uppercase("HIGH")] + #[case::empty("")] + fn reasoning_effort_parse_rejects(#[case] value: &str) { + assert!(value.parse::().is_err()); + } +} diff --git a/litellm-rust/crates/llms-types/src/formats/chat_completions/response.rs b/litellm-rust/crates/llms-types/src/formats/chat_completions/response.rs new file mode 100644 index 00000000000..2ce07e3c465 --- /dev/null +++ b/litellm-rust/crates/llms-types/src/formats/chat_completions/response.rs @@ -0,0 +1,88 @@ +use serde_json::{Map, Value}; + +#[macro_rules_attribute::apply(crate::wire_type)] +pub struct ChatCompletionToolCallFunctionChunk { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + pub arguments: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider_specific_fields: Option>, +} + +#[macro_rules_attribute::apply(crate::wire_type)] +pub struct ChatCompletionToolCallChunk { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub id: Option, + #[serde(rename = "type")] + pub tool_type: String, + pub function: ChatCompletionToolCallFunctionChunk, + pub index: i64, +} + +#[macro_rules_attribute::apply(crate::wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ChatCompletionThinkingBlock { + Thinking { + #[serde(default, skip_serializing_if = "Option::is_none")] + thinking: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + signature: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + cache_control: Option, + }, + RedactedThinking { + #[serde(default, skip_serializing_if = "Option::is_none")] + data: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + cache_control: Option, + }, +} + +/// OpenAI `usage`, including the `prompt_tokens_details` split LiteLLM's Python +/// path reports so cost tracking sees the same numbers on either path. +#[macro_rules_attribute::apply(crate::wire_type)] +#[derive(Default)] +pub struct PromptTokensDetails { + pub cached_tokens: u64, + pub cache_creation_tokens: u64, + pub text_tokens: u64, +} + +#[macro_rules_attribute::apply(crate::wire_type)] +#[derive(Default)] +pub struct ChatCompletionsUsage { + pub prompt_tokens: u64, + pub completion_tokens: u64, + pub total_tokens: u64, + pub prompt_tokens_details: PromptTokensDetails, +} + +#[macro_rules_attribute::apply(crate::wire_type)] +pub struct ChatCompletionsChoiceMessage { + pub role: String, + // Whether an empty turn is `None` or `""` is the provider's choice, not a + // shared invariant: Anthropic's transform ends on `merged_text or None` + // while Converse assigns the joined string unconditionally. Each config + // mirrors its own, so keep this optional and serialize it even when None. + pub content: Option, +} + +#[macro_rules_attribute::apply(crate::wire_type)] +pub struct ChatCompletionsChoice { + pub index: u64, + pub message: ChatCompletionsChoiceMessage, + pub finish_reason: String, +} + +/// The normalized response handed back to the host. +/// +/// There is deliberately no `id`: Python mints the `chatcmpl-…` id on the +/// `ModelResponse` it already created, and echoing the provider's own id here +/// would change it. Pinned by `response_carries_no_id` in the Anthropic chat transformation tests. +#[macro_rules_attribute::apply(crate::wire_type)] +pub struct ChatCompletionsResponse { + pub created: u64, + pub model: String, + pub choices: Vec, + pub usage: ChatCompletionsUsage, +} diff --git a/litellm-rust/crates/llms-types/src/formats/chat_completions/streaming.rs b/litellm-rust/crates/llms-types/src/formats/chat_completions/streaming.rs new file mode 100644 index 00000000000..520b064d10d --- /dev/null +++ b/litellm-rust/crates/llms-types/src/formats/chat_completions/streaming.rs @@ -0,0 +1,48 @@ +use serde_json::{Map, Value}; + +use crate::formats::chat_completions::{ + ChatCompletionThinkingBlock, ChatCompletionToolCallChunk, ChatCompletionsUsage, +}; + +#[macro_rules_attribute::apply(crate::wire_type)] +#[derive(Default)] +pub struct ChatCompletionDelta { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub content: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub role: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tool_calls: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reasoning_content: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub thinking_blocks: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider_specific_fields: Option>, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(crate::wire_type)] +pub struct ChatCompletionStreamingChoice { + pub index: u64, + pub delta: ChatCompletionDelta, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub finish_reason: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub logprobs: Option, +} + +#[macro_rules_attribute::apply(crate::wire_type)] +pub struct ChatCompletionChunk { + pub id: String, + pub created: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub model: Option, + pub object: String, + pub choices: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub usage: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider_specific_fields: Option>, +} diff --git a/litellm-rust/crates/llms-types/src/formats/messages/AGENTS.md b/litellm-rust/crates/llms-types/src/formats/messages/AGENTS.md index 20a5632f267..febc648fcd6 100644 --- a/litellm-rust/crates/llms-types/src/formats/messages/AGENTS.md +++ b/litellm-rust/crates/llms-types/src/formats/messages/AGENTS.md @@ -1,19 +1,25 @@ -This directory owns shared Messages API data contracts and serialization: request and response bodies, messages, content blocks, usage, and stream-event payloads. Messages is a format independent of the provider that originated it. Keep one canonical definition of each shared contract here +# rules -Adapter contracts and execution inputs such as `MessagesTransformContext` belong in `llms/src/base_llm/messages`. Provider rewriting and interpretation belong in `llms/src//messages`. Call envelopes, live streams, and call orchestration belong in `inference-messages` - -Represent web-search results, encrypted-content fields, and thinking configuration as data here. Decisions to flatten results, remove encrypted content, select thinking budgets, or require beta headers belong to provider transformations. Data-shape validation belongs here, while model capability checks and request adaptation do not +- Messages originated at Anthropic and is served by many hosts. Types here describe the format, independent of any host +- Spoken by Anthropic, Bedrock InvokeModel, Vertex AI `rawPredict`, Azure AI Foundry, DeepSeek and MiniMax, each adapted in `llms/src//messages/` +- Hosts extend it with + - their own content blocks or media sources, which get typed in `providers//` once a consumer needs them (MiniMax image, video and mid-conversation system blocks) + - beta features gated by the `anthropic-beta` header, whose accepted values differ per host (`providers/anthropic/beta.rs`) +- Hosts deviate by + - moving `model` or the API version out of the body or into the URL (Bedrock, Vertex) + - framing the stream differently (AWS event stream on Bedrock instead of SSE) + - rejecting fields or discriminators (DeepSeek rejects `type: custom` tools and billing system blocks) + - Handle these in the provider adapter, never by loosening or forking a type here +- Request and response bodies, messages, content blocks, tools, usage and stream-event payloads live here + - Web-search results, encrypted-content fields and thinking configuration are data here + - Flattening results, stripping encrypted content, choosing thinking budgets and requiring betas are provider policy in `llms` +- `MessagesTransformContext` and other adapter inputs live in `llms/src/base_llm/messages/`. Call envelopes live in `inference-messages` +- Beta-only blocks and tools (MCP, compaction, tool changes, advisor, fallback, browser state, toolsets) follow the beta reference # references -These upstream contracts document fields and discriminators. They are documentation links, not saved golden snapshots. Beta-only blocks and tools (MCP, compaction, tool changes, advisor, fallback, browser state, and toolsets) follow the beta reference - -## Messages bodies, content, tools, and streaming - - https://platform.claude.com/docs/en/api/http/messages/create - https://platform.claude.com/docs/en/api/messages/create.md - https://platform.claude.com/docs/en/api/beta/messages/create.md - https://platform.claude.com/docs/en/build-with-claude/streaming.md - https://platform.claude.com/docs/en/build-with-claude/context-editing - -Anthropic-compatible hosts document their deviations in the `llms/src//messages` guides. Copy a host reference into `../../providers/AGENTS.md` only when that host gets a typed extension in `providers` diff --git a/litellm-rust/crates/llms-types/src/formats/messages/content.rs b/litellm-rust/crates/llms-types/src/formats/messages/content.rs index 595ba68c103..028198eafd3 100644 --- a/litellm-rust/crates/llms-types/src/formats/messages/content.rs +++ b/litellm-rust/crates/llms-types/src/formats/messages/content.rs @@ -710,3 +710,518 @@ pub enum FallbackTrigger { extra: Map, }, } + +#[cfg(test)] +mod tests { + + use crate::formats::messages::AdvisorToolResultContent; + + use crate::formats::messages::{BashCodeExecutionOutput, BashCodeExecutionToolResultContent}; + + use crate::formats::messages::{BlockContent, BrowserStateChange}; + + use crate::formats::messages::Citation; + + use crate::formats::messages::CodeExecutionOutput; + + use crate::formats::messages::CodeExecutionToolResultContent; + + use crate::formats::messages::ContentSource; + + use crate::formats::messages::FallbackTrigger; + + use crate::formats::messages::{McpToolResultContent, McpToolResultText}; + + use crate::formats::messages::MessagesContentPart; + + use crate::formats::messages::MessagesToolParam; + + use crate::formats::messages::{TextEditorCodeExecutionToolResultContent, TextEditorFileType}; + + use crate::formats::messages::{ToolCaller, ToolChange, ToolChangeTarget}; + + use crate::formats::messages::ToolSearchReference; + + use crate::formats::messages::ToolSearchToolResultContent; + + use crate::formats::messages::WebFetchDocument; + + use crate::formats::messages::WebFetchToolResultContent; + + use crate::formats::messages::WebSearchToolResultContent; + + use crate::test_support::*; + + use rstest::rstest; + + use serde_json::{Value, json}; + + #[rstest] + #[case::base64(json!({"type":"base64","media_type":"image/png","data":"AA=="}))] + #[case::url(json!({"type":"url","url":"https://example.test/image"}))] + #[case::file(json!({"type":"file","file_id":"file_1"}))] + #[case::text(json!({"type":"text","media_type":"text/plain","data":"document"}))] + #[case::content(json!({"type":"content","content":"nested"}))] + fn content_sources_round_trip(#[case] wire: Value) { + round_trip::(wire); + } + + fn part(wire: Value) -> MessagesContentPart { + round_trip::(wire) + } + + #[rstest] + fn text_blocks_expose_every_citation_location() { + let block = part(json!({ + "type":"text", + "text":"cited", + "citations":[ + {"type":"char_location","cited_text":"a","document_index":0,"start_char_index":1,"end_char_index":6,"file_id":"file_1"}, + {"type":"page_location","cited_text":"b","document_index":1,"start_page_number":1,"end_page_number":2}, + {"type":"content_block_location","cited_text":"c","document_index":2,"start_block_index":0,"end_block_index":1}, + {"type":"web_search_result_location","cited_text":"d","url":"https://example.test","encrypted_index":"opaque","title":"result"}, + {"type":"search_result_location","cited_text":"e","search_result_index":3,"source":"kb","start_block_index":0,"end_block_index":2} + ], + "cache_control":{"type":"ephemeral"} + })); + let MessagesContentPart::Text(text) = &block else { + panic!("expected text block"); + }; + assert_eq!(text.text, "cited"); + let [ + Citation::CharLocation(chars), + Citation::PageLocation(page), + Citation::ContentBlockLocation(blocks), + Citation::WebSearchResultLocation(search), + Citation::SearchResultLocation(result), + ] = text.citations.as_deref().unwrap() + else { + panic!("expected one citation of each location type"); + }; + assert_eq!((chars.start_char_index, chars.end_char_index), (1, 6)); + assert_eq!(chars.file_id.as_deref(), Some("file_1")); + assert_eq!((page.start_page_number, page.end_page_number), (1, 2)); + assert_eq!(blocks.document_index, 2); + assert_eq!(search.encrypted_index, "opaque"); + assert_eq!(result.search_result_index, 3); + assert_eq!(result.source, "kb"); + } + + #[rstest] + fn text_block_null_optionals_are_omitted() { + let block: MessagesContentPart = serde_json::from_value( + json!({"type":"text","text":"hi","citations":null,"cache_control":null,"future":null}), + ) + .unwrap(); + let MessagesContentPart::Text(text) = &block else { + panic!("expected text block"); + }; + assert!(text.citations.is_none()); + assert!(text.cache_control.is_none()); + assert_eq!(text.extra.get("future"), Some(&Value::Null)); + assert_eq!( + serde_json::to_value(block).unwrap(), + json!({"type":"text","text":"hi","future":null}) + ); + } + + #[rstest] + fn request_media_blocks_expose_sources() { + let MessagesContentPart::Image(image) = part(json!({ + "type":"image", + "source":{"type":"base64","media_type":"image/png","data":"AA=="}, + "transformations":{"oversized_image":"error"} + })) else { + panic!("expected image block"); + }; + assert!( + matches!(&image.source, ContentSource::Base64 { media_type, .. } if media_type == "image/png") + ); + assert_eq!(image.extra["transformations"]["oversized_image"], "error"); + let MessagesContentPart::Document(document) = part(json!({ + "type":"document", + "source":{"type":"content","content":[ + {"type":"text","text":"Section 1"}, + {"type":"image","source":{"type":"url","url":"https://example.test/chart.png"}} + ]}, + "title":"Q3 report", + "context":"quarterly", + "citations":{"enabled":true} + })) else { + panic!("expected document block"); + }; + assert_eq!(document.title.as_deref(), Some("Q3 report")); + assert_eq!(document.citations.as_ref().unwrap().enabled, Some(true)); + let ContentSource::Content { + content: BlockContent::Blocks(blocks), + .. + } = &document.source + else { + panic!("expected content-block source"); + }; + assert!(matches!( + blocks.as_slice(), + [MessagesContentPart::Text(_), MessagesContentPart::Image(_)] + )); + let MessagesContentPart::SearchResult(search) = part(json!({ + "type":"search_result", + "source":"https://example.test/result", + "title":"result", + "content":[{"type":"text","text":"found"}], + "citations":{"enabled":false} + })) else { + panic!("expected search result block"); + }; + assert_eq!(search.source, "https://example.test/result"); + assert!( + matches!(search.content.as_slice(), [MessagesContentPart::Text(text)] if text.text == "found") + ); + } + + #[rstest] + fn reasoning_and_tool_blocks_expose_required_fields() { + let MessagesContentPart::Thinking(thinking) = + part(json!({"type":"thinking","thinking":"plan","signature":"sig"})) + else { + panic!("expected thinking block"); + }; + assert_eq!( + (thinking.thinking.as_str(), thinking.signature.as_str()), + ("plan", "sig") + ); + let MessagesContentPart::RedactedThinking(redacted) = + part(json!({"type":"redacted_thinking","data":"opaque"})) + else { + panic!("expected redacted thinking block"); + }; + assert_eq!(redacted.data, "opaque"); + let MessagesContentPart::ToolUse(tool_use) = part(json!({ + "type":"tool_use", + "id":"toolu_1", + "name":"lookup", + "input":{"query":[1,null]}, + "caller":{"type":"code_execution_20260120","tool_id":"srvtoolu_1"}, + "toolset_name":"browser" + })) else { + panic!("expected tool use block"); + }; + assert_eq!(tool_use.input["query"], json!([1, null])); + assert!( + matches!(&tool_use.caller, Some(ToolCaller::CodeExecution20260120 { tool_id, .. }) if tool_id == "srvtoolu_1") + ); + assert_eq!(tool_use.toolset_name.as_deref(), Some("browser")); + let MessagesContentPart::ToolResult(result) = part(json!({ + "type":"tool_result", + "tool_use_id":"toolu_1", + "is_error":false, + "content":[ + {"type":"tool_reference","tool_name":"lookup"}, + {"type":"browser_state","tabs":[{"tab_id":"1","title":"","url":"","active":true}], + "state_changes":[{"type":"download_completed","download_id":"d1","url":"https://example.test/f","size_bytes":3}]} + ] + })) else { + panic!("expected tool result block"); + }; + let Some(BlockContent::Blocks(blocks)) = &result.content else { + panic!("expected nested result blocks"); + }; + let [ + MessagesContentPart::ToolReference(reference), + MessagesContentPart::BrowserState(browser), + ] = blocks.as_slice() + else { + panic!("expected tool reference and browser state"); + }; + assert_eq!(reference.tool_name, "lookup"); + assert_eq!(browser.tabs[0].active, Some(true)); + assert!(matches!( + browser.state_changes.as_deref(), + Some([BrowserStateChange::DownloadCompleted { + size_bytes: Some(3), + path: None, + .. + }]) + )); + let MessagesContentPart::ToolResult(text_result) = + part(json!({"type":"tool_result","tool_use_id":"toolu_2","content":"done"})) + else { + panic!("expected tool result block"); + }; + assert_eq!(text_result.content, Some(BlockContent::Text("done".into()))); + assert!(text_result.is_error.is_none()); + } + + #[rstest] + fn server_tool_results_expose_nested_result_unions() { + let MessagesContentPart::ServerToolUse(server) = part( + json!({"type":"server_tool_use","id":"srvtoolu_1","name":"web_search","input":{"query":"rust"}}), + ) else { + panic!("expected server tool use"); + }; + assert_eq!(server.name, "web_search"); + let MessagesContentPart::WebSearchToolResult(search) = part(json!({ + "type":"web_search_tool_result", + "tool_use_id":"srvtoolu_1", + "content":[{"type":"web_search_result","url":"https://example.test","title":"t","encrypted_content":"e","page_age":"1d"}] + })) else { + panic!("expected web search result"); + }; + let WebSearchToolResultContent::Results(results) = &search.content else { + panic!("expected search results"); + }; + assert_eq!(results[0].page_age.as_deref(), Some("1d")); + let MessagesContentPart::WebSearchToolResult(search_error) = part(json!({ + "type":"web_search_tool_result", + "tool_use_id":"srvtoolu_1", + "content":{"type":"web_search_tool_result_error","error_code":"max_uses_exceeded"} + })) else { + panic!("expected web search error"); + }; + assert!( + matches!(&search_error.content, WebSearchToolResultContent::Error(error) if error.error_code == "max_uses_exceeded") + ); + let MessagesContentPart::WebFetchToolResult(fetch) = part(json!({ + "type":"web_fetch_tool_result", + "tool_use_id":"srvtoolu_2", + "content":{"type":"web_fetch_result","url":"https://example.test","retrieved_at":"2026-01-01T00:00:00Z", + "content":{"type":"document","source":{"type":"text","media_type":"text/plain","data":"page"}}} + })) else { + panic!("expected web fetch result"); + }; + let WebFetchToolResultContent::WebFetchResult(fetched) = &fetch.content else { + panic!("expected fetched page"); + }; + let WebFetchDocument::Document(document) = &fetched.content; + assert!(matches!(&document.source, ContentSource::Text { data, .. } if data == "page")); + assert!(fetched.extra.is_empty()); + let MessagesContentPart::CodeExecutionToolResult(code) = part(json!({ + "type":"code_execution_tool_result", + "tool_use_id":"srvtoolu_3", + "content":{"type":"encrypted_code_execution_result","encrypted_stdout":"opaque","stderr":"","return_code":-1, + "content":[{"type":"code_execution_output","file_id":"file_1"}]} + })) else { + panic!("expected code execution result"); + }; + let CodeExecutionToolResultContent::EncryptedCodeExecutionResult(encrypted) = &code.content + else { + panic!("expected encrypted execution result"); + }; + assert_eq!(encrypted.return_code, -1); + assert!( + matches!(encrypted.content.as_slice(), [CodeExecutionOutput::CodeExecutionOutput { file_id, .. }] if file_id == "file_1") + ); + let MessagesContentPart::BashCodeExecutionToolResult(bash) = part(json!({ + "type":"bash_code_execution_tool_result", + "tool_use_id":"srvtoolu_7", + "content":{"type":"bash_code_execution_result","stdout":"ok","stderr":"","return_code":0, + "content":[{"type":"bash_code_execution_output","file_id":"file_2"}]} + })) else { + panic!("expected bash code execution result"); + }; + let BashCodeExecutionToolResultContent::BashCodeExecutionResult(ran) = &bash.content else { + panic!("expected bash execution result"); + }; + assert_eq!((ran.stdout.as_str(), ran.return_code), ("ok", 0)); + assert!( + matches!(ran.content.as_slice(), [BashCodeExecutionOutput::BashCodeExecutionOutput { file_id, .. }] if file_id == "file_2") + ); + assert!(ran.extra.is_empty()); + let MessagesContentPart::TextEditorCodeExecutionToolResult(editor) = part(json!({ + "type":"text_editor_code_execution_tool_result", + "tool_use_id":"srvtoolu_4", + "content":{"type":"text_editor_code_execution_view_result","content":"fn main() {}","file_type":"text","num_lines":1} + })) else { + panic!("expected text editor result"); + }; + let TextEditorCodeExecutionToolResultContent::TextEditorCodeExecutionViewResult(view) = + &editor.content + else { + panic!("expected view result"); + }; + assert_eq!(view.file_type, TextEditorFileType::Text); + assert_eq!(view.num_lines, Some(1)); + assert!(view.start_line.is_none()); + let MessagesContentPart::ToolSearchToolResult(tool_search) = part(json!({ + "type":"tool_search_tool_result", + "tool_use_id":"srvtoolu_5", + "content":{"type":"tool_search_tool_result_error","error_code":"unavailable","error_message":"down"} + })) else { + panic!("expected tool search result"); + }; + assert!( + matches!(&tool_search.content, ToolSearchToolResultContent::ToolSearchToolResultError(error) if error.error_message.as_deref() == Some("down")) + ); + let MessagesContentPart::ToolSearchToolResult(found) = part(json!({ + "type":"tool_search_tool_result", + "tool_use_id":"srvtoolu_8", + "content":{"type":"tool_search_tool_search_result","tool_references":[{"type":"tool_reference","tool_name":"lookup"}]} + })) else { + panic!("expected tool search result"); + }; + let ToolSearchToolResultContent::ToolSearchToolSearchResult(result) = &found.content else { + panic!("expected tool search hits"); + }; + let [ToolSearchReference::ToolReference(reference)] = result.tool_references.as_slice() + else { + panic!("expected one tool reference"); + }; + assert_eq!(reference.tool_name, "lookup"); + assert!(reference.extra.is_empty() && result.extra.is_empty()); + let MessagesContentPart::AdvisorToolResult(advisor) = part(json!({ + "type":"advisor_tool_result", + "tool_use_id":"srvtoolu_6", + "content":{"type":"advisor_result","text":"advice"} + })) else { + panic!("expected advisor result"); + }; + assert!( + matches!(&advisor.content, AdvisorToolResultContent::AdvisorResult { text, stop_reason: None, .. } if text == "advice") + ); + } + + #[rstest] + fn beta_blocks_expose_mcp_compaction_and_fallback_fields() { + let MessagesContentPart::McpToolUse(mcp) = part( + json!({"type":"mcp_tool_use","id":"mcptoolu_1","name":"search","server_name":"kb","input":{}}), + ) else { + panic!("expected MCP tool use"); + }; + assert_eq!(mcp.server_name, "kb"); + let MessagesContentPart::McpToolResult(mcp_result) = part( + json!({"type":"mcp_tool_result","tool_use_id":"mcptoolu_1","is_error":true,"content":"failed"}), + ) else { + panic!("expected MCP tool result"); + }; + assert_eq!(mcp_result.is_error, Some(true)); + assert_eq!( + mcp_result.content, + Some(McpToolResultContent::Text("failed".into())) + ); + let MessagesContentPart::McpToolResult(text_result) = part(json!({ + "type":"mcp_tool_result", + "tool_use_id":"mcptoolu_3", + "content":[{"type":"text","text":"found"}] + })) else { + panic!("expected MCP tool result"); + }; + let Some(McpToolResultContent::Blocks(blocks)) = &text_result.content else { + panic!("expected MCP text blocks"); + }; + let [McpToolResultText::Text(text)] = blocks.as_slice() else { + panic!("expected one text block"); + }; + assert_eq!(text.text, "found"); + let MessagesContentPart::McpToolResult(empty_result) = + part(json!({"type":"mcp_tool_result","tool_use_id":"mcptoolu_2"})) + else { + panic!("expected MCP tool result"); + }; + assert!(empty_result.content.is_none()); + assert!(empty_result.extra.is_empty()); + let MessagesContentPart::McpToolListing(listing) = part(json!({ + "type":"mcp_tool_listing", + "mcp_server_name":"kb", + "tools":[{"name":"search","input_schema":{"type":"object"}}] + })) else { + panic!("expected MCP tool listing"); + }; + assert!(listing.tools[0].description.is_none()); + let MessagesContentPart::Compaction(compaction) = part(json!({ + "type":"compaction", + "content":"summary", + "encrypted_content":"opaque", + "tool_changes":[ + {"type":"tool_addition","tool":{"type":"tool_definition","definition":{"name":"lookup","input_schema":{"type":"object"}}}}, + {"type":"tool_removal","tool":{"type":"mcp_tool_reference","server_name":"kb","name":"search"}} + ] + })) else { + panic!("expected compaction block"); + }; + let [ + ToolChange::ToolAddition(addition), + ToolChange::ToolRemoval(removal), + ] = compaction.tool_changes.as_deref().unwrap() + else { + panic!("expected tool addition and removal"); + }; + let ToolChangeTarget::ToolDefinition { definition, .. } = &addition.tool else { + panic!("expected inline tool definition"); + }; + assert!( + matches!(definition.as_ref(), MessagesToolParam::Custom(tool) if tool.name == "lookup") + ); + assert!( + matches!(&removal.tool, ToolChangeTarget::McpToolReference { server_name, .. } if server_name == "kb") + ); + let failed: MessagesContentPart = + serde_json::from_value(json!({"type":"compaction","content":null})).unwrap(); + assert_eq!(failed, MessagesContentPart::Compaction(Default::default())); + let MessagesContentPart::Fallback(fallback) = part(json!({ + "type":"fallback", + "from":{"model":"claude-opus-5-5"}, + "to":{"model":"claude-sonnet-5-5"}, + "trigger":{"type":"refusal","category":"cyber"} + })) else { + panic!("expected fallback block"); + }; + assert_eq!(fallback.from.model, "claude-opus-5-5"); + assert_eq!(fallback.to.model, "claude-sonnet-5-5"); + let Some(FallbackTrigger::Refusal { category, extra }) = &fallback.trigger else { + panic!("expected refusal trigger"); + }; + assert_eq!(category.as_deref(), Some("cyber")); + assert!(extra.is_empty() && fallback.extra.is_empty()); + let MessagesContentPart::Fallback(untriggered) = part(json!({ + "type":"fallback", + "from":{"model":"claude-opus-5-5"}, + "to":{"model":"claude-sonnet-5-5"} + })) else { + panic!("expected fallback block"); + }; + assert!(untriggered.trigger.is_none()); + } + + #[rstest] + #[case::missing_tag(json!({"text":"hello"}))] + #[case::unknown_tag(json!({"type":"future_block","text":"hello"}))] + #[case::text_without_text(json!({"type":"text"}))] + #[case::wrong_text(json!({"type":"text","text":7}))] + #[case::citation_missing_location(json!({"type":"text","text":"a","citations":[{"type":"char_location","cited_text":"a","document_index":0}]}))] + #[case::unknown_citation(json!({"type":"text","text":"a","citations":[{"type":"future_location"}]}))] + #[case::thinking_without_signature(json!({"type":"thinking","thinking":"plan"}))] + #[case::tool_use_without_id(json!({"type":"tool_use","name":"lookup","input":{}}))] + #[case::tool_use_array_input(json!({"type":"tool_use","id":"t","name":"lookup","input":[]}))] + #[case::tool_result_without_id(json!({"type":"tool_result","content":"done"}))] + #[case::malformed_nested_block(json!({"type":"tool_result","tool_use_id":"t","content":[{"type":"text","text":7}]}))] + #[case::malformed_source(json!({"type":"document","source":{"type":"url","url":7}}))] + #[case::image_without_source(json!({"type":"image"}))] + #[case::search_result_without_title(json!({"type":"search_result","source":"s","content":[]}))] + #[case::web_search_result_without_url(json!({"type":"web_search_tool_result","tool_use_id":"t","content":[{"type":"web_search_result","title":"t","encrypted_content":"e"}]}))] + #[case::unknown_fetch_result(json!({"type":"web_fetch_tool_result","tool_use_id":"t","content":{"type":"future"}}))] + #[case::fractional_return_code(json!({"type":"bash_code_execution_tool_result","tool_use_id":"t","content":{"type":"bash_code_execution_result","stdout":"","stderr":"","return_code":0.5,"content":[]}}))] + #[case::unknown_file_type(json!({"type":"text_editor_code_execution_tool_result","tool_use_id":"t","content":{"type":"text_editor_code_execution_view_result","content":"","file_type":"video"}}))] + #[case::negative_line_count(json!({"type":"text_editor_code_execution_tool_result","tool_use_id":"t","content":{"type":"text_editor_code_execution_str_replace_result","new_lines":-1}}))] + #[case::unknown_tool_change(json!({"type":"compaction","tool_changes":[{"type":"tool_addition","tool":{"type":"future"}}]}))] + #[case::tool_search_non_reference(json!({"type":"tool_search_tool_result","tool_use_id":"t","content":{"type":"tool_search_tool_search_result","tool_references":[{"type":"text","text":"lookup"}]}}))] + #[case::web_fetch_text_content(json!({"type":"web_fetch_tool_result","tool_use_id":"t","content":{"type":"web_fetch_result","url":"https://example.test","content":{"type":"text","text":"page"}}}))] + #[case::bash_result_with_code_output(json!({"type":"bash_code_execution_tool_result","tool_use_id":"t","content":{"type":"bash_code_execution_result","stdout":"","stderr":"","return_code":0,"content":[{"type":"code_execution_output","file_id":"f"}]}}))] + #[case::code_result_with_bash_output(json!({"type":"code_execution_tool_result","tool_use_id":"t","content":{"type":"code_execution_result","stdout":"","stderr":"","return_code":0,"content":[{"type":"bash_code_execution_output","file_id":"f"}]}}))] + #[case::fallback_unknown_trigger(json!({"type":"fallback","from":{"model":"a"},"to":{"model":"b"},"trigger":{"type":"overload"}}))] + #[case::unknown_state_change(json!({"type":"browser_state","tabs":[],"state_changes":[{"type":"tab_closed","tab_id":"1"}]}))] + fn content_parts_reject_malformed_known_fields(#[case] wire: Value) { + assert!(serde_json::from_value::(wire).is_err()); + } + + #[rstest] + #[case::image(json!([{"type":"image","source":{"type":"url","url":"https://example.test/a.png"}}]))] + #[case::tool_reference(json!([{"type":"tool_reference","tool_name":"lookup"}]))] + #[case::untagged(json!([{"text":"found"}]))] + fn mcp_tool_result_content_rejects_non_text_blocks(#[case] content: Value) { + assert!( + serde_json::from_value::( + json!({"type":"mcp_tool_result","tool_use_id":"mcptoolu_1","content":content}) + ) + .is_err() + ); + } +} diff --git a/litellm-rust/crates/llms-types/src/formats/messages/metadata.rs b/litellm-rust/crates/llms-types/src/formats/messages/metadata.rs index 6e5ebb2b98d..8e92c4ce3d3 100644 --- a/litellm-rust/crates/llms-types/src/formats/messages/metadata.rs +++ b/litellm-rust/crates/llms-types/src/formats/messages/metadata.rs @@ -338,3 +338,293 @@ pub struct CacheMissedTokens { #[serde(flatten)] pub extra: Map, } + +#[cfg(test)] +mod tests { + + use crate::formats::messages::CacheMissReason; + + use crate::formats::messages::ContainerReference; + + use crate::formats::messages::ContextManagementResponse; + + use crate::formats::messages::McpServer; + + use crate::formats::messages::{ + MessageRole, MessageType, MessagesCompaction, MessagesContainer, + }; + + use crate::formats::messages::{ + MessagesDiagnostics, MessagesDiagnosticsParam, MessagesMetadata, + }; + + use crate::formats::messages::OutputFormat; + + use crate::formats::messages::Safeguard; + + use crate::formats::messages::{SkillType, StopDetails, StopDetailsType, StopReason}; + + use crate::json_schema::JsonSchema; + + use crate::{recognized::Recognized, test_support::*}; + + use rstest::rstest; + + use serde_json::{Value, json}; + + #[rstest] + fn metadata_contracts_round_trip() { + let metadata = round_trip::(json!({"user_id":"user_1","future":true})); + assert_eq!(metadata.user_id.as_deref(), Some("user_1")); + assert_eq!(metadata.extra.get("future"), Some(&json!(true))); + let output_format = round_trip::(json!({ + "type":"json_schema", + "schema":{"type":"object","properties":{"name":{"type":"string"}}}, + "strict":true + })); + assert_eq!(output_format.strict, Some(true)); + assert!(output_format.extra.is_empty()); + let JsonSchema::Object(schema) = &output_format.schema else { + panic!("expected output object schema"); + }; + assert!(schema.properties.as_ref().unwrap().contains_key("name")); + let compaction = + round_trip::(json!({"type":"summarize","instructions":"briefly"})); + assert_eq!(compaction.instructions.as_deref(), Some("briefly")); + assert!(compaction.extra.is_empty()); + let container = round_trip::(json!({ + "id":"container_1", + "expires_at":"2026-01-01T00:00:00Z", + "skills":[{"type":"custom","skill_id":"skill_1","version":"1"}] + })); + assert_eq!(container.id.as_deref(), Some("container_1")); + assert!(container.extra.is_empty()); + let [skill] = container.skills.as_ref().unwrap().as_slice() else { + panic!("expected container skill"); + }; + assert_eq!(skill.skill_type, SkillType::Custom); + assert_eq!(skill.skill_id, "skill_1"); + assert_eq!(skill.version.as_deref(), Some("1")); + assert!(skill.extra.is_empty()); + let reference = round_trip::(json!({"id":"container_1"})); + let ContainerReference::Parameters(parameters) = reference else { + panic!("expected container parameters"); + }; + assert_eq!(parameters.id.as_deref(), container.id.as_deref()); + assert!(parameters.skills.is_none()); + round_trip::( + json!({"id":"container_1","skills":[{"type":"anthropic","skill_id":"pptx"}]}), + ); + let server = round_trip::(json!({ + "type":"url", + "url":"https://example.test/mcp", + "name":"search", + "authorization_token":"token", + "tool_configuration":{"allowed_tools":["search"],"enabled":true} + })); + assert_eq!(server.url, "https://example.test/mcp"); + assert_eq!(server.name, "search"); + assert_eq!(server.authorization_token.as_deref(), Some("token")); + assert!(server.extra.is_empty()); + let configuration = server.tool_configuration.as_ref().unwrap(); + assert_eq!(configuration.enabled, Some(true)); + assert_eq!( + configuration.allowed_tools.as_deref(), + Some([String::from("search")].as_slice()) + ); + assert!(configuration.extra.is_empty()); + let context = round_trip::(json!({ + "applied_edits":[ + {"type":"clear_tool_uses_20250919","cleared_input_tokens":7,"cleared_tool_uses":2}, + {"type":"clear_thinking_20251015","cleared_input_tokens":3,"cleared_thinking_turns":1} + ] + })); + let [tool_uses, thinking] = context.applied_edits.as_ref().unwrap().as_slice() else { + panic!("expected two applied edits"); + }; + assert_eq!( + tool_uses.edit_type.as_deref(), + Some("clear_tool_uses_20250919") + ); + assert_eq!( + (tool_uses.cleared_input_tokens, tool_uses.cleared_tool_uses), + (Some(7), Some(2)) + ); + assert!(tool_uses.cleared_thinking_turns.is_none()); + assert_eq!( + ( + thinking.cleared_input_tokens, + thinking.cleared_thinking_turns + ), + (Some(3), Some(1)) + ); + assert!(tool_uses.extra.is_empty() && thinking.extra.is_empty()); + let safeguard = round_trip::( + json!({"type":"classifier","classifier_context":{"source":"test"}}), + ); + assert_eq!(safeguard.safeguard_type, "classifier"); + assert_eq!( + safeguard.classifier_context.as_ref().unwrap().get("source"), + Some(&json!("test")) + ); + assert!(safeguard.extra.is_empty()); + } + + #[rstest] + fn stop_details_expose_refusal_and_keep_unknown_types() { + let refusal = round_trip::( + json!({"type":"refusal","category":"cyber","explanation":"blocked"}), + ); + assert_eq!( + refusal.detail_type, + Recognized::Known(StopDetailsType::Refusal) + ); + assert_eq!(refusal.category.as_deref(), Some("cyber")); + assert_eq!(refusal.explanation.as_deref(), Some("blocked")); + assert!(refusal.extra.is_empty()); + let safeguard = + round_trip::(json!({"type":"safeguard","safeguard_types":["classifier"]})); + assert_eq!( + safeguard.detail_type, + Recognized::Unrecognized(json!("safeguard")) + ); + assert_eq!(safeguard.extra["safeguard_types"], json!(["classifier"])); + } + + #[rstest] + fn container_rejects_skill_without_id() { + assert!( + serde_json::from_value::( + json!({"skills":[{"type":"custom","version":"1"}]}) + ) + .is_err() + ); + } + + #[rstest] + #[case::without_url(json!({"type":"url","name":"search"}))] + #[case::without_name(json!({"type":"url","url":"https://example.test/mcp"}))] + fn mcp_server_rejects_missing_required_fields(#[case] wire: Value) { + assert!(serde_json::from_value::(wire).is_err()); + } + + #[rstest] + fn output_format_rejects_missing_schema() { + assert!(serde_json::from_value::(json!({"type":"json_schema"})).is_err()); + } + + #[rstest] + #[case::user(json!("user"))] + #[case::assistant(json!("assistant"))] + #[case::system(json!("system"))] + #[case::future(json!("future_role"))] + fn message_roles_round_trip(#[case] wire: Value) { + round_trip::(wire); + } + + #[rstest] + #[case::message(json!("message"))] + #[case::future(json!("future_type"))] + fn message_types_round_trip(#[case] wire: Value) { + round_trip::(wire); + } + + #[rstest] + #[case::end_turn(json!("end_turn"))] + #[case::refusal(json!("refusal"))] + #[case::compaction(json!("compaction"))] + #[case::future(json!("future_reason"))] + fn stop_reasons_round_trip(#[case] wire: Value) { + round_trip::(wire); + } + + #[rstest] + #[case::model(json!({"type":"model_changed","cache_missed_input_tokens":12}), Some(12))] + #[case::system(json!({"type":"system_changed","cache_missed_input_tokens":3}), Some(3))] + #[case::tools(json!({"type":"tools_changed","cache_missed_input_tokens":0}), Some(0))] + #[case::messages(json!({"type":"messages_changed","cache_missed_input_tokens":9}), Some(9))] + #[case::not_found(json!({"type":"previous_message_not_found"}), None)] + #[case::unavailable(json!({"type":"unavailable"}), None)] + fn diagnostics_expose_cache_miss_reason(#[case] reason: Value, #[case] missed: Option) { + let diagnostics = round_trip::(json!({"cache_miss_reason":reason})); + let Some(Recognized::Known(reason)) = &diagnostics.cache_miss_reason else { + panic!("expected known cache miss reason"); + }; + let actual = match reason { + CacheMissReason::ModelChanged(tokens) + | CacheMissReason::SystemChanged(tokens) + | CacheMissReason::ToolsChanged(tokens) + | CacheMissReason::MessagesChanged(tokens) => { + assert!(tokens.extra.is_empty()); + Some(tokens.cache_missed_input_tokens) + } + CacheMissReason::PreviousMessageNotFound { extra } + | CacheMissReason::Unavailable { extra } => { + assert!(extra.is_empty()); + None + } + }; + assert_eq!(actual, missed); + assert!(diagnostics.extra.is_empty()); + } + + #[rstest] + #[case::model(json!({"type":"model_changed","cache_missed_input_tokens":12}), "model_changed")] + #[case::system(json!({"type":"system_changed","cache_missed_input_tokens":12}), "system_changed")] + #[case::tools(json!({"type":"tools_changed","cache_missed_input_tokens":12}), "tools_changed")] + #[case::messages(json!({"type":"messages_changed","cache_missed_input_tokens":12}), "messages_changed")] + fn cache_miss_reason_variants_keep_their_tags(#[case] reason: Value, #[case] tag: &str) { + let parsed: CacheMissReason = serde_json::from_value(reason).unwrap(); + assert_eq!(serde_json::to_value(parsed).unwrap()["type"], json!(tag)); + } + + #[rstest] + #[case::pending(json!({"cache_miss_reason":null}))] + #[case::future_reason(json!({"cache_miss_reason":{"type":"future_changed","cache_missed_input_tokens":1}}))] + #[case::missing_tokens(json!({"cache_miss_reason":{"type":"model_changed"}}))] + fn diagnostics_keep_pending_and_unmodeled_reasons(#[case] wire: Value) { + let diagnostics: MessagesDiagnostics = serde_json::from_value(wire.clone()).unwrap(); + match &diagnostics.cache_miss_reason { + None => assert_eq!(serde_json::to_value(&diagnostics).unwrap(), json!({})), + Some(Recognized::Unrecognized(kept)) => { + assert_eq!(kept, &wire["cache_miss_reason"]); + assert_eq!(serde_json::to_value(&diagnostics).unwrap(), wire); + } + Some(Recognized::Known(reason)) => panic!("unexpected known reason {reason:?}"), + } + } + + #[rstest] + #[case::previous(json!({"previous_message_id":"msg_1"}), Some(Some("msg_1")))] + #[case::first_turn(json!({"previous_message_id":null}), Some(None))] + #[case::absent(json!({}), None)] + fn diagnostics_param_distinguishes_null_from_absent_previous_message( + #[case] wire: Value, + #[case] expected: Option>, + ) { + let param = round_trip::(wire); + assert_eq!( + param.previous_message_id.as_ref().map(Option::as_deref), + expected + ); + assert!(param.extra.is_empty()); + } + + #[rstest] + fn diagnostics_param_rejects_non_string_previous_message() { + assert!( + serde_json::from_value::(json!({"previous_message_id":7})) + .is_err() + ); + } + + #[rstest] + fn cache_miss_reason_rejects_non_numeric_tokens() { + assert!( + serde_json::from_value::( + json!({"type":"model_changed","cache_missed_input_tokens":"many"}) + ) + .is_err() + ); + } +} diff --git a/litellm-rust/crates/llms-types/src/formats/messages/mod.rs b/litellm-rust/crates/llms-types/src/formats/messages/mod.rs index 08695ca7ecc..e45b02a4f83 100644 --- a/litellm-rust/crates/llms-types/src/formats/messages/mod.rs +++ b/litellm-rust/crates/llms-types/src/formats/messages/mod.rs @@ -6,49 +6,9 @@ pub mod streaming; mod tools; mod usage; -pub use content::{ - AdvisorToolResultContent, BashCodeExecutionOutput, BashCodeExecutionToolResultContent, - BlockContent, BrowserStateBlock, BrowserStateChange, BrowserTab, CharCitation, Citation, - CitationsConfig, CodeExecutionOutput, CodeExecutionResult, CodeExecutionToolResultContent, - CompactionBlock, ContainerUploadBlock, ContentBlockCitation, ContentSource, DocumentBlock, - EncryptedCodeExecutionResult, FallbackBlock, FallbackModel, FallbackTrigger, ImageBlock, - McpListedTool, McpToolListingBlock, McpToolResultBlock, McpToolResultContent, - McpToolResultText, McpToolUseBlock, MessagesContentPart, PageCitation, RedactedThinkingBlock, - SearchResultBlock, SearchResultCitation, ServerToolError, ServerToolResultBlock, - ServerToolUseBlock, TextBlock, TextEditorCodeExecutionToolResultContent, - TextEditorCreateResult, TextEditorFileType, TextEditorStrReplaceResult, TextEditorViewResult, - ThinkingBlock, ToolCaller, ToolChange, ToolChangeBlock, ToolChangeTarget, ToolReferenceBlock, - ToolResultBlock, ToolSearchReference, ToolSearchResult, ToolSearchToolResultContent, - ToolUseBlock, WebFetchDocument, WebFetchResult, WebFetchToolResultContent, WebSearchCitation, - WebSearchResult, WebSearchResultError, WebSearchResultErrorType, WebSearchResultType, - WebSearchToolResultContent, -}; -pub use metadata::{ - AppliedEdit, CacheMissReason, CacheMissedTokens, CompactionType, ContainerReference, - ContainerSkill, ContextManagementResponse, McpServer, McpServerType, McpToolConfiguration, - MessageRole, MessageType, MessagesCompaction, MessagesContainer, MessagesDiagnostics, - MessagesDiagnosticsParam, MessagesMetadata, OutputFormat, OutputFormatType, Safeguard, - SkillType, StopDetails, StopDetailsType, StopReason, -}; -pub use request::{ - AdaptiveThinking, CacheControl, ContentBlock, ContentBlockType, ContextEdit, ContextManagement, - ContextTrigger, DisabledThinking, EffortLevel, EnabledThinking, Message, MessageContent, - MessagesOptionalParams, MessagesRequest, MessagesTool, OutputConfig, Speed, SystemPrompt, - ThinkingConfig, ThinkingDisplay, -}; -pub use response::MessagesResponse; -pub use tools::{ - AdvisorTool, AdvisorToolName, AllowedCaller, BashToolName, BrowserToolsetConfigs, - BuiltinMessagesTool, ClientTool, CodeExecutionToolName, ComputerTool, ComputerTool20251124, - ComputerToolName, ComputerToolsetConfigs, CustomTool, CustomToolType, McpToolset, - MemoryToolName, MessagesToolParam, ResponseInclusion, ServerTool, StrReplaceBasedEditToolName, - StrReplaceEditorName, TextEditorTool20250728, ToolChoice, ToolChoiceType, ToolResultUrlSource, - ToolSearchBm25ToolName, ToolSearchRegexToolName, Toolset, ToolsetToolConfig, - UrlSourceToolReference, UserInputUrlSource, UserLocationType, WebFetchTool, - WebFetchTool20260309, WebFetchTool20260318, WebFetchToolName, WebFetchUrlSources, - WebSearchTool, WebSearchTool20260318, WebSearchToolName, WebSearchUserLocation, -}; -pub use usage::{ - CacheCreationUsage, MessagesOutputTokensDetails, MessagesUsage, ServerToolUsage, - UsageIteration, UsageIterationType, -}; +pub use content::*; +pub use metadata::*; +pub use request::*; +pub use response::*; +pub use tools::*; +pub use usage::*; diff --git a/litellm-rust/crates/llms-types/src/formats/messages/request.rs b/litellm-rust/crates/llms-types/src/formats/messages/request.rs index d065f45d41c..9dbfa068025 100644 --- a/litellm-rust/crates/llms-types/src/formats/messages/request.rs +++ b/litellm-rust/crates/llms-types/src/formats/messages/request.rs @@ -387,10 +387,17 @@ impl Message { #[cfg(test)] mod tests { + use super::*; use rstest::rstest; use serde_json::json; - use super::*; + use crate::formats::messages::ContextTrigger; + + use crate::formats::messages::{ContentBlock, ContentBlockType}; + + use crate::recognized::Recognized; + + use serde_json::{Map, Value}; fn round_trip(value: &Value) -> Value { let parsed: T = serde_json::from_value(value.clone()).unwrap(); @@ -718,4 +725,126 @@ mod tests { json!(<&'static str>::from(level)) ); } + + #[rstest] + #[case::null(json!(null))] + #[case::number(json!(1))] + #[case::boolean(json!(true))] + #[case::array(json!(["tool_use"]))] + #[case::object(json!({"type": "tool_use"}))] + fn content_block_type_rejects_non_string_json(#[case] value: Value) { + assert!(serde_json::from_value::(value).is_err()); + } + + #[cfg(feature = "schema")] + #[rstest] + fn content_block_type_schema_remains_a_string() { + let schema = schemars::schema_for!(ContentBlockType).to_value(); + assert_eq!(schema.get("type"), Some(&json!("string"))); + assert_eq!( + schema.get("title"), + Some(&json!(stringify!(ContentBlockType))) + ); + } + + #[rstest] + #[case::text("text", ContentBlockType::Text)] + #[case::thinking("thinking", ContentBlockType::Thinking)] + #[case::redacted_thinking("redacted_thinking", ContentBlockType::RedactedThinking)] + #[case::tool_use("tool_use", ContentBlockType::ToolUse)] + #[case::server_tool_use("server_tool_use", ContentBlockType::ServerToolUse)] + #[case::tool_result("tool_result", ContentBlockType::ToolResult)] + #[case::compaction("compaction", ContentBlockType::Compaction)] + #[case::advisor_result("advisor_tool_result", ContentBlockType::AdvisorToolResult)] + #[case::web_search_result("web_search_tool_result", ContentBlockType::WebSearchToolResult)] + #[case::image("image", ContentBlockType::Other("image".into()))] + #[case::document("document", ContentBlockType::Other("document".into()))] + #[case::tool_addition("tool_addition", ContentBlockType::Other("tool_addition".into()))] + #[case::tool_removal("tool_removal", ContentBlockType::Other("tool_removal".into()))] + #[case::advisor("advisor_result", ContentBlockType::Other("advisor_result".into()))] + #[case::web_search_error( + "web_search_tool_result_error", + ContentBlockType::Other("web_search_tool_result_error".into()) + )] + #[case::future_block("future_block", ContentBlockType::Other("future_block".into()))] + #[case::case_sensitive("Tool_Use", ContentBlockType::Other("Tool_Use".into()))] + #[case::empty("", ContentBlockType::Other(String::new()))] + fn block_type_is_typed_and_round_trips_with_extra_fields( + #[case] wire: &str, + #[case] expected: ContentBlockType, + ) { + let input = json!({"type": wire, "future_field": {"nested": [1, null]}}); + let block: ContentBlock = serde_json::from_value(input.clone()).unwrap(); + assert_eq!(block.block_type.as_ref(), Some(&expected)); + assert_eq!(serde_json::to_value(block).unwrap(), input); + } + + #[rstest] + #[case::number(json!(1))] + #[case::boolean(json!(true))] + #[case::array(json!(["text"]))] + #[case::enum_object(json!({"text": null}))] + fn block_type_rejects_non_string_values(#[case] value: Value) { + assert!(serde_json::from_value::(json!({"type": value})).is_err()); + } + + #[rstest] + #[case::same_type(json!({"type": "tool_use"}), ContentBlockType::ToolUse, true)] + #[case::other_type(json!({"type": "tool_result"}), ContentBlockType::ToolUse, false)] + #[case::unknown_type(json!({"type": "future_tool"}), ContentBlockType::ToolUse, false)] + #[case::no_type(json!({"text": "x"}), ContentBlockType::Text, false)] + fn is_type_matches_the_exact_block_type( + #[case] block: Value, + #[case] block_type: ContentBlockType, + #[case] expected: bool, + ) { + let block: ContentBlock = serde_json::from_value(block).unwrap(); + assert_eq!(block.is_type(block_type), expected); + } + + #[rstest] + #[case::input_tokens("input_tokens", false)] + #[case::tool_uses("tool_uses", true)] + fn context_trigger_exposes_typed_threshold(#[case] tag: &str, #[case] tool_uses: bool) { + let wire = json!({"type":tag,"value":1024,"extension":true}); + let trigger: ContextTrigger = serde_json::from_value(wire.clone()).unwrap(); + match &trigger { + ContextTrigger::InputTokens { value, extra } => { + assert!(!tool_uses); + assert_eq!(*value, 1024); + assert_eq!(extra.get("extension"), Some(&json!(true))); + } + ContextTrigger::ToolUses { value, extra } => { + assert!(tool_uses); + assert_eq!(*value, 1024); + assert_eq!(extra.get("extension"), Some(&json!(true))); + } + } + assert_eq!(serde_json::to_value(trigger).unwrap(), wire); + } + + #[rstest] + #[case::negative(json!({"type":"input_tokens","value":-1}))] + #[case::wrong_shape(json!({"type":"input_tokens","value":"1024"}))] + #[case::missing_value(json!({"type":"input_tokens"}))] + #[case::null_value(json!({"type":"input_tokens","value":null}))] + #[case::tool_uses_negative(json!({"type":"tool_uses","value":-1}))] + #[case::tool_uses_fractional(json!({"type":"tool_uses","value":1.5}))] + #[case::tool_uses_missing_value(json!({"type":"tool_uses"}))] + #[case::missing_discriminator(json!({"value":1}))] + #[case::unknown_discriminator(json!({"type":"other","value":1}))] + fn token_threshold_requires_unsigned_integer(#[case] wire: Value) { + assert!(serde_json::from_value::(wire).is_err()); + } + + #[rstest] + fn existing_content_blocks_preserve_opaque_nested_fields() { + let wire = + json!({"type":"tool_result","content":[{"type":"text","text":7}],"source":{"url":7}}); + let block: crate::formats::messages::ContentBlock = + serde_json::from_value(wire.clone()).unwrap(); + assert_eq!(block.content.as_ref(), wire.get("content")); + assert_eq!(block.extra.get("source"), wire.get("source")); + assert_eq!(serde_json::to_value(block).unwrap(), wire); + } } diff --git a/litellm-rust/crates/llms-types/src/formats/messages/response.rs b/litellm-rust/crates/llms-types/src/formats/messages/response.rs index 84966aa2a0a..9b0c5d31b3a 100644 --- a/litellm-rust/crates/llms-types/src/formats/messages/response.rs +++ b/litellm-rust/crates/llms-types/src/formats/messages/response.rs @@ -20,11 +20,10 @@ pub struct MessagesResponse { #[cfg(test)] mod tests { + use super::*; use rstest::rstest; use serde_json::json; - use super::*; - fn response( stop_reason: Option<&str>, stop_sequence: Option<&str>, diff --git a/litellm-rust/crates/llms-types/src/formats/messages/streaming.rs b/litellm-rust/crates/llms-types/src/formats/messages/streaming.rs index 88f05edd6ed..77c9d8cb742 100644 --- a/litellm-rust/crates/llms-types/src/formats/messages/streaming.rs +++ b/litellm-rust/crates/llms-types/src/formats/messages/streaming.rs @@ -141,3 +141,45 @@ pub enum MessagesStreamEvent { error: MessagesStreamError, }, } + +#[cfg(test)] +mod tests { + use crate::formats::messages::streaming::MessagesStreamEvent; + + use rstest::rstest; + use serde_json::{Value, json}; + + #[rstest] + #[case::start(json!({ + "type": "message_start", + "message": { + "id": "msg", "type": "message", "role": "assistant", "model": "test-model", + "content": [], "stop_reason": null, "stop_sequence": null, + "usage": {"input_tokens": 3, "future_usage": {"count": 9}}, + "future_message": [1, 2] + } + }))] + #[case::block(json!({ + "type": "content_block_start", "index": 0, + "content_block": {"type": "future", "payload": {"keep": true}} + }))] + #[case::delta(json!({ + "type": "message_delta", "delta": {"stop_reason": "end_turn", "future_delta": 42}, + "usage": {"output_tokens": 7, "future_usage": true} + }))] + #[case::error(json!({ + "type": "error", "error": {"type": "future_error", "message": "failed", "future": 42} + }))] + fn stream_events_preserve_extensible_fields(#[case] wire: Value) { + let event: MessagesStreamEvent = serde_json::from_value(wire.clone()).unwrap(); + assert_eq!(serde_json::to_value(event).unwrap(), wire); + } + + #[rstest] + #[case::invalid_index(json!({"type": "content_block_stop", "index": "zero"}))] + #[case::missing_delta(json!({"type": "content_block_delta", "index": 0}))] + #[case::unknown_event(json!({"type": "future_event"}))] + fn malformed_or_unrecognized_events_remain_rejected(#[case] wire: Value) { + assert!(serde_json::from_value::(wire).is_err()); + } +} diff --git a/litellm-rust/crates/llms-types/src/formats/messages/tools.rs b/litellm-rust/crates/llms-types/src/formats/messages/tools.rs index e6c5d6dc345..6f5acf9fb3c 100644 --- a/litellm-rust/crates/llms-types/src/formats/messages/tools.rs +++ b/litellm-rust/crates/llms-types/src/formats/messages/tools.rs @@ -562,3 +562,672 @@ pub enum MessagesToolParam { Builtin(Box), Custom(Box), } + +#[cfg(test)] +mod tests { + use crate::formats::messages::{ + AdvisorTool, AdvisorToolName, CacheControl, UrlSourceToolReference, + }; + use serde_json::Map; + + use crate::formats::messages::AllowedCaller; + + use crate::formats::messages::BashToolName; + + use crate::formats::messages::{BrowserToolsetConfigs, BuiltinMessagesTool}; + + use crate::formats::messages::{CitationsConfig, ClientTool}; + + use crate::formats::messages::CodeExecutionToolName; + + use crate::formats::messages::{ + ComputerTool, ComputerTool20251124, ComputerToolName, ComputerToolsetConfigs, + }; + + use crate::formats::messages::{CustomTool, CustomToolType}; + + use crate::formats::messages::McpListedTool; + + use crate::formats::messages::{McpToolset, MemoryToolName}; + + use crate::formats::messages::MessagesToolParam; + + use crate::formats::messages::ResponseInclusion; + + use crate::formats::messages::ServerTool; + + use crate::formats::messages::{StrReplaceBasedEditToolName, StrReplaceEditorName}; + + use crate::formats::messages::TextEditorTool20250728; + + use crate::formats::messages::{ + ToolChoice, ToolChoiceType, ToolResultUrlSource, ToolSearchBm25ToolName, + }; + + use crate::formats::messages::ToolSearchRegexToolName; + + use crate::formats::messages::{Toolset, ToolsetToolConfig}; + + use crate::formats::messages::{UserInputUrlSource, UserLocationType}; + + use crate::formats::messages::{ + WebFetchTool, WebFetchTool20260309, WebFetchTool20260318, WebFetchToolName, + }; + + use crate::formats::messages::{ + WebFetchUrlSources, WebSearchTool, WebSearchTool20260318, WebSearchToolName, + }; + + use crate::{ + formats::messages::WebSearchUserLocation, + json_schema::{JsonSchema, JsonSchemaObject, JsonSchemaType}, + }; + + use crate::test_support::*; + use indexmap::IndexMap; + use rstest::rstest; + + use serde_json::{Value, json}; + + #[rstest] + fn tool_choice_round_trips() { + let choice = round_trip::( + json!({"type":"tool","name":"search","disable_parallel_tool_use":true}), + ); + assert_eq!(choice.choice_type, ToolChoiceType::Tool); + assert_eq!(choice.name.as_deref(), Some("search")); + assert_eq!(choice.disable_parallel_tool_use, Some(true)); + assert!(choice.extra.is_empty()); + } + + fn client_tool(name: N) -> ClientTool { + ClientTool { + name, + allowed_callers: None, + cache_control: None, + defer_loading: None, + input_examples: None, + strict: None, + extra: Map::new(), + } + } + + fn server_tool(name: N) -> ServerTool { + ServerTool { + name, + allowed_callers: None, + cache_control: None, + defer_loading: None, + strict: None, + extra: Map::new(), + } + } + + fn computer_tool(display_width_px: u64, display_height_px: u64) -> ComputerTool { + ComputerTool { + name: ComputerToolName::Computer, + display_width_px, + display_height_px, + display_number: None, + allowed_callers: None, + cache_control: None, + defer_loading: None, + input_examples: None, + strict: None, + extra: Map::new(), + } + } + + fn web_search_tool() -> WebSearchTool { + WebSearchTool { + name: WebSearchToolName::WebSearch, + allowed_callers: None, + allowed_domains: None, + blocked_domains: None, + cache_control: None, + defer_loading: None, + max_uses: None, + strict: None, + user_location: None, + extra: Map::new(), + } + } + + fn web_fetch_tool() -> WebFetchTool { + WebFetchTool { + name: WebFetchToolName::WebFetch, + allowed_callers: None, + allowed_domains: None, + blocked_domains: None, + cache_control: None, + citations: None, + defer_loading: None, + max_content_tokens: None, + max_uses: None, + strict: None, + url_sources: None, + extra: Map::new(), + } + } + + fn web_fetch_url_sources() -> WebFetchUrlSources { + WebFetchUrlSources { + client_tool_results: None, + server_tool_results: None, + user_input: None, + extra: Map::new(), + } + } + + fn toolset_config(enabled: Option, defer_loading: Option) -> ToolsetToolConfig { + ToolsetToolConfig { + defer_loading, + enabled, + extra: Map::new(), + } + } + + fn browser_toolset_configs() -> BrowserToolsetConfigs { + BrowserToolsetConfigs { + type_text: None, + close_tab: None, + double_click: None, + file_upload: None, + find: None, + form_input: None, + get_page_text: None, + hold_key: None, + hover: None, + javascript_exec: None, + key: None, + left_click: None, + left_click_drag: None, + left_mouse_down: None, + left_mouse_up: None, + list_tabs: None, + middle_click: None, + mouse_move: None, + navigate: None, + new_tab: None, + read_console: None, + read_network: None, + read_page: None, + right_click: None, + screenshot: None, + scroll: None, + scroll_to: None, + switch_tab: None, + triple_click: None, + wait: None, + zoom: None, + extra: Map::new(), + } + } + + fn computer_toolset_configs() -> ComputerToolsetConfigs { + ComputerToolsetConfigs { + type_text: None, + cursor_position: None, + double_click: None, + hold_key: None, + key: None, + left_click: None, + left_click_drag: None, + left_mouse_down: None, + left_mouse_up: None, + middle_click: None, + mouse_move: None, + right_click: None, + screenshot: None, + scroll: None, + triple_click: None, + wait: None, + zoom: None, + extra: Map::new(), + } + } + + #[rstest] + #[case::bash_20241022( + json!({"type":"bash_20241022","name":"bash","input_examples":[{"command":"ls"}]}), + BuiltinMessagesTool::Bash20241022(ClientTool { + input_examples: Some(vec![Map::from_iter([(String::from("command"), json!("ls"))])]), + ..client_tool(BashToolName::Bash) + }) + )] + #[case::bash_20250124( + json!({"type":"bash_20250124","name":"bash","allowed_callers":["direct","code_execution_20260521"]}), + BuiltinMessagesTool::Bash20250124(ClientTool { + allowed_callers: Some(vec![AllowedCaller::Direct, AllowedCaller::CodeExecution20260521]), + ..client_tool(BashToolName::Bash) + }) + )] + #[case::text_editor_20241022( + json!({"type":"text_editor_20241022","name":"str_replace_editor","defer_loading":true}), + BuiltinMessagesTool::TextEditor20241022(ClientTool { + defer_loading: Some(true), + ..client_tool(StrReplaceEditorName::StrReplaceEditor) + }) + )] + #[case::text_editor_20250124( + json!({"type":"text_editor_20250124","name":"str_replace_editor","strict":true}), + BuiltinMessagesTool::TextEditor20250124(ClientTool { + strict: Some(true), + ..client_tool(StrReplaceEditorName::StrReplaceEditor) + }) + )] + #[case::text_editor_20250429( + json!({"type":"text_editor_20250429","name":"str_replace_based_edit_tool","cache_control":{"type":"ephemeral","ttl":"1h"}}), + BuiltinMessagesTool::TextEditor20250429(ClientTool { + cache_control: Some(cache_control("ephemeral", "1h")), + ..client_tool(StrReplaceBasedEditToolName::StrReplaceBasedEditTool) + }) + )] + #[case::text_editor_20250728( + json!({"type":"text_editor_20250728","name":"str_replace_based_edit_tool","max_characters":10000}), + BuiltinMessagesTool::TextEditor20250728(TextEditorTool20250728 { + name: StrReplaceBasedEditToolName::StrReplaceBasedEditTool, + allowed_callers: None, + cache_control: None, + defer_loading: None, + input_examples: None, + max_characters: Some(10000), + strict: None, + extra: Map::new(), + }) + )] + #[case::memory_20250818( + json!({"type":"memory_20250818","name":"memory","allowed_callers":["code_execution_20250825"]}), + BuiltinMessagesTool::Memory20250818(ClientTool { + allowed_callers: Some(vec![AllowedCaller::CodeExecution20250825]), + ..client_tool(MemoryToolName::Memory) + }) + )] + #[case::computer_20241022( + json!({"type":"computer_20241022","name":"computer","display_width_px":1024,"display_height_px":768,"display_number":1}), + BuiltinMessagesTool::Computer20241022(ComputerTool { + display_number: Some(1), + ..computer_tool(1024, 768) + }) + )] + #[case::computer_20250124( + json!({"type":"computer_20250124","name":"computer","display_width_px":1280,"display_height_px":800}), + BuiltinMessagesTool::Computer20250124(computer_tool(1280, 800)) + )] + #[case::computer_20251124( + json!({"type":"computer_20251124","name":"computer","display_width_px":1920,"display_height_px":1080,"enable_zoom":true}), + BuiltinMessagesTool::Computer20251124(ComputerTool20251124 { + name: ComputerToolName::Computer, + display_width_px: 1920, + display_height_px: 1080, + display_number: None, + enable_zoom: Some(true), + allowed_callers: None, + cache_control: None, + defer_loading: None, + input_examples: None, + strict: None, + extra: Map::new(), + }) + )] + #[case::code_execution_20250522( + json!({"type":"code_execution_20250522","name":"code_execution","strict":false}), + BuiltinMessagesTool::CodeExecution20250522(ServerTool { + strict: Some(false), + ..server_tool(CodeExecutionToolName::CodeExecution) + }) + )] + #[case::code_execution_20250825( + json!({"type":"code_execution_20250825","name":"code_execution","defer_loading":false}), + BuiltinMessagesTool::CodeExecution20250825(ServerTool { + defer_loading: Some(false), + ..server_tool(CodeExecutionToolName::CodeExecution) + }) + )] + #[case::code_execution_20260120( + json!({"type":"code_execution_20260120","name":"code_execution","allowed_callers":["direct"]}), + BuiltinMessagesTool::CodeExecution20260120(ServerTool { + allowed_callers: Some(vec![AllowedCaller::Direct]), + ..server_tool(CodeExecutionToolName::CodeExecution) + }) + )] + #[case::code_execution_20260521( + json!({"type":"code_execution_20260521","name":"code_execution","allowed_callers":["code_execution_20260120"]}), + BuiltinMessagesTool::CodeExecution20260521(ServerTool { + allowed_callers: Some(vec![AllowedCaller::CodeExecution20260120]), + ..server_tool(CodeExecutionToolName::CodeExecution) + }) + )] + #[case::tool_search_regex_20251119( + json!({"type":"tool_search_tool_regex_20251119","name":"tool_search_tool_regex","defer_loading":true}), + BuiltinMessagesTool::ToolSearchRegex20251119(ServerTool { + defer_loading: Some(true), + ..server_tool(ToolSearchRegexToolName::ToolSearchToolRegex) + }) + )] + #[case::tool_search_regex( + json!({"type":"tool_search_tool_regex","name":"tool_search_tool_regex","strict":true}), + BuiltinMessagesTool::ToolSearchRegex(ServerTool { + strict: Some(true), + ..server_tool(ToolSearchRegexToolName::ToolSearchToolRegex) + }) + )] + #[case::tool_search_bm25_20251119( + json!({"type":"tool_search_tool_bm25_20251119","name":"tool_search_tool_bm25","defer_loading":false}), + BuiltinMessagesTool::ToolSearchBm2520251119(ServerTool { + defer_loading: Some(false), + ..server_tool(ToolSearchBm25ToolName::ToolSearchToolBm25) + }) + )] + #[case::tool_search_bm25( + json!({"type":"tool_search_tool_bm25","name":"tool_search_tool_bm25","strict":false}), + BuiltinMessagesTool::ToolSearchBm25(ServerTool { + strict: Some(false), + ..server_tool(ToolSearchBm25ToolName::ToolSearchToolBm25) + }) + )] + #[case::web_search_20250305( + json!({"type":"web_search_20250305","name":"web_search","max_uses":3,"allowed_domains":["example.test"], + "user_location":{"type":"approximate","city":"San Francisco","country":"US","timezone":"America/Los_Angeles"}}), + BuiltinMessagesTool::WebSearch20250305(WebSearchTool { + max_uses: Some(3), + allowed_domains: Some(vec![String::from("example.test")]), + user_location: Some(WebSearchUserLocation { + location_type: UserLocationType::Approximate, + city: Some(String::from("San Francisco")), + country: Some(String::from("US")), + region: None, + timezone: Some(String::from("America/Los_Angeles")), + extra: Map::new(), + }), + ..web_search_tool() + }) + )] + #[case::web_search_20260209( + json!({"type":"web_search_20260209","name":"web_search","blocked_domains":["blocked.test"]}), + BuiltinMessagesTool::WebSearch20260209(WebSearchTool { + blocked_domains: Some(vec![String::from("blocked.test")]), + ..web_search_tool() + }) + )] + #[case::web_search_20260318( + json!({"type":"web_search_20260318","name":"web_search","response_inclusion":"excluded"}), + BuiltinMessagesTool::WebSearch20260318(WebSearchTool20260318 { + name: WebSearchToolName::WebSearch, + allowed_callers: None, + allowed_domains: None, + blocked_domains: None, + cache_control: None, + defer_loading: None, + max_uses: None, + response_inclusion: Some(ResponseInclusion::Excluded), + strict: None, + user_location: None, + extra: Map::new(), + }) + )] + #[case::web_fetch_20250910( + json!({"type":"web_fetch_20250910","name":"web_fetch","max_content_tokens":5000,"citations":{"enabled":true}, + "url_sources":{"client_tool_results":{"type":"only","tools":[{"type":"tool_reference","name":"lookup"}]}, + "server_tool_results":{"type":"all"},"user_input":{"type":"none"}}}), + BuiltinMessagesTool::WebFetch20250910(WebFetchTool { + max_content_tokens: Some(5000), + citations: Some(CitationsConfig { + enabled: Some(true), + extra: Map::new(), + }), + url_sources: Some(WebFetchUrlSources { + client_tool_results: Some(ToolResultUrlSource::Only { + tools: vec![tool_reference("lookup")], + extra: Map::new(), + }), + server_tool_results: Some(ToolResultUrlSource::All { extra: Map::new() }), + user_input: Some(UserInputUrlSource::None { extra: Map::new() }), + extra: Map::new(), + }), + ..web_fetch_tool() + }) + )] + #[case::web_fetch_20260209( + json!({"type":"web_fetch_20260209","name":"web_fetch","url_sources":{"server_tool_results":{"type":"except","tools":[{"type":"tool_reference","name":"web_search"}]}}}), + BuiltinMessagesTool::WebFetch20260209(WebFetchTool { + url_sources: Some(WebFetchUrlSources { + server_tool_results: Some(ToolResultUrlSource::Except { + tools: vec![tool_reference("web_search")], + extra: Map::new(), + }), + ..web_fetch_url_sources() + }), + ..web_fetch_tool() + }) + )] + #[case::web_fetch_20260309( + json!({"type":"web_fetch_20260309","name":"web_fetch","use_cache":false}), + BuiltinMessagesTool::WebFetch20260309(WebFetchTool20260309 { + name: WebFetchToolName::WebFetch, + allowed_callers: None, + allowed_domains: None, + blocked_domains: None, + cache_control: None, + citations: None, + defer_loading: None, + max_content_tokens: None, + max_uses: None, + strict: None, + url_sources: None, + use_cache: Some(false), + extra: Map::new(), + }) + )] + #[case::web_fetch_20260318( + json!({"type":"web_fetch_20260318","name":"web_fetch","use_cache":true,"response_inclusion":"full"}), + BuiltinMessagesTool::WebFetch20260318(WebFetchTool20260318 { + name: WebFetchToolName::WebFetch, + allowed_callers: None, + allowed_domains: None, + blocked_domains: None, + cache_control: None, + citations: None, + defer_loading: None, + max_content_tokens: None, + max_uses: None, + response_inclusion: Some(ResponseInclusion::Full), + strict: None, + url_sources: None, + use_cache: Some(true), + extra: Map::new(), + }) + )] + #[case::advisor_20260301( + json!({"type":"advisor_20260301","name":"advisor","model":"claude-opus-5-5","max_tokens":2048,"caching":{"type":"ephemeral","ttl":"5m"}}), + BuiltinMessagesTool::Advisor20260301(AdvisorTool { + name: AdvisorToolName::Advisor, + model: String::from("claude-opus-5-5"), + allowed_callers: None, + cache_control: None, + caching: Some(cache_control("ephemeral", "5m")), + defer_loading: None, + max_tokens: Some(2048), + max_uses: None, + strict: None, + extra: Map::new(), + }) + )] + #[case::browser_toolset_20260801( + json!({"type":"browser_toolset_20260801","configs":{"type":{"enabled":false},"javascript_exec":{"defer_loading":true}}}), + BuiltinMessagesTool::BrowserToolset20260801(Toolset { + cache_control: None, + configs: Some(Box::new(BrowserToolsetConfigs { + type_text: Some(toolset_config(Some(false), None)), + javascript_exec: Some(toolset_config(None, Some(true))), + ..browser_toolset_configs() + })), + extra: Map::new(), + }) + )] + #[case::computer_toolset_20260801( + json!({"type":"computer_toolset_20260801","configs":{"zoom":{"enabled":false},"cursor_position":{"enabled":true}}}), + BuiltinMessagesTool::ComputerToolset20260801(Toolset { + cache_control: None, + configs: Some(Box::new(ComputerToolsetConfigs { + zoom: Some(toolset_config(Some(false), None)), + cursor_position: Some(toolset_config(Some(true), None)), + ..computer_toolset_configs() + })), + extra: Map::new(), + }) + )] + #[case::mcp_toolset( + json!({"type":"mcp_toolset","mcp_server_name":"kb","default_config":{"enabled":false}, + "configs":{"search":{"enabled":true,"defer_loading":true}}, + "tools":[{"name":"search","input_schema":{"type":"object"},"description":"Search"}]}), + BuiltinMessagesTool::McpToolset(McpToolset { + mcp_server_name: String::from("kb"), + cache_control: None, + configs: Some(IndexMap::from([(String::from("search"), toolset_config(Some(true), Some(true)))])), + default_config: Some(toolset_config(Some(false), None)), + tools: Some(vec![McpListedTool { + name: String::from("search"), + description: Some(String::from("Search")), + input_schema: JsonSchema::Object(Box::new(JsonSchemaObject { + schema_type: Some(JsonSchemaType::Name(String::from("object"))), + ..JsonSchemaObject::default() + })), + extra: Map::new(), + }]), + extra: Map::new(), + }) + )] + fn builtin_tools_decode_typed_definitions( + #[case] wire: Value, + #[case] expected: BuiltinMessagesTool, + ) { + assert_eq!(round_trip::(wire), expected); + } + + #[rstest] + fn builtin_tools_preserve_unknown_fields() { + let tool = round_trip::( + json!({"type":"bash_20250124","name":"bash","extension":[1,null]}), + ); + let BuiltinMessagesTool::Bash20250124(bash) = tool else { + panic!("expected bash tool"); + }; + assert_eq!(bash.extra.get("extension"), Some(&json!([1, null]))); + } + + #[rstest] + #[case::missing_discriminator(json!({"name":"bash"}))] + #[case::unknown_discriminator(json!({"type":"future_tool","name":"bash"}))] + #[case::null_discriminator(json!({"type":null,"name":"bash"}))] + #[case::bash_without_name(json!({"type":"bash_20250124"}))] + #[case::bash_wrong_name(json!({"type":"bash_20241022","name":"shell"}))] + #[case::legacy_editor_with_new_name(json!({"type":"text_editor_20250124","name":"str_replace_based_edit_tool"}))] + #[case::new_editor_with_legacy_name(json!({"type":"text_editor_20250728","name":"str_replace_editor"}))] + #[case::editor_20250429_with_legacy_name(json!({"type":"text_editor_20250429","name":"str_replace_editor"}))] + #[case::memory_without_name(json!({"type":"memory_20250818"}))] + #[case::computer_without_width(json!({"type":"computer_20250124","name":"computer","display_height_px":768}))] + #[case::computer_without_height(json!({"type":"computer_20241022","name":"computer","display_width_px":1024}))] + #[case::zoom_computer_without_display(json!({"type":"computer_20251124","name":"computer"}))] + #[case::code_execution_without_name(json!({"type":"code_execution_20250825"}))] + #[case::regex_search_with_bm25_name(json!({"type":"tool_search_tool_regex","name":"tool_search_tool_bm25"}))] + #[case::bm25_search_with_regex_name(json!({"type":"tool_search_tool_bm25_20251119","name":"tool_search_tool_regex"}))] + #[case::web_search_with_fetch_name(json!({"type":"web_search_20250305","name":"web_fetch"}))] + #[case::web_fetch_without_name(json!({"type":"web_fetch_20260318"}))] + #[case::advisor_without_model(json!({"type":"advisor_20260301","name":"advisor"}))] + #[case::advisor_without_name(json!({"type":"advisor_20260301","model":"claude-opus-5-5"}))] + #[case::mcp_toolset_without_server(json!({"type":"mcp_toolset"}))] + #[case::unknown_allowed_caller(json!({"type":"code_execution_20250825","name":"code_execution","allowed_callers":["code_execution_20250522"]}))] + #[case::unknown_response_inclusion(json!({"type":"web_search_20260318","name":"web_search","response_inclusion":"partial"}))] + #[case::user_input_only_filter(json!({"type":"web_fetch_20250910","name":"web_fetch","url_sources":{"user_input":{"type":"only","tools":[]}}}))] + #[case::negative_limit(json!({"type":"web_search_20250305","name":"web_search","max_uses":-1}))] + #[case::wrong_mcp_config(json!({"type":"mcp_toolset","mcp_server_name":"kb","configs":{"search":{"enabled":"yes"}}}))] + #[case::wrong_browser_config(json!({"type":"browser_toolset_20260801","configs":{"navigate":{"enabled":1}}}))] + fn builtin_tools_reject_malformed_fields(#[case] wire: Value) { + assert!(serde_json::from_value::(wire.clone()).is_err()); + assert!(serde_json::from_value::(wire).is_err()); + } + + #[rstest] + fn custom_tool_exposes_schema_and_preserves_extensions() { + let tool = round_trip::(json!({ + "type":"custom", + "name":"lookup", + "input_schema":{"type":"object","properties":{"query":{"type":"string"}}}, + "strict":false, + "defer_loading":true, + "allowed_callers":["direct","code_execution_20260120"], + "extension":{"nested":[1,null]} + })); + assert_eq!( + tool.allowed_callers.as_deref(), + Some([AllowedCaller::Direct, AllowedCaller::CodeExecution20260120].as_slice()) + ); + assert_eq!(tool.tool_type, Some(CustomToolType::Custom)); + assert_eq!(tool.name, "lookup"); + assert_eq!((tool.strict, tool.defer_loading), (Some(false), Some(true))); + let JsonSchema::Object(schema) = &tool.input_schema else { + panic!("expected an object schema"); + }; + assert!(schema.properties.as_ref().unwrap().contains_key("query")); + assert_eq!(tool.extra["extension"], json!({"nested":[1,null]})); + } + + #[rstest] + fn custom_tool_omits_null_discriminator() { + let tool: CustomTool = + serde_json::from_value(json!({"type":null,"name":"lookup","input_schema":true})) + .unwrap(); + assert!(tool.tool_type.is_none()); + assert!(tool.description.is_none()); + assert_eq!( + serde_json::to_value(tool).unwrap(), + json!({"name":"lookup","input_schema":true}) + ); + } + + #[rstest] + #[case::missing_name(json!({"input_schema":{}}))] + #[case::missing_schema(json!({"name":"lookup"}))] + #[case::wrong_name_shape(json!({"name":7,"input_schema":{}}))] + #[case::builtin_tool(json!({"name":"lookup","input_schema":{},"type":"bash_20250124"}))] + #[case::unknown_discriminator(json!({"name":"lookup","input_schema":{},"type":"future_tool"}))] + #[case::unknown_allowed_caller(json!({"name":"lookup","input_schema":{},"allowed_callers":["anyone"]}))] + fn custom_tool_rejects_invalid_shapes(#[case] wire: Value) { + assert!(serde_json::from_value::(wire).is_err()); + } + + #[rstest] + #[case::builtin(json!({"type":"web_search_20250305","name":"web_search","max_uses":2}), true)] + #[case::builtin_toolset(json!({"type":"mcp_toolset","mcp_server_name":"kb"}), true)] + #[case::custom(json!({"name":"lookup","input_schema":{"type":"object"}}), false)] + fn tool_params_dispatch_on_discriminator(#[case] wire: Value, #[case] builtin: bool) { + let tool = round_trip::(wire); + assert_eq!(matches!(tool, MessagesToolParam::Builtin(_)), builtin); + } + + #[rstest] + fn tool_params_reject_unknown_tool_types() { + assert!( + serde_json::from_value::( + json!({"type":"future_tool","name":"lookup","input_schema":{}}) + ) + .is_err() + ); + } + + fn tool_reference(name: &str) -> UrlSourceToolReference { + UrlSourceToolReference::ToolReference { + name: String::from(name), + extra: Map::new(), + } + } + + fn cache_control(cache_type: &str, ttl: &str) -> CacheControl { + CacheControl { + cache_type: Some(String::from(cache_type)), + ttl: Some(String::from(ttl)), + ..CacheControl::default() + } + } +} diff --git a/litellm-rust/crates/llms-types/src/formats/messages/usage.rs b/litellm-rust/crates/llms-types/src/formats/messages/usage.rs index 4ce0ef1d5c4..6219dbf85bc 100644 --- a/litellm-rust/crates/llms-types/src/formats/messages/usage.rs +++ b/litellm-rust/crates/llms-types/src/formats/messages/usage.rs @@ -73,3 +73,122 @@ pub struct MessagesOutputTokensDetails { #[serde(flatten)] pub extra: Map, } + +#[cfg(test)] +mod tests { + + use crate::formats::messages::MessagesUsage; + + use crate::formats::messages::UsageIterationType; + + use crate::{recognized::Recognized, test_support::*}; + + use rstest::rstest; + + use serde_json::json; + + #[rstest] + fn usage_contracts_round_trip() { + let usage = round_trip::(json!({ + "input_tokens":10, + "output_tokens":4, + "server_tool_use":{"web_search_requests":2,"web_fetch_requests":1}, + "cache_creation":{"ephemeral_1h_input_tokens":3,"ephemeral_5m_input_tokens":1}, + "output_tokens_details":{"thinking_tokens":2}, + "iterations":[ + {"type":"compaction","input_tokens":7,"output_tokens":1}, + {"type":"message","input_tokens":3,"output_tokens":3} + ], + "service_tier":"priority", + "inference_geo":"global", + "speed":"fast" + })); + assert_eq!(usage.inference_geo.as_deref(), Some("global")); + assert_eq!(usage.input_tokens, Some(10)); + assert_eq!(usage.output_tokens, Some(4)); + assert!(usage.extra.is_empty()); + let server = usage.server_tool_use.as_ref().unwrap(); + assert_eq!(server.web_search_requests, Some(2)); + assert_eq!(server.web_fetch_requests, Some(1)); + let cache = usage.cache_creation.as_ref().unwrap(); + assert_eq!(cache.ephemeral_1h_input_tokens, Some(3)); + assert_eq!(cache.ephemeral_5m_input_tokens, Some(1)); + assert_eq!( + usage + .output_tokens_details + .as_ref() + .unwrap() + .thinking_tokens, + Some(2) + ); + let [compaction, message] = usage.iterations.as_ref().unwrap().as_slice() else { + panic!("expected usage iterations"); + }; + assert_eq!( + compaction.iteration_type, + Recognized::Known(UsageIterationType::Compaction) + ); + assert_eq!( + message.iteration_type, + Recognized::Known(UsageIterationType::Message) + ); + assert_eq!( + compaction.input_tokens.unwrap() + message.input_tokens.unwrap(), + usage.input_tokens.unwrap() + ); + assert_eq!( + compaction.output_tokens.unwrap() + message.output_tokens.unwrap(), + usage.output_tokens.unwrap() + ); + } + + #[rstest] + fn usage_iterations_expose_advisor_and_fallback_models() { + let usage = round_trip::(json!({ + "iterations":[ + {"type":"advisor_message","model":"claude-opus-5-5","input_tokens":5,"output_tokens":2, + "cache_creation_input_tokens":4,"cache_read_input_tokens":1, + "cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":4}}, + {"type":"fallback_message","model":"claude-sonnet-5-5","input_tokens":6,"output_tokens":3, + "cache_creation_input_tokens":0,"cache_read_input_tokens":2}, + {"type":"future_iteration","input_tokens":1} + ] + })); + let [advisor, fallback, future] = usage.iterations.as_deref().unwrap() else { + panic!("expected three iterations"); + }; + assert_eq!( + advisor.iteration_type, + Recognized::Known(UsageIterationType::AdvisorMessage) + ); + assert_eq!(advisor.model.as_deref(), Some("claude-opus-5-5")); + assert_eq!( + ( + advisor.cache_creation_input_tokens, + advisor.cache_read_input_tokens + ), + (Some(4), Some(1)) + ); + assert_eq!( + advisor + .cache_creation + .as_ref() + .unwrap() + .ephemeral_5m_input_tokens, + advisor.cache_creation_input_tokens + ); + assert!(advisor.extra.is_empty()); + assert_eq!( + fallback.iteration_type, + Recognized::Known(UsageIterationType::FallbackMessage) + ); + assert_eq!(fallback.model.as_deref(), Some("claude-sonnet-5-5")); + assert_eq!(fallback.cache_read_input_tokens, Some(2)); + assert!(fallback.extra.is_empty()); + assert_eq!( + future.iteration_type, + Recognized::Unrecognized(json!("future_iteration")) + ); + assert_eq!(future.input_tokens, Some(1)); + } +} diff --git a/litellm-rust/crates/llms-types/src/formats/mod.rs b/litellm-rust/crates/llms-types/src/formats/mod.rs index 0353077c3a9..53f2577090b 100644 --- a/litellm-rust/crates/llms-types/src/formats/mod.rs +++ b/litellm-rust/crates/llms-types/src/formats/mod.rs @@ -1,6 +1,3 @@ -//! Wire types for each provider API format (OpenAI Chat Completions, -//! Anthropic Messages, Responses, ...), one submodule per format. - pub mod audio_transcription; pub mod batches; pub mod chat_completions; diff --git a/litellm-rust/crates/llms-types/src/formats/ocr.rs b/litellm-rust/crates/llms-types/src/formats/ocr.rs deleted file mode 100644 index 68c991a6e8b..00000000000 --- a/litellm-rust/crates/llms-types/src/formats/ocr.rs +++ /dev/null @@ -1,164 +0,0 @@ -use std::collections::BTreeMap; - -use serde_json::{Map, Value}; -use serde_with::serde_as; - -use crate::serde_compat::{FiniteF64, LaxI64}; - -#[macro_rules_attribute::apply(crate::wire_type)] -#[serde(tag = "type")] -pub enum OcrDocument { - #[serde(rename = "document_url")] - DocumentUrl { - document_url: String, - #[serde(flatten)] - extra_fields: BTreeMap>, - }, - #[serde(rename = "image_url")] - ImageUrl { - image_url: String, - #[serde(flatten)] - extra_fields: BTreeMap>, - }, -} - -impl OcrDocument { - pub fn source(&self) -> &str { - match self { - Self::DocumentUrl { document_url, .. } => document_url, - Self::ImageUrl { image_url, .. } => image_url, - } - } - - pub fn is_remote(&self) -> bool { - let source = self.source(); - source.starts_with("http://") || source.starts_with("https://") - } - - pub fn with_source(self, source: String) -> Self { - match self { - Self::DocumentUrl { extra_fields, .. } => Self::DocumentUrl { - document_url: source, - extra_fields, - }, - Self::ImageUrl { extra_fields, .. } => Self::ImageUrl { - image_url: source, - extra_fields, - }, - } - } -} - -#[macro_rules_attribute::apply(crate::wire_type)] -#[derive(Copy, Default, Eq)] -#[serde(rename_all = "lowercase")] -pub enum OcrResponseFormat { - #[default] - Litellm, - Native, -} - -#[serde_as] -#[macro_rules_attribute::apply(crate::wire_type)] -#[derive(Default)] -pub struct OcrPageDimensions { - #[serde_as(deserialize_as = "Option")] - pub dpi: Option, - #[serde_as(deserialize_as = "Option")] - pub height: Option, - #[serde_as(deserialize_as = "Option")] - pub width: Option, -} - -#[macro_rules_attribute::apply(crate::wire_type)] -#[derive(Default)] -pub struct OcrPageImage { - pub image_base64: Option, - pub bbox: Option>, - #[serde(flatten)] - pub extra_fields: Map, -} - -#[serde_as] -#[macro_rules_attribute::apply(crate::wire_type)] -#[derive(Default)] -pub struct OcrPage { - #[serde_as(deserialize_as = "LaxI64")] - pub index: i64, - pub markdown: String, - pub images: Option>, - pub dimensions: Option, - #[serde(flatten)] - pub extra_fields: Map, -} - -#[serde_as] -#[macro_rules_attribute::apply(crate::wire_type)] -#[derive(Default)] -pub struct OcrUsageInfo { - #[serde_as(deserialize_as = "Option")] - pub pages_processed: Option, - #[serde_as(deserialize_as = "Option")] - pub pages_processed_annotation: Option, - #[serde_as(deserialize_as = "Option")] - pub credits: Option, - #[serde_as(deserialize_as = "Option")] - pub doc_size_bytes: Option, - #[serde(flatten)] - pub extra_fields: Map, -} - -#[macro_rules_attribute::apply(crate::wire_type)] -pub struct LiteLLMOcrResponse { - pub pages: Vec, - pub model: String, - pub document_annotation: Option, - pub usage_info: Option, - pub content: Option, - pub tables: Option>>, - #[serde(rename = "keyValuePairs")] - pub key_value_pairs: Option>>, - #[serde(default = "ocr_object")] - pub object: String, - #[serde(flatten)] - pub extra_fields: Map, - #[serde(skip_serializing_if = "Option::is_none")] - pub provider_native_response: Option>, -} - -impl LiteLLMOcrResponse { - pub fn new(model: impl Into, pages: Vec) -> Self { - Self { - pages, - model: model.into(), - document_annotation: None, - usage_info: None, - content: None, - tables: None, - key_value_pairs: None, - object: ocr_object(), - extra_fields: Map::new(), - provider_native_response: None, - } - } - - pub fn into_json(self) -> Value { - serde_json::to_value(self).expect("OCR response fields are JSON-compatible") - } -} - -fn ocr_object() -> String { - "ocr".into() -} - -#[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(crate::wire_type)] -#[derive(Default)] -pub struct OcrBoundingBox { - pub top_left_x: Option, - pub top_left_y: Option, - pub bottom_right_x: Option, - pub bottom_right_y: Option, - #[serde(flatten)] - pub extra: Map, -} diff --git a/litellm-rust/crates/llms-types/src/formats/ocr/AGENTS.md b/litellm-rust/crates/llms-types/src/formats/ocr/AGENTS.md new file mode 100644 index 00000000000..a59ec9e4dcd --- /dev/null +++ b/litellm-rust/crates/llms-types/src/formats/ocr/AGENTS.md @@ -0,0 +1,12 @@ +# rules + +- LiteLLM's OCR contract follows the Mistral OCR response shape. It is the normalized output every OCR provider returns, not a Mistral type +- Mistral, and Mistral models hosted on Azure AI and Vertex AI, speak it natively +- Cohere, Reducto, AWS Textract, Azure Document Intelligence and DeepSeek OCR on Vertex translate their native output into it in `llms/src//ocr/` +- `OcrBoundingBox` is the corner coordinates providers copy into the normalized `bbox` +- Normalized `tables` and `keyValuePairs` stay open because providers pass through different native shapes +- `BaseOcrConfig`, OCR errors and inline-document helpers live in `llms/src/base_llm/ocr/` + +# references + +- https://docs.mistral.ai/api/endpoint/ocr diff --git a/litellm-rust/crates/llms-types/src/formats/ocr/mod.rs b/litellm-rust/crates/llms-types/src/formats/ocr/mod.rs new file mode 100644 index 00000000000..b8be6322b5c --- /dev/null +++ b/litellm-rust/crates/llms-types/src/formats/ocr/mod.rs @@ -0,0 +1,5 @@ +mod request; +mod response; + +pub use request::*; +pub use response::*; diff --git a/litellm-rust/crates/llms-types/src/formats/ocr/request.rs b/litellm-rust/crates/llms-types/src/formats/ocr/request.rs new file mode 100644 index 00000000000..c199d81a9a4 --- /dev/null +++ b/litellm-rust/crates/llms-types/src/formats/ocr/request.rs @@ -0,0 +1,95 @@ +use std::collections::BTreeMap; + +#[macro_rules_attribute::apply(crate::wire_type)] +#[serde(tag = "type")] +pub enum OcrDocument { + #[serde(rename = "document_url")] + DocumentUrl { + document_url: String, + #[serde(flatten)] + extra_fields: BTreeMap>, + }, + #[serde(rename = "image_url")] + ImageUrl { + image_url: String, + #[serde(flatten)] + extra_fields: BTreeMap>, + }, +} + +impl OcrDocument { + pub fn source(&self) -> &str { + match self { + Self::DocumentUrl { document_url, .. } => document_url, + Self::ImageUrl { image_url, .. } => image_url, + } + } + + pub fn is_remote(&self) -> bool { + let source = self.source(); + source.starts_with("http://") || source.starts_with("https://") + } + + pub fn with_source(self, source: String) -> Self { + match self { + Self::DocumentUrl { extra_fields, .. } => Self::DocumentUrl { + document_url: source, + extra_fields, + }, + Self::ImageUrl { extra_fields, .. } => Self::ImageUrl { + image_url: source, + extra_fields, + }, + } + } +} + +#[macro_rules_attribute::apply(crate::wire_type)] +#[derive(Copy, Default, Eq)] +#[serde(rename_all = "lowercase")] +pub enum OcrResponseFormat { + #[default] + Litellm, + Native, +} + +#[cfg(test)] +mod tests { + use crate::formats::ocr::OcrDocument; + + use rstest::rstest; + use serde_json::{Value, json}; + + #[rstest] + fn document_rejects_non_string_provider_fields() { + assert!( + serde_json::from_value::(json!({ + "type": "image_url", "image_url": "https://example.com/image", "detail": 42 + })) + .is_err() + ); + } + + #[rstest] + #[case::document_url("document_url", "document_name", "application/pdf")] + #[case::image_url("image_url", "detail", "image/png")] + fn document_variants_preserve_provider_fields_when_rewriting_sources( + #[case] kind: &str, + #[case] field: &str, + #[case] mime_type: &str, + #[values(json!("kept"), Value::Null)] extra: Value, + ) { + let original = "https://example.com/input"; + let replacement = format!("data:{mime_type};base64,AA=="); + let document: OcrDocument = + serde_json::from_value(json!({"type": kind, kind: original, field: extra})).unwrap(); + assert_eq!(document.source(), original); + assert!(document.is_remote()); + let rewritten = document.with_source(replacement.clone()); + assert!(!rewritten.is_remote()); + assert_eq!( + serde_json::to_value(rewritten).unwrap(), + json!({"type": kind, kind: replacement, field: extra}) + ); + } +} diff --git a/litellm-rust/crates/llms-types/src/formats/ocr/response.rs b/litellm-rust/crates/llms-types/src/formats/ocr/response.rs new file mode 100644 index 00000000000..6c1958127f0 --- /dev/null +++ b/litellm-rust/crates/llms-types/src/formats/ocr/response.rs @@ -0,0 +1,224 @@ +use serde_json::{Map, Value}; +use serde_with::serde_as; + +use crate::serde_compat::{FiniteF64, LaxI64}; + +#[serde_as] +#[macro_rules_attribute::apply(crate::wire_type)] +#[derive(Default)] +pub struct OcrPageDimensions { + #[serde_as(deserialize_as = "Option")] + pub dpi: Option, + #[serde_as(deserialize_as = "Option")] + pub height: Option, + #[serde_as(deserialize_as = "Option")] + pub width: Option, +} + +#[macro_rules_attribute::apply(crate::wire_type)] +#[derive(Default)] +pub struct OcrPageImage { + pub image_base64: Option, + pub bbox: Option>, + #[serde(flatten)] + pub extra_fields: Map, +} + +#[serde_as] +#[macro_rules_attribute::apply(crate::wire_type)] +#[derive(Default)] +pub struct OcrPage { + #[serde_as(deserialize_as = "LaxI64")] + pub index: i64, + pub markdown: String, + pub images: Option>, + pub dimensions: Option, + #[serde(flatten)] + pub extra_fields: Map, +} + +#[serde_as] +#[macro_rules_attribute::apply(crate::wire_type)] +#[derive(Default)] +pub struct OcrUsageInfo { + #[serde_as(deserialize_as = "Option")] + pub pages_processed: Option, + #[serde_as(deserialize_as = "Option")] + pub pages_processed_annotation: Option, + #[serde_as(deserialize_as = "Option")] + pub credits: Option, + #[serde_as(deserialize_as = "Option")] + pub doc_size_bytes: Option, + #[serde(flatten)] + pub extra_fields: Map, +} + +#[macro_rules_attribute::apply(crate::wire_type)] +pub struct LiteLLMOcrResponse { + pub pages: Vec, + pub model: String, + pub document_annotation: Option, + pub usage_info: Option, + pub content: Option, + pub tables: Option>>, + #[serde(rename = "keyValuePairs")] + pub key_value_pairs: Option>>, + #[serde(default = "ocr_object")] + pub object: String, + #[serde(flatten)] + pub extra_fields: Map, + #[serde(skip_serializing_if = "Option::is_none")] + pub provider_native_response: Option>, +} + +impl LiteLLMOcrResponse { + pub fn new(model: impl Into, pages: Vec) -> Self { + Self { + pages, + model: model.into(), + document_annotation: None, + usage_info: None, + content: None, + tables: None, + key_value_pairs: None, + object: ocr_object(), + extra_fields: Map::new(), + provider_native_response: None, + } + } + + pub fn into_json(self) -> Value { + serde_json::to_value(self).expect("OCR response fields are JSON-compatible") + } +} + +fn ocr_object() -> String { + "ocr".into() +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(crate::wire_type)] +#[derive(Default)] +pub struct OcrBoundingBox { + pub top_left_x: Option, + pub top_left_y: Option, + pub bottom_right_x: Option, + pub bottom_right_y: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[cfg(test)] +mod tests { + use crate::formats::ocr::{LiteLLMOcrResponse, OcrBoundingBox, OcrPage}; + + use rstest::rstest; + use serde_json::{Map, Value, json}; + + #[rstest] + #[case::missing_page_fields(json!({"pages": [{}]}))] + #[case::invalid_markdown(json!({"pages": [{"index": 0, "markdown": false}]}))] + #[case::invalid_image_bounds(json!({"pages": [{"index": 0, "markdown": "", "images": [{"bbox": []}]}]}))] + #[case::fractional_page_count(json!({"usage_info": {"pages_processed": 1.5}}))] + #[case::invalid_table(json!({"tables": [false]}))] + #[case::invalid_key_value_pair(json!({"keyValuePairs": [[]]}))] + #[case::invalid_native_response(json!({"provider_native_response": []}))] + fn normalized_response_rejects_invalid_shared_fields(#[case] fields: Value) { + let payload: Map = json!({"model": "model", "pages": []}) + .as_object() + .unwrap() + .iter() + .chain(fields.as_object().unwrap()) + .map(|(key, value)| (key.clone(), value.clone())) + .collect(); + assert!(serde_json::from_value::(Value::Object(payload)).is_err()); + } + + #[rstest] + #[case::large_integer(json!("9007199254740993.0"), 9_007_199_254_740_993)] + #[case::signed_decimal(json!("+2.000"), 2)] + #[case::separator(json!("1_000"), 1000)] + #[case::boolean(json!(true), 1)] + #[case::integral_float(json!(2.0), 2)] + fn numeric_coercion_preserves_integer_precision(#[case] value: Value, #[case] expected: i64) { + let page: OcrPage = + serde_json::from_value(json!({"index": value, "markdown": ""})).unwrap(); + assert_eq!(page.index, expected); + assert_eq!( + serde_json::to_value(page).unwrap()["index"], + json!(expected) + ); + } + + #[rstest] + #[case::exponent(json!("1e2"))] + #[case::missing_integer(json!(".0"))] + #[case::missing_fraction(json!("2."))] + #[case::leading_separator(json!("_2"))] + #[case::repeated_separator(json!("2__0"))] + #[case::fractional_float(json!(2.5))] + #[case::null(json!(null))] + fn page_index_rejects_invalid_integers(#[case] value: Value) { + assert!( + serde_json::from_value::(json!({"index": value, "markdown": ""})).is_err() + ); + } + + #[rstest] + #[case::absent_native(None)] + #[case::present_native(Some(Map::from_iter([("native".into(), json!({"nested": [null, 1]}))])))] + fn response_serialization_preserves_extensions_and_native_presence( + #[case] native: Option>, + ) { + let response = LiteLLMOcrResponse { + extra_fields: Map::from_iter([("provider_field".into(), json!("kept"))]), + provider_native_response: native.clone(), + ..LiteLLMOcrResponse::new("model", vec![]) + }; + let serialized = response.into_json(); + assert_eq!(serialized["provider_field"], "kept"); + assert_eq!( + serialized.get("provider_native_response").cloned(), + native.clone().map(Value::Object) + ); + let decoded: LiteLLMOcrResponse = serde_json::from_value(serialized.clone()).unwrap(); + assert_eq!(decoded.provider_native_response, native); + assert_eq!(decoded.into_json(), serialized); + } + + #[rstest] + fn bounding_box_exposes_corner_coordinates_and_keeps_extensions() { + let wire = json!({"top_left_x":1,"top_left_y":2.5,"bottom_right_x":30,"bottom_right_y":40,"future":true}); + let bounds: OcrBoundingBox = serde_json::from_value(wire.clone()).unwrap(); + assert_eq!(bounds.top_left_x, Some(1.into())); + assert_eq!( + bounds + .top_left_y + .as_ref() + .and_then(serde_json::Number::as_f64), + Some(2.5) + ); + assert_eq!(bounds.bottom_right_x, Some(30.into())); + assert_eq!(bounds.bottom_right_y, Some(40.into())); + assert_eq!(Value::Object(bounds.extra.clone()), json!({"future":true})); + assert_eq!(serde_json::to_value(bounds).unwrap(), wire); + } + + #[rstest] + fn partial_bounding_box_omits_null_corners() { + let bounds: OcrBoundingBox = + serde_json::from_value(json!({"top_left_x":null,"bottom_right_y":4})).unwrap(); + assert!(bounds.top_left_x.is_none()); + assert_eq!( + serde_json::to_value(bounds).unwrap(), + json!({"bottom_right_y":4}) + ); + } + + #[rstest] + #[case::string_corner(json!({"top_left_x":"1"}))] + #[case::array_corner(json!({"bottom_right_y":[4]}))] + fn bounding_box_rejects_non_numeric_corners(#[case] wire: Value) { + assert!(serde_json::from_value::(wire).is_err()); + } +} diff --git a/litellm-rust/crates/llms-types/src/formats/responses/AGENTS.md b/litellm-rust/crates/llms-types/src/formats/responses/AGENTS.md new file mode 100644 index 00000000000..606c1862ad7 --- /dev/null +++ b/litellm-rust/crates/llms-types/src/formats/responses/AGENTS.md @@ -0,0 +1,15 @@ +# rules + +- Responses originated at OpenAI and is now offered by other hosts and gateways. Types here describe the format, not OpenAI's deployment of it +- Served over HTTP and over the WebSocket mode. `ResponsesWsEvent` is a wire event and lives here +- `ResponsesWsTransformResult` wraps a provider transformation's output, so it lives in `llms/src/base_llm/responses/` +- Hosts that accept only part of the format, or add output item types, are handled in `llms/src//responses/`. Keep unknown output items passing through instead of rejecting them +- Responses-to-Chat emulation and model-specific parameter rewriting are transformations, not format rules + +# references + +- https://developers.openai.com/api/reference/resources/responses +- https://developers.openai.com/api/reference/resources/responses/streaming-events +- https://github.com/openai/openai-python/blob/main/src/openai/types/responses/response.py +- https://github.com/openai/openai-python/blob/main/src/openai/types/responses/response_output_item.py +- https://github.com/openai/openai-python/blob/main/src/openai/types/responses/mcp_tool_call_error.py diff --git a/litellm-rust/crates/llms-types/src/formats/responses/mod.rs b/litellm-rust/crates/llms-types/src/formats/responses/mod.rs index a45a971bae6..7fb0324dd43 100644 --- a/litellm-rust/crates/llms-types/src/formats/responses/mod.rs +++ b/litellm-rust/crates/llms-types/src/formats/responses/mod.rs @@ -3,4 +3,4 @@ mod response; pub mod streaming_websocket; pub use output::*; -pub use response::ResponsesApiResponse; +pub use response::*; diff --git a/litellm-rust/crates/llms-types/src/formats/responses/output.rs b/litellm-rust/crates/llms-types/src/formats/responses/output.rs index ba4c011d458..bbb2fb8149a 100644 --- a/litellm-rust/crates/llms-types/src/formats/responses/output.rs +++ b/litellm-rust/crates/llms-types/src/formats/responses/output.rs @@ -829,3 +829,584 @@ pub struct ResponsesMcpApprovalResponse { #[serde(flatten)] pub extra: Map, } + +#[cfg(test)] +mod tests { + use crate::formats::{ + chat_completions::{PromptCacheBreakpoint, PromptCacheMode}, + responses::{ + ResponsesAdditionalTools, ResponsesApplyPatchCall, ResponsesApplyPatchCallOutput, + ResponsesApplyPatchOperation, ResponsesCodeOutput, ResponsesCompaction, + ResponsesComputerAction, ResponsesComputerCall, ResponsesComputerCallOutput, + ResponsesComputerOutput, ResponsesContentPart, ResponsesCoordinate, + ResponsesCustomToolCall, ResponsesCustomToolCallOutput, ResponsesFunctionCall, + ResponsesFunctionCallOutput, ResponsesImageGenerationCall, ResponsesInputContent, + ResponsesLocalShellAction, ResponsesLocalShellCall, ResponsesLocalShellCallOutput, + ResponsesMcpApprovalRequest, ResponsesMcpApprovalResponse, ResponsesMcpError, + ResponsesMcpErrorDetail, ResponsesOutputItem, ResponsesProgram, ResponsesProgramOutput, + ResponsesSafetyCheck, ResponsesShellAction, ResponsesShellCall, + ResponsesShellCallOutput, ResponsesShellEnvironment, ResponsesShellOutcome, + ResponsesShellOutputChunk, ResponsesToolCaller, ResponsesToolOutput, + ResponsesToolSearchCall, ResponsesToolSearchOutput, ResponsesWebSearchAction, + }, + }; + + use rstest::rstest; + + use serde_json::{Value, json}; + + #[rstest] + #[case::message(json!({"type":"message","role":"assistant","content":[{"type":"output_text","text":"answer","annotations":[{"type":"url_citation","url":"https://example.test","start_index":0,"end_index":6}]}]}))] + #[case::function_call(json!({"type":"function_call","call_id":"call_1","name":"lookup","arguments":"{\"query\":\"q\"}"}))] + #[case::custom_tool(json!({"type":"custom_tool_call","name":"lookup","input":"q"}))] + #[case::reasoning(json!({"type":"reasoning","summary":[{"type":"summary_text","text":"summary"}],"encrypted_content":"opaque"}))] + #[case::web_search(json!({"type":"web_search_call","action":{"type":"search","queries":["q"],"sources":[{"type":"url","url":"https://example.test"}]}}))] + #[case::file_search(json!({"type":"file_search_call","queries":["q"],"results":[{"file_id":"file_1","score":1,"attributes":{"custom":[1,null]}}]}))] + #[case::code(json!({"type":"code_interpreter_call","outputs":[{"type":"logs","logs":"done"},{"type":"image","url":"https://example.test"}]}))] + #[case::image(json!({"type":"image_generation_call","result":"generated"}))] + #[case::mcp(json!({"type":"mcp_call","server_label":"server","name":"lookup","arguments":"{}","output":"done"}))] + fn output_items_round_trip(#[case] wire: Value) { + let item: ResponsesOutputItem = serde_json::from_value(wire.clone()).unwrap(); + match &item { + ResponsesOutputItem::Message(message) => { + assert_eq!(message.role.as_deref(), Some("assistant")); + let Some(content) = &message.content else { + panic!("expected content") + }; + let [ + ResponsesContentPart::OutputText { + text, + annotations: Some(annotations), + .. + }, + ] = content.as_slice() + else { + panic!("expected output text and annotations") + }; + assert_eq!(text, "answer"); + let [crate::formats::responses::ResponsesAnnotation::UrlCitation(citation)] = + annotations.as_slice() + else { + panic!("expected URL citation") + }; + assert_eq!(citation.url.as_deref(), Some("https://example.test")); + assert_eq!(citation.start_index, Some(0)); + } + ResponsesOutputItem::FunctionCall(call) => { + assert_eq!(call.call_id.as_deref(), Some("call_1")); + assert_eq!(call.name.as_deref(), Some("lookup")); + assert_eq!(call.arguments.as_deref(), Some("{\"query\":\"q\"}")); + } + ResponsesOutputItem::CustomToolCall(call) => { + assert_eq!(call.name.as_deref(), Some("lookup")); + assert_eq!(call.input.as_deref(), Some("q")); + } + ResponsesOutputItem::Reasoning(reasoning) => { + assert_eq!(reasoning.encrypted_content.as_deref(), Some("opaque")); + let Some(summary) = &reasoning.summary else { + panic!("expected summary") + }; + let [ResponsesContentPart::SummaryText { text, .. }] = summary.as_slice() else { + panic!("expected summary text") + }; + assert_eq!(text, "summary"); + } + ResponsesOutputItem::WebSearchCall(call) => { + let Some(ResponsesWebSearchAction::Search { + queries: Some(queries), + sources: Some(sources), + .. + }) = &call.action + else { + panic!("expected search action") + }; + assert_eq!(queries, &["q"]); + assert_eq!(sources[0].url.as_deref(), Some("https://example.test")); + } + ResponsesOutputItem::FileSearchCall(call) => { + assert_eq!(call.queries.as_deref(), Some(["q".to_owned()].as_slice())); + let Some(results) = &call.results else { + panic!("expected search results") + }; + assert_eq!(results[0].file_id.as_deref(), Some("file_1")); + assert_eq!(results[0].score, Some(1.into())); + assert_eq!( + results[0].attributes.as_ref().unwrap()["custom"], + json!([1, null]) + ); + } + ResponsesOutputItem::CodeInterpreterCall(call) => { + let Some(outputs) = &call.outputs else { + panic!("expected code outputs") + }; + let [ + ResponsesCodeOutput::Logs { logs, .. }, + ResponsesCodeOutput::Image { url, .. }, + ] = outputs.as_slice() + else { + panic!("expected logs and image") + }; + assert_eq!(logs, "done"); + assert_eq!(url, "https://example.test"); + } + ResponsesOutputItem::ImageGenerationCall(call) => { + assert_eq!(call.result.as_deref(), Some("generated")) + } + ResponsesOutputItem::McpCall(call) => { + assert_eq!(call.server_label.as_deref(), Some("server")); + assert_eq!(call.name.as_deref(), Some("lookup")); + assert_eq!(call.arguments.as_deref(), Some("{}")); + assert_eq!(call.output.as_deref(), Some("done")); + } + other => panic!("unexpected variant {other:?}"), + } + assert_eq!(serde_json::to_value(item).unwrap(), wire); + } + + #[rstest] + #[case::find_in_page( + json!({"type":"find_in_page","url":"https://example.test","pattern":"needle"}), + json!({"type":"find_in_page","url":"https://example.test","pattern":"needle"}) + )] + #[case::legacy_find( + json!({"type":"find","url":"https://example.test","pattern":"needle"}), + json!({"type":"find_in_page","url":"https://example.test","pattern":"needle"}) + )] + fn web_search_find_action_exposes_url_and_pattern( + #[case] wire: Value, + #[case] serialized: Value, + ) { + let action: ResponsesWebSearchAction = serde_json::from_value(wire).unwrap(); + let ResponsesWebSearchAction::FindInPage { + url, + pattern, + extra, + } = &action + else { + panic!("expected find_in_page action"); + }; + assert_eq!(url, "https://example.test"); + assert_eq!(pattern, "needle"); + assert!(extra.is_empty()); + assert_eq!(serde_json::to_value(action).unwrap(), serialized); + } + + #[rstest] + #[case::with_url(json!({"type":"open_page","url":"https://example.test"}), Some("https://example.test"))] + #[case::without_url(json!({"type":"open_page"}), None)] + fn web_search_open_page_url_is_optional(#[case] wire: Value, #[case] expected: Option<&str>) { + let action: ResponsesWebSearchAction = serde_json::from_value(wire.clone()).unwrap(); + let ResponsesWebSearchAction::OpenPage { url, .. } = &action else { + panic!("expected open_page action"); + }; + assert_eq!(url.as_deref(), expected); + assert_eq!(serde_json::to_value(action).unwrap(), wire); + } + + #[rstest] + #[case::find_pattern(json!({"type":"find_in_page","url":"u","pattern":false}))] + #[case::find_missing_url(json!({"type":"find_in_page","pattern":"p"}))] + #[case::open_page_url(json!({"type":"open_page","url":7}))] + fn web_search_action_rejects_malformed_fields(#[case] wire: Value) { + assert!(serde_json::from_value::(wire).is_err()); + } + + #[rstest] + #[case::refusal(json!({"type":"refusal","refusal":"refused","future":null}), ResponsesContentPart::Refusal { refusal:"refused".into(), extra:serde_json::Map::from_iter([("future".into(), Value::Null)]) })] + #[case::reasoning(json!({"type":"reasoning_text","text":"reason"}), ResponsesContentPart::ReasoningText { text:"reason".into(), extra:Default::default() })] + fn content_parts_expose_typed_variants( + #[case] wire: Value, + #[case] expected: ResponsesContentPart, + ) { + let content: ResponsesContentPart = serde_json::from_value(wire.clone()).unwrap(); + assert_eq!(content, expected); + assert_eq!(serde_json::to_value(content).unwrap(), wire); + } + + #[rstest] + fn mcp_discovery_exposes_tools_and_nested_schemas() { + let wire = json!({ + "type":"mcp_list_tools","id":"item_1","server_label":"tools", + "tools":[{"name":"lookup","description":"Look up a value","input_schema":{"type":"object","properties":{"query":{"type":"string"}}},"annotations":{"read_only":false},"future":null}], + "extension":[1,null] + }); + let item: ResponsesOutputItem = serde_json::from_value(wire.clone()).unwrap(); + let ResponsesOutputItem::McpListTools(discovery) = &item else { + panic!("expected discovery") + }; + assert_eq!(discovery.server_label.as_deref(), Some("tools")); + let tool = &discovery.tools.as_ref().unwrap()[0]; + assert_eq!(tool.name.as_deref(), Some("lookup")); + let Some(crate::json_schema::JsonSchema::Object(schema)) = &tool.input_schema else { + panic!("expected tool schema") + }; + assert!(schema.properties.as_ref().unwrap().contains_key("query")); + assert_eq!(tool.annotations, Some(json!({"read_only":false}))); + assert_eq!(tool.extra["future"], Value::Null); + assert_eq!(serde_json::to_value(item).unwrap(), wire); + } + + #[rstest] + #[case::partial(json!({"server_label":"tools","tools":[]}))] + #[case::failure(json!({"server_label":"tools","error":"unavailable"}))] + fn mcp_discovery_accepts_partial_payloads(#[case] wire: Value) { + let discovery: crate::formats::responses::ResponsesMcpListTools = + serde_json::from_value(wire.clone()).unwrap(); + assert_eq!(discovery.server_label.as_deref(), Some("tools")); + assert!(discovery.id.is_none()); + assert_eq!(serde_json::to_value(discovery).unwrap(), wire); + } + + #[rstest] + #[case::message(json!("unavailable"), ResponsesMcpError::Message("unavailable".into()))] + #[case::protocol(json!({"type":"mcp_protocol_error","code":-32600,"message":"invalid","extension":null}), ResponsesMcpError::Detail(ResponsesMcpErrorDetail::McpProtocolError {code:-32600,message:"invalid".into(),extra:serde_json::Map::from_iter([("extension".into(), Value::Null)])}))] + #[case::http(json!({"type":"http_error","code":503,"message":"unavailable"}), ResponsesMcpError::Detail(ResponsesMcpErrorDetail::HttpError {code:503,message:"unavailable".into(),extra:Default::default()}))] + #[case::tool(json!({"type":"mcp_tool_execution_error","content":{"arbitrary":[1,null]}}), ResponsesMcpError::Detail(ResponsesMcpErrorDetail::McpToolExecutionError {content:json!({"arbitrary":[1,null]}),extra:Default::default()}))] + fn mcp_call_errors_expose_typed_variants( + #[case] wire: Value, + #[case] expected: ResponsesMcpError, + ) { + let error: ResponsesMcpError = serde_json::from_value(wire.clone()).unwrap(); + assert_eq!(error, expected); + assert_eq!(serde_json::to_value(&error).unwrap(), wire); + let call: ResponsesOutputItem = + serde_json::from_value(json!({"type":"mcp_call","error":wire})).unwrap(); + let ResponsesOutputItem::McpCall(call) = call else { + panic!("expected call") + }; + assert_eq!(call.error, Some(error)); + } + + #[rstest] + #[case::tools_not_array(json!({"type":"mcp_list_tools","tools":{}}))] + #[case::invalid_name(json!({"type":"mcp_list_tools","tools":[{"name":7}]}))] + #[case::invalid_schema(json!({"type":"mcp_list_tools","tools":[{"input_schema":{"properties":{"query":7}}}]}))] + #[case::invalid_error_code(json!({"type":"mcp_call","error":{"type":"http_error","code":"503","message":"unavailable"}}))] + #[case::missing_error_content(json!({"type":"mcp_call","error":{"type":"mcp_tool_execution_error"}}))] + #[case::unknown_error_tag(json!({"type":"mcp_call","error":{"type":"future"}}))] + fn mcp_output_rejects_malformed_known_fields(#[case] wire: Value) { + assert!(serde_json::from_value::(wire).is_err()); + } + + fn program_caller() -> Option { + Some(ResponsesToolCaller::Program { + caller_id: "prog_1".into(), + extra: Default::default(), + }) + } + + fn direct_caller() -> Option { + Some(ResponsesToolCaller::Direct { + extra: Default::default(), + }) + } + + fn safety_check() -> ResponsesSafetyCheck { + ResponsesSafetyCheck { + id: text("sc_1"), + code: text("malicious_instructions"), + message: text("check"), + ..Default::default() + } + } + + #[rstest] + #[case::function_call( + json!({"type":"function_call","id":"fc_1","status":"completed","call_id":"call_1","name":"lookup","arguments":"{}","namespace":"ns","async":true,"caller":{"type":"program","caller_id":"prog_1"}}), + ResponsesOutputItem::FunctionCall(ResponsesFunctionCall { + id: text("fc_1"), status: text("completed"), call_id: text("call_1"), name: text("lookup"), + arguments: text("{}"), namespace: text("ns"), r#async: Some(true), caller: program_caller(), ..Default::default() + }) + )] + #[case::custom_tool_call( + json!({"type":"custom_tool_call","call_id":"call_1","name":"lookup","input":"q","namespace":"ns","async":false,"caller":{"type":"direct"}}), + ResponsesOutputItem::CustomToolCall(ResponsesCustomToolCall { + call_id: text("call_1"), name: text("lookup"), input: text("q"), namespace: text("ns"), + r#async: Some(false), caller: direct_caller(), ..Default::default() + }) + )] + #[case::image_generation_call( + json!({"type":"image_generation_call","id":"ig_1","status":"completed","result":"b64","action":"edit","background":"opaque","output_format":"webp","quality":"high","revised_prompt":"a cat","size":"1536x864"}), + ResponsesOutputItem::ImageGenerationCall(ResponsesImageGenerationCall { + id: text("ig_1"), status: text("completed"), result: text("b64"), action: text("edit"), background: text("opaque"), + output_format: text("webp"), quality: text("high"), revised_prompt: text("a cat"), size: text("1536x864"), ..Default::default() + }) + )] + #[case::function_call_output_text( + json!({"type":"function_call_output","id":"fco_1","status":"completed","call_id":"call_1","output":"done","caller":{"type":"direct"},"created_by":"user_1","name":"lookup","namespace":"ns"}), + ResponsesOutputItem::FunctionCallOutput(ResponsesFunctionCallOutput { + id: text("fco_1"), status: text("completed"), call_id: text("call_1"), output: Some(ResponsesToolOutput::Text("done".into())), + caller: direct_caller(), created_by: text("user_1"), name: text("lookup"), namespace: text("ns"), ..Default::default() + }) + )] + #[case::function_call_output_content( + json!({"type":"function_call_output","call_id":"call_1","output":[ + {"type":"input_text","text":"t","prompt_cache_breakpoint":{"mode":"explicit"}}, + {"type":"input_image","detail":"low","file_id":"file_1","image_url":"https://example.test/i.png"}, + {"type":"input_file","detail":"high","file_data":"data","file_id":"file_2","file_url":"https://example.test/f","filename":"f.pdf"} + ]}), + ResponsesOutputItem::FunctionCallOutput(ResponsesFunctionCallOutput { + call_id: text("call_1"), + output: Some(ResponsesToolOutput::Content(vec![ + ResponsesInputContent::InputText { + text: "t".into(), + prompt_cache_breakpoint: Some(PromptCacheBreakpoint { mode: PromptCacheMode::Explicit, extra: Default::default() }), + extra: Default::default(), + }, + ResponsesInputContent::InputImage { + detail: "low".into(), file_id: text("file_1"), image_url: text("https://example.test/i.png"), + prompt_cache_breakpoint: None, extra: Default::default(), + }, + ResponsesInputContent::InputFile { + detail: text("high"), file_data: text("data"), file_id: text("file_2"), file_url: text("https://example.test/f"), + filename: text("f.pdf"), prompt_cache_breakpoint: None, extra: Default::default(), + }, + ])), + ..Default::default() + }) + )] + #[case::custom_tool_call_output( + json!({"type":"custom_tool_call_output","id":"cto_1","status":"completed","call_id":"call_1","output":[{"type":"input_text","text":"t"}],"caller":{"type":"program","caller_id":"prog_1"},"created_by":"user_1"}), + ResponsesOutputItem::CustomToolCallOutput(ResponsesCustomToolCallOutput { + id: text("cto_1"), status: text("completed"), call_id: text("call_1"), + output: Some(ResponsesToolOutput::Content(vec![ResponsesInputContent::InputText { + text: "t".into(), prompt_cache_breakpoint: None, extra: Default::default(), + }])), + caller: program_caller(), created_by: text("user_1"), ..Default::default() + }) + )] + #[case::computer_call( + json!({"type":"computer_call","id":"cu_1","status":"completed","call_id":"call_1", + "pending_safety_checks":[{"id":"sc_1","code":"malicious_instructions","message":"check"}], + "action":{"type":"click","button":"left","x":1,"y":2,"keys":["shift"]}, + "actions":[ + {"type":"double_click","x":3,"y":4}, + {"type":"drag","path":[{"x":5,"y":6},{"x":7,"y":8}],"keys":["ctrl"]}, + {"type":"keypress","keys":["enter"]}, + {"type":"move","x":-1,"y":9}, + {"type":"screenshot"}, + {"type":"scroll","scroll_x":0,"scroll_y":-10,"x":11,"y":12}, + {"type":"type","text":"hello"}, + {"type":"wait"} + ]}), + ResponsesOutputItem::ComputerCall(ResponsesComputerCall { + id: text("cu_1"), status: text("completed"), call_id: text("call_1"), pending_safety_checks: Some(vec![safety_check()]), + action: Some(ResponsesComputerAction::Click { button: "left".into(), x: 1, y: 2, keys: Some(vec!["shift".into()]), extra: Default::default() }), + actions: Some(vec![ + ResponsesComputerAction::DoubleClick { x: 3, y: 4, keys: None, extra: Default::default() }, + ResponsesComputerAction::Drag { + path: vec![ + ResponsesCoordinate { x: 5, y: 6, extra: Default::default() }, + ResponsesCoordinate { x: 7, y: 8, extra: Default::default() }, + ], + keys: Some(vec!["ctrl".into()]), + extra: Default::default(), + }, + ResponsesComputerAction::Keypress { keys: vec!["enter".into()], extra: Default::default() }, + ResponsesComputerAction::Move { x: -1, y: 9, keys: None, extra: Default::default() }, + ResponsesComputerAction::Screenshot { extra: Default::default() }, + ResponsesComputerAction::Scroll { scroll_x: 0, scroll_y: -10, x: 11, y: 12, keys: None, extra: Default::default() }, + ResponsesComputerAction::Type { text: "hello".into(), extra: Default::default() }, + ResponsesComputerAction::Wait { extra: Default::default() }, + ]), + ..Default::default() + }) + )] + #[case::computer_call_output( + json!({"type":"computer_call_output","id":"cuo_1","status":"completed","call_id":"call_1","output":{"type":"computer_screenshot","file_id":"file_1","image_url":"https://example.test/s.png"},"acknowledged_safety_checks":[{"id":"sc_1","code":"malicious_instructions","message":"check"}],"created_by":"user_1"}), + ResponsesOutputItem::ComputerCallOutput(ResponsesComputerCallOutput { + id: text("cuo_1"), status: text("completed"), call_id: text("call_1"), + output: Some(ResponsesComputerOutput::ComputerScreenshot { file_id: text("file_1"), image_url: text("https://example.test/s.png"), extra: Default::default() }), + acknowledged_safety_checks: Some(vec![safety_check()]), created_by: text("user_1"), ..Default::default() + }) + )] + #[case::program( + json!({"type":"program","id":"prog_item","call_id":"prog_1","code":"run()","fingerprint":"fp"}), + ResponsesOutputItem::Program(ResponsesProgram { + id: text("prog_item"), call_id: text("prog_1"), code: text("run()"), fingerprint: text("fp"), ..Default::default() + }) + )] + #[case::program_output( + json!({"type":"program_output","id":"po_1","status":"completed","call_id":"prog_1","result":"42"}), + ResponsesOutputItem::ProgramOutput(ResponsesProgramOutput { + id: text("po_1"), status: text("completed"), call_id: text("prog_1"), result: text("42"), ..Default::default() + }) + )] + #[case::tool_search_call( + json!({"type":"tool_search_call","id":"ts_1","status":"completed","call_id":"call_1","arguments":{"query":["weather",null]},"execution":"server","created_by":"user_1"}), + ResponsesOutputItem::ToolSearchCall(ResponsesToolSearchCall { + id: text("ts_1"), status: text("completed"), call_id: text("call_1"), arguments: Some(json!({"query":["weather",null]})), + execution: text("server"), created_by: text("user_1"), ..Default::default() + }) + )] + #[case::tool_search_output( + json!({"type":"tool_search_output","id":"tso_1","status":"completed","call_id":"call_1","execution":"client","tools":[{"type":"function","name":"lookup","parameters":null,"strict":true}],"created_by":"user_1"}), + ResponsesOutputItem::ToolSearchOutput(ResponsesToolSearchOutput { + id: text("tso_1"), status: text("completed"), call_id: text("call_1"), execution: text("client"), + tools: Some(vec![serde_json::Map::from_iter([ + ("type".into(), json!("function")), ("name".into(), json!("lookup")), + ("parameters".into(), Value::Null), ("strict".into(), json!(true)), + ])]), + created_by: text("user_1"), ..Default::default() + }) + )] + #[case::additional_tools( + json!({"type":"additional_tools","id":"at_1","role":"developer","tools":[{"type":"local_shell"}]}), + ResponsesOutputItem::AdditionalTools(ResponsesAdditionalTools { + id: text("at_1"), role: text("developer"), + tools: Some(vec![serde_json::Map::from_iter([("type".into(), json!("local_shell"))])]), ..Default::default() + }) + )] + #[case::compaction( + json!({"type":"compaction","id":"cmp_1","encrypted_content":"opaque","created_by":"user_1"}), + ResponsesOutputItem::Compaction(ResponsesCompaction { + id: text("cmp_1"), encrypted_content: text("opaque"), created_by: text("user_1"), ..Default::default() + }) + )] + #[case::local_shell_call( + json!({"type":"local_shell_call","id":"ls_1","status":"completed","call_id":"call_1","action":{"type":"exec","command":["ls","-a"],"env":{"HOME":"/home/u"},"timeout_ms":1000,"user":"u","working_directory":"/tmp"}}), + ResponsesOutputItem::LocalShellCall(ResponsesLocalShellCall { + id: text("ls_1"), status: text("completed"), call_id: text("call_1"), + action: Some(ResponsesLocalShellAction::Exec { + command: vec!["ls".into(), "-a".into()], env: [("HOME".into(), "/home/u".into())].into(), + timeout_ms: Some(1000), user: text("u"), working_directory: text("/tmp"), extra: Default::default(), + }), + ..Default::default() + }) + )] + #[case::local_shell_call_output( + json!({"type":"local_shell_call_output","id":"lso_1","status":"completed","output":"{\"stdout\":\"x\"}"}), + ResponsesOutputItem::LocalShellCallOutput(ResponsesLocalShellCallOutput { + id: text("lso_1"), status: text("completed"), output: text("{\"stdout\":\"x\"}"), ..Default::default() + }) + )] + #[case::shell_call_local( + json!({"type":"shell_call","id":"sh_1","status":"in_progress","call_id":"call_1","action":{"commands":["ls"],"max_output_length":100,"timeout_ms":500},"environment":{"type":"local"},"caller":{"type":"direct"},"created_by":"user_1"}), + ResponsesOutputItem::ShellCall(ResponsesShellCall { + id: text("sh_1"), status: text("in_progress"), call_id: text("call_1"), + action: Some(ResponsesShellAction { commands: Some(vec!["ls".into()]), max_output_length: Some(100), timeout_ms: Some(500), ..Default::default() }), + environment: Some(ResponsesShellEnvironment::Local { extra: Default::default() }), + caller: direct_caller(), created_by: text("user_1"), ..Default::default() + }) + )] + #[case::shell_call_container( + json!({"type":"shell_call","call_id":"call_1","environment":{"type":"container_reference","container_id":"cntr_1"}}), + ResponsesOutputItem::ShellCall(ResponsesShellCall { + call_id: text("call_1"), + environment: Some(ResponsesShellEnvironment::ContainerReference { container_id: "cntr_1".into(), extra: Default::default() }), + ..Default::default() + }) + )] + #[case::shell_call_output( + json!({"type":"shell_call_output","id":"sho_1","status":"completed","call_id":"call_1","max_output_length":100,"output":[ + {"outcome":{"type":"exit","exit_code":2},"stdout":"out","stderr":"err","created_by":"user_1"}, + {"outcome":{"type":"timeout"},"stdout":"","stderr":""} + ],"caller":{"type":"program","caller_id":"prog_1"},"created_by":"user_1"}), + ResponsesOutputItem::ShellCallOutput(ResponsesShellCallOutput { + id: text("sho_1"), status: text("completed"), call_id: text("call_1"), max_output_length: Some(100), + output: Some(vec![ + ResponsesShellOutputChunk { + outcome: Some(ResponsesShellOutcome::Exit { exit_code: 2, extra: Default::default() }), + stdout: text("out"), stderr: text("err"), created_by: text("user_1"), ..Default::default() + }, + ResponsesShellOutputChunk { + outcome: Some(ResponsesShellOutcome::Timeout { extra: Default::default() }), + stdout: text(""), stderr: text(""), ..Default::default() + }, + ]), + caller: program_caller(), created_by: text("user_1"), ..Default::default() + }) + )] + #[case::apply_patch_create( + json!({"type":"apply_patch_call","id":"ap_1","status":"completed","call_id":"call_1","operation":{"type":"create_file","path":"a.txt","diff":"+a"},"caller":{"type":"direct"},"created_by":"user_1"}), + ResponsesOutputItem::ApplyPatchCall(ResponsesApplyPatchCall { + id: text("ap_1"), status: text("completed"), call_id: text("call_1"), + operation: Some(ResponsesApplyPatchOperation::CreateFile { path: "a.txt".into(), diff: "+a".into(), extra: Default::default() }), + caller: direct_caller(), created_by: text("user_1"), ..Default::default() + }) + )] + #[case::apply_patch_delete( + json!({"type":"apply_patch_call","call_id":"call_1","operation":{"type":"delete_file","path":"a.txt"}}), + ResponsesOutputItem::ApplyPatchCall(ResponsesApplyPatchCall { + call_id: text("call_1"), + operation: Some(ResponsesApplyPatchOperation::DeleteFile { path: "a.txt".into(), extra: Default::default() }), + ..Default::default() + }) + )] + #[case::apply_patch_update( + json!({"type":"apply_patch_call","call_id":"call_1","operation":{"type":"update_file","path":"a.txt","diff":"-a\n+b"}}), + ResponsesOutputItem::ApplyPatchCall(ResponsesApplyPatchCall { + call_id: text("call_1"), + operation: Some(ResponsesApplyPatchOperation::UpdateFile { path: "a.txt".into(), diff: "-a\n+b".into(), extra: Default::default() }), + ..Default::default() + }) + )] + #[case::apply_patch_call_output( + json!({"type":"apply_patch_call_output","id":"apo_1","status":"failed","call_id":"call_1","output":"conflict","caller":{"type":"direct"},"created_by":"user_1"}), + ResponsesOutputItem::ApplyPatchCallOutput(ResponsesApplyPatchCallOutput { + id: text("apo_1"), status: text("failed"), call_id: text("call_1"), output: text("conflict"), + caller: direct_caller(), created_by: text("user_1"), ..Default::default() + }) + )] + #[case::mcp_approval_request( + json!({"type":"mcp_approval_request","id":"apr_1","server_label":"s","name":"n","arguments":"{}"}), + ResponsesOutputItem::McpApprovalRequest(ResponsesMcpApprovalRequest { + id: text("apr_1"), server_label: text("s"), name: text("n"), arguments: text("{}"), ..Default::default() + }) + )] + #[case::mcp_approval_response( + json!({"type":"mcp_approval_response","id":"aprr_1","approval_request_id":"apr_1","approve":false,"reason":"denied"}), + ResponsesOutputItem::McpApprovalResponse(ResponsesMcpApprovalResponse { + id: text("aprr_1"), approval_request_id: text("apr_1"), approve: Some(false), reason: text("denied"), ..Default::default() + }) + )] + fn documented_output_items_parse_into_typed_variants( + #[case] wire: Value, + #[case] expected: ResponsesOutputItem, + ) { + let item: ResponsesOutputItem = serde_json::from_value(wire.clone()).unwrap(); + assert_eq!(item, expected); + assert_eq!(serde_json::to_value(item).unwrap(), wire); + } + + #[rstest] + #[case::unknown_caller(json!({"type":"function_call","caller":{"type":"future"}}))] + #[case::program_caller_missing_id(json!({"type":"shell_call","caller":{"type":"program"}}))] + #[case::untagged_caller(json!({"type":"custom_tool_call","caller":{}}))] + #[case::async_not_bool(json!({"type":"function_call","async":"yes"}))] + #[case::output_not_text_or_list(json!({"type":"function_call_output","output":{"text":"t"}}))] + #[case::unknown_input_content(json!({"type":"function_call_output","output":[{"type":"input_audio"}]}))] + #[case::input_text_missing_text(json!({"type":"custom_tool_call_output","output":[{"type":"input_text"}]}))] + #[case::input_image_missing_detail(json!({"type":"custom_tool_call_output","output":[{"type":"input_image","file_id":"f"}]}))] + #[case::bad_cache_mode(json!({"type":"custom_tool_call_output","output":[{"type":"input_text","text":"t","prompt_cache_breakpoint":{"mode":"implicit"}}]}))] + #[case::unknown_computer_action(json!({"type":"computer_call","action":{"type":"teleport"}}))] + #[case::click_missing_button(json!({"type":"computer_call","action":{"type":"click","x":1,"y":2}}))] + #[case::click_fractional_x(json!({"type":"computer_call","action":{"type":"click","button":"left","x":1.5,"y":2}}))] + #[case::drag_point_missing_y(json!({"type":"computer_call","actions":[{"type":"drag","path":[{"x":1}]}]}))] + #[case::keypress_keys_not_list(json!({"type":"computer_call","actions":[{"type":"keypress","keys":"enter"}]}))] + #[case::safety_check_not_object(json!({"type":"computer_call","pending_safety_checks":["sc_1"]}))] + #[case::unknown_screenshot_tag(json!({"type":"computer_call_output","output":{"type":"screenshot"}}))] + #[case::untagged_screenshot(json!({"type":"computer_call_output","output":{"file_id":"f"}}))] + #[case::tool_not_object(json!({"type":"tool_search_output","tools":["lookup"]}))] + #[case::unknown_local_shell_action(json!({"type":"local_shell_call","action":{"type":"spawn","command":[],"env":{}}}))] + #[case::local_shell_missing_env(json!({"type":"local_shell_call","action":{"type":"exec","command":["ls"]}}))] + #[case::local_shell_env_value(json!({"type":"local_shell_call","action":{"type":"exec","command":["ls"],"env":{"A":1}}}))] + #[case::shell_commands_not_list(json!({"type":"shell_call","action":{"commands":"ls"}}))] + #[case::unknown_shell_environment(json!({"type":"shell_call","environment":{"type":"container_auto"}}))] + #[case::container_missing_id(json!({"type":"shell_call","environment":{"type":"container_reference"}}))] + #[case::unknown_shell_outcome(json!({"type":"shell_call_output","output":[{"outcome":{"type":"killed"}}]}))] + #[case::exit_missing_code(json!({"type":"shell_call_output","output":[{"outcome":{"type":"exit"}}]}))] + #[case::negative_max_output(json!({"type":"shell_call_output","max_output_length":-1}))] + #[case::unknown_patch_operation(json!({"type":"apply_patch_call","operation":{"type":"rename_file","path":"a"}}))] + #[case::update_missing_diff(json!({"type":"apply_patch_call","operation":{"type":"update_file","path":"a"}}))] + #[case::approve_not_bool(json!({"type":"mcp_approval_response","approve":"true"}))] + #[case::compaction_content_not_string(json!({"type":"compaction","encrypted_content":7}))] + #[case::program_code_not_string(json!({"type":"program","code":["run()"]}))] + fn output_items_reject_malformed_typed_fields(#[case] wire: Value) { + assert!(serde_json::from_value::(wire).is_err()); + } + + fn text(value: &str) -> Option { + Some(value.to_owned()) + } +} diff --git a/litellm-rust/crates/llms-types/src/formats/responses/streaming_websocket.rs b/litellm-rust/crates/llms-types/src/formats/responses/streaming_websocket.rs index 6053eea3e14..741540b5741 100644 --- a/litellm-rust/crates/llms-types/src/formats/responses/streaming_websocket.rs +++ b/litellm-rust/crates/llms-types/src/formats/responses/streaming_websocket.rs @@ -106,9 +106,18 @@ pub struct ResponsesEventResponse { #[cfg(test)] mod tests { + use super::*; use rstest::rstest; - use super::*; + use crate::formats::responses::ResponsesOutputItem; + + use crate::{ + formats::responses::streaming_websocket::{ResponsesEventResponse, ResponsesWsEventType}, + recognized::Recognized, + test_support::*, + }; + + use serde_json::{Value, json}; #[test] fn error_frame_matches_proxy_shape() { @@ -135,4 +144,145 @@ mod tests { let event: ResponsesWsEvent = serde_json::from_value(payload).expect("valid event"); assert_eq!(event.model(), expected); } + + #[rstest] + #[case::create("response.create", ResponsesWsEventType::ResponseCreate)] + #[case::created("response.created", ResponsesWsEventType::ResponseCreated)] + #[case::completed("response.completed", ResponsesWsEventType::ResponseCompleted)] + #[case::failed("response.failed", ResponsesWsEventType::ResponseFailed)] + #[case::incomplete("response.incomplete", ResponsesWsEventType::ResponseIncomplete)] + #[case::error("error", ResponsesWsEventType::Error)] + #[case::unknown( + "response.output_text.delta", + ResponsesWsEventType::Other("response.output_text.delta".to_string()) + )] + #[case::empty("", ResponsesWsEventType::Other(String::new()))] + #[case::case_sensitive( + "Response.Completed", + ResponsesWsEventType::Other("Response.Completed".into()) + )] + #[case::escaped("future\"\\\n", ResponsesWsEventType::Other("future\"\\\n".into()))] + fn websocket_event_type_round_trips( + #[case] wire: &str, + #[case] expected: ResponsesWsEventType, + ) { + let serialized = serde_json::to_string(&expected).unwrap(); + assert_eq!(serialized, serde_json::to_string(wire).unwrap()); + assert_eq!( + serde_json::from_str::(&serialized).unwrap(), + expected + ); + } + + #[rstest] + #[case::number("17")] + #[case::boolean("true")] + #[case::null("null")] + #[case::array("[]")] + #[case::object("{}")] + fn websocket_event_type_rejects_non_strings(#[case] wire: &str) { + assert!(serde_json::from_str::(wire).is_err()); + } + + #[cfg(feature = "schema")] + #[rstest] + fn websocket_event_type_schema_is_open_string() { + let schema = schemars::schema_for!(ResponsesWsEventType); + assert_eq!( + schema.to_value().get("type"), + Some(&serde_json::json!("string")) + ); + } + + #[rstest] + fn nested_event_response_exposes_typed_output_and_preserves_extensions() { + let wire = json!({ + "id":"response_1", + "model":"example-model", + "status":"completed", + "output":[{"type":"function_call","call_id":"call_1","name":"lookup","arguments":"{}","extension":true}], + "extension":{"nested":[1,null]} + }); + let response: ResponsesEventResponse = serde_json::from_value(wire.clone()).unwrap(); + assert_eq!(response.id.as_deref(), Some("response_1")); + assert_eq!(response.model.as_deref(), Some("example-model")); + let Some(output) = &response.output else { + panic!("expected typed output"); + }; + let [Recognized::Known(ResponsesOutputItem::FunctionCall(call))] = output.as_slice() else { + panic!("expected function call"); + }; + assert_eq!(call.name.as_deref(), Some("lookup")); + assert_eq!(call.arguments.as_deref(), Some("{}")); + assert_eq!(serde_json::to_value(response).unwrap(), wire); + } + + #[rstest] + #[case::empty(json!({}))] + #[case::partial(json!({"id":"response_1","output":[]}))] + fn nested_event_response_accepts_partial_metadata(#[case] wire: Value) { + round_trip::(wire); + } + + #[rstest] + #[case::wrong_model(json!({"model":7}))] + #[case::wrong_status(json!({"status":false}))] + #[case::wrong_output(json!({"output":{}}))] + fn event_response_rejects_malformed_typed_fields(#[case] wire: Value) { + assert!(serde_json::from_value::(wire).is_err()); + } + + #[rstest] + #[case::unknown_type(json!({"type":"future_item","id":"item_1","payload":[1,null]}))] + #[case::missing_tag(json!({"id":"item_1"}))] + #[case::malformed_known(json!({"type":"message","content":[{"type":"output_text","text":7}]}))] + fn event_response_keeps_unmodeled_output_items_beside_typed_ones(#[case] item: Value) { + let wire = json!({"output":[{"type":"function_call","name":"lookup"}, item.clone()]}); + let response: ResponsesEventResponse = serde_json::from_value(wire.clone()).unwrap(); + let Some( + [ + Recognized::Known(ResponsesOutputItem::FunctionCall(call)), + Recognized::Unrecognized(kept), + ], + ) = response.output.as_deref() + else { + panic!("expected one typed item and one preserved item"); + }; + assert_eq!(call.name.as_deref(), Some("lookup")); + assert_eq!(kept, &item); + assert_eq!(serde_json::to_value(response).unwrap(), wire); + } + + #[rstest] + fn event_response_optional_fields_omit_missing_and_null() { + let response: ResponsesEventResponse = serde_json::from_value(json!({ + "id":null,"model":null,"status":null,"output":null,"future":null + })) + .unwrap(); + assert!(response.id.is_none()); + assert!(response.model.is_none()); + assert!(response.status.is_none()); + assert!(response.output.is_none()); + assert_eq!( + serde_json::to_value(response).unwrap(), + json!({"future":null}) + ); + } + + #[rstest] + #[case::compaction(json!({"type":"compaction","id":"cmp_1","encrypted_content":"opaque"}))] + #[case::approval(json!({"type":"mcp_approval_request","id":"apr_1","server_label":"s","name":"n","arguments":"{}"}))] + #[case::shell(json!({"type":"shell_call","call_id":"call_1","action":{"commands":["ls"]}}))] + fn event_response_types_documented_output_items(#[case] item: Value) { + let wire = json!({"output":[item.clone()]}); + let response: ResponsesEventResponse = serde_json::from_value(wire.clone()).unwrap(); + let Some([Recognized::Known(known)]) = response.output.as_deref() else { + panic!("expected one typed item"); + }; + assert_eq!( + known, + &serde_json::from_value::(item).unwrap() + ); + assert_eq!(serde_json::to_value(response).unwrap(), wire); + } } diff --git a/litellm-rust/crates/llms-types/src/json_schema.rs b/litellm-rust/crates/llms-types/src/json_schema.rs index 50a96ebef09..c1e7658300a 100644 --- a/litellm-rust/crates/llms-types/src/json_schema.rs +++ b/litellm-rust/crates/llms-types/src/json_schema.rs @@ -49,3 +49,147 @@ pub struct JsonSchemaObject { #[serde(flatten)] pub extra: Map, } + +#[cfg(test)] +mod tests { + use crate::{ + json_schema::{JsonSchema, JsonSchemaItems, JsonSchemaObject, JsonSchemaType}, + test_support::*, + }; + use rstest::rstest; + + use serde_json::{Value, json}; + + #[rstest] + #[case::boolean(json!(false))] + #[case::nested(json!({ + "type":["object","null"], + "properties":{"nested":{"type":"array","items":{"$ref":"#/$defs/item"}},"free":true}, + "additionalProperties":{"type":"string"}, + "$defs":{"item":{"anyOf":[{"type":"string"},false]}}, + "required":["nested"], + "enum":[{"custom":[1,null]}], + "future":null + }))] + fn recursive_schema_round_trips(#[case] wire: Value) { + let schema = round_trip::(wire); + match schema { + JsonSchema::Boolean(allowed) => assert!(!allowed), + JsonSchema::Object(schema) => { + assert_eq!( + schema.schema_type, + Some(JsonSchemaType::Names(vec!["object".into(), "null".into()])) + ); + let properties = schema.properties.as_ref().unwrap(); + let JsonSchema::Object(nested) = &properties["nested"] else { + panic!("expected nested schema") + }; + let Some(JsonSchemaItems::Schema(items)) = &nested.items else { + panic!("expected single items schema") + }; + let JsonSchema::Object(items) = items.as_ref() else { + panic!("expected items schema") + }; + assert_eq!(items.reference.as_deref(), Some("#/$defs/item")); + assert_eq!(properties["free"], JsonSchema::Boolean(true)); + let JsonSchema::Object(definition) = &schema.defs.as_ref().unwrap()["item"] else { + panic!("expected schema definition") + }; + assert_eq!( + definition.any_of.as_ref().unwrap()[1], + JsonSchema::Boolean(false) + ); + assert_eq!(schema.extra["enum"], json!([{"custom":[1,null]}])); + } + } + } + + #[rstest] + fn schema_maps_keep_wire_key_order() { + let wire = r#"{"type":"object","properties":{"zeta":{"type":"string"},"alpha":{"type":"integer"}},"$defs":{"y":true,"b":false}}"#; + let schema: JsonSchemaObject = serde_json::from_str(wire).unwrap(); + let names: Vec<&str> = schema + .properties + .as_ref() + .unwrap() + .keys() + .map(String::as_str) + .collect(); + assert_eq!(names, ["zeta", "alpha"]); + assert_eq!(serde_json::to_string(&schema).unwrap(), wire); + } + + #[rstest] + fn schema_object_round_trips_supported_keywords() { + let schema = round_trip::(json!({ + "type":"object", + "properties":{"name":{"type":"string"}}, + "required":["name"], + "additionalProperties":false, + "$ref":"#/$defs/value", + "strict":true, + "extension":{"nested":[1,null]} + })); + assert_eq!( + schema.schema_type, + Some(JsonSchemaType::Name("object".into())) + ); + assert_eq!(schema.required.as_deref(), Some(["name".into()].as_slice())); + assert_eq!( + schema.additional_properties.as_deref(), + Some(&JsonSchema::Boolean(false)) + ); + assert_eq!(schema.reference.as_deref(), Some("#/$defs/value")); + assert_eq!(schema.strict, Some(true)); + } + + #[rstest] + #[case::scalar(json!(7))] + #[case::properties_shape(json!({"properties":[]}))] + #[case::nested_schema(json!({"properties":{"name":7}}))] + #[case::schema_type(json!({"type":["string",7]}))] + #[case::items_shape(json!({"items":7}))] + #[case::tuple_member(json!({"items":[{"type":"string"},7]}))] + #[case::prefix_items_shape(json!({"prefixItems":{"type":"string"}}))] + #[case::composite_shape(json!({"anyOf":["string"]}))] + fn schemas_reject_malformed_known_keywords(#[case] wire: Value) { + assert!(serde_json::from_value::(wire).is_err()); + } + + #[rstest] + #[case::draft_07_tuple(json!({"type":"array","items":[{"type":"string"},true]}))] + #[case::draft_2020_12_tuple(json!({"type":"array","prefixItems":[{"type":"string"},true],"items":false}))] + fn tuple_schemas_expose_positional_members(#[case] wire: Value) { + let schema = round_trip::(wire); + let members = match (&schema.items, &schema.prefix_items) { + (Some(JsonSchemaItems::Tuple(members)), None) => members, + (Some(JsonSchemaItems::Schema(rest)), Some(members)) => { + assert_eq!(rest.as_ref(), &JsonSchema::Boolean(false)); + members + } + _ => panic!("expected positional members"), + }; + let [JsonSchema::Object(first), JsonSchema::Boolean(true)] = members.as_slice() else { + panic!("expected string schema then true"); + }; + assert_eq!( + first.schema_type, + Some(JsonSchemaType::Name("string".into())) + ); + assert!(schema.extra.is_empty()); + } + + #[rstest] + fn empty_schema_omits_null_optionals_and_preserves_arbitrary_keywords() { + let schema: JsonSchemaObject = serde_json::from_value( + json!({"type":null,"properties":null,"const":{"arbitrary":[1,null]},"future":null}), + ) + .unwrap(); + assert!(schema.schema_type.is_none()); + assert!(schema.properties.is_none()); + assert_eq!( + serde_json::to_value(schema).unwrap(), + json!({"const":{"arbitrary":[1,null]},"future":null}) + ); + } +} diff --git a/litellm-rust/crates/llms-types/src/lib.rs b/litellm-rust/crates/llms-types/src/lib.rs index 814122f2ab9..67d933e39cc 100644 --- a/litellm-rust/crates/llms-types/src/lib.rs +++ b/litellm-rust/crates/llms-types/src/lib.rs @@ -10,3 +10,43 @@ pub mod json_schema; pub mod providers; pub mod recognized; pub mod serde_compat; + +#[cfg(test)] +mod test_support; + +#[cfg(test)] +mod tests { + use crate::formats::chat_completions::ChatMessage; + + use rstest::rstest; + use serde_json::json; + + #[rstest] + fn wire_type_preserves_serialization() { + let message = ChatMessage { + role: "user".to_owned(), + content: None, + name: None, + extra: Default::default(), + }; + + assert_eq!( + serde_json::to_value(message).unwrap(), + json!({"role": "user"}) + ); + } + + #[cfg(feature = "schema")] + #[rstest] + fn wire_type_supports_schema_generation() { + let schema = schemars::schema_for!(ChatMessage); + + assert!( + schema + .to_value() + .get("properties") + .and_then(serde_json::Value::as_object) + .is_some_and(|properties| properties.contains_key("role")) + ); + } +} diff --git a/litellm-rust/crates/llms-types/src/providers/AGENTS.md b/litellm-rust/crates/llms-types/src/providers/AGENTS.md deleted file mode 100644 index 128d29652d1..00000000000 --- a/litellm-rust/crates/llms-types/src/providers/AGENTS.md +++ /dev/null @@ -1,18 +0,0 @@ -# references - -These links describe the provider-specific wire types in this directory. Shared Messages payload references live in `../formats/messages/AGENTS.md` - -## anthropic.rs - -`AnthropicBeta`, `BetaSet` and `BetaProvider` represent the `anthropic-beta` header values, their -wire spelling and per-host support - -- https://platform.claude.com/docs/en/api/beta-headers.md - -## minimax.rs - -MiniMax's Messages reference documents its image, video, and mid-conversation system content extensions. Its cache reference documents `cache_control` on those blocks - -- https://platform.minimax.io/docs/api-reference/text-chat-anthropic -- https://platform.minimax.io/docs/api-reference/text-chat-anthropic.md -- https://platform.minimax.io/docs/api-reference/anthropic-api-compatible-cache.md diff --git a/litellm-rust/crates/llms-types/src/providers/anthropic/AGENTS.md b/litellm-rust/crates/llms-types/src/providers/anthropic/AGENTS.md new file mode 100644 index 00000000000..f2984624b01 --- /dev/null +++ b/litellm-rust/crates/llms-types/src/providers/anthropic/AGENTS.md @@ -0,0 +1,9 @@ +# rules + +- Shared across Anthropic's Messages, chat, count-tokens and batches adapters: API base, endpoint paths, header names, API version and default headers +- `beta.rs` holds `AnthropicBeta`, `BetaSet` and `BetaProvider`, the `anthropic-beta` values, their wire spelling and which hosts accept each one +- Choosing which betas a request needs is policy in `llms/src/anthropic/` + +# references + +- https://platform.claude.com/docs/en/api/beta-headers.md diff --git a/litellm-rust/crates/llms-types/src/providers/anthropic.rs b/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs similarity index 99% rename from litellm-rust/crates/llms-types/src/providers/anthropic.rs rename to litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs index 97bb26a4088..0db4de3f43c 100644 --- a/litellm-rust/crates/llms-types/src/providers/anthropic.rs +++ b/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs @@ -380,16 +380,14 @@ impl fmt::Display for BetaSet { #[cfg(test)] mod tests { - use std::collections::BTreeSet; - + use super::*; use indexmap::IndexMap; use rstest::rstest; - - use super::*; + use std::collections::BTreeSet; fn beta_headers_config() -> IndexMap { serde_json::from_str(include_str!( - "../../../../../litellm/anthropic_beta_headers_config.json" + "../../../../../../litellm/anthropic_beta_headers_config.json" )) .unwrap() } diff --git a/litellm-rust/crates/llms-types/src/providers/anthropic/constants.rs b/litellm-rust/crates/llms-types/src/providers/anthropic/constants.rs new file mode 100644 index 00000000000..941d1e75461 --- /dev/null +++ b/litellm-rust/crates/llms-types/src/providers/anthropic/constants.rs @@ -0,0 +1,15 @@ +pub const API_BASE: &str = "https://api.anthropic.com"; +pub const MESSAGES_PATH: &str = "/v1/messages"; +pub const BATCHES_PATH: &str = "/v1/messages/batches"; +pub const COUNT_TOKENS_PATH: &str = "/v1/messages/count_tokens"; + +pub const API_KEY_HEADER: &str = "x-api-key"; +pub const BETA_HEADER: &str = "anthropic-beta"; +pub const VERSION_HEADER: &str = "anthropic-version"; +pub const DIRECT_BROWSER_ACCESS_HEADER: &str = "anthropic-dangerous-direct-browser-access"; +pub const API_VERSION: &str = "2023-06-01"; + +pub const DEFAULT_HEADERS: &[(&str, &str)] = &[ + (VERSION_HEADER, API_VERSION), + ("content-type", "application/json"), +]; diff --git a/litellm-rust/crates/llms-types/src/providers/anthropic/mod.rs b/litellm-rust/crates/llms-types/src/providers/anthropic/mod.rs new file mode 100644 index 00000000000..d37a1509b70 --- /dev/null +++ b/litellm-rust/crates/llms-types/src/providers/anthropic/mod.rs @@ -0,0 +1,5 @@ +mod beta; +mod constants; + +pub use beta::*; +pub use constants::*; diff --git a/litellm-rust/crates/llms-types/src/providers/bedrock/AGENTS.md b/litellm-rust/crates/llms-types/src/providers/bedrock/AGENTS.md new file mode 100644 index 00000000000..8c57247dc11 --- /dev/null +++ b/litellm-rust/crates/llms-types/src/providers/bedrock/AGENTS.md @@ -0,0 +1,10 @@ +# rules + +- Shared across Bedrock's chat, Messages and audio-transcription adapters: Converse and InvokeModel paths, the invocation-metrics key and the Converse response body +- The Converse response is a partial projection. Unread blocks decode as `ConverseContentBlock::Other` so adapters can decline them + +# references + +- https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html +- https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_InvokeModel.html +- https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_InvokeModelWithResponseStream.html diff --git a/litellm-rust/crates/llms-types/src/providers/bedrock/constants.rs b/litellm-rust/crates/llms-types/src/providers/bedrock/constants.rs new file mode 100644 index 00000000000..82c248421a3 --- /dev/null +++ b/litellm-rust/crates/llms-types/src/providers/bedrock/constants.rs @@ -0,0 +1,4 @@ +pub const CONVERSE_PATH: &str = "/converse"; +pub const INVOKE_PATH: &str = "invoke"; +pub const INVOKE_STREAM_PATH: &str = "invoke-with-response-stream"; +pub const INVOCATION_METRICS_KEY: &str = "amazon-bedrock-invocationMetrics"; diff --git a/litellm-rust/crates/llms-types/src/providers/bedrock/converse.rs b/litellm-rust/crates/llms-types/src/providers/bedrock/converse.rs new file mode 100644 index 00000000000..a7a435ffebc --- /dev/null +++ b/litellm-rust/crates/llms-types/src/providers/bedrock/converse.rs @@ -0,0 +1,76 @@ +use serde::Deserialize; +use serde_json::Value; + +#[derive(Debug, Deserialize)] +pub struct ConverseResponse { + pub output: ConverseOutput, + #[serde(default)] + pub usage: ConverseUsage, + #[serde(rename = "stopReason")] + pub stop_reason: Option, +} + +#[derive(Debug, Deserialize)] +pub struct ConverseOutput { + pub message: ConverseMessage, +} + +#[derive(Debug, Deserialize)] +pub struct ConverseMessage { + pub content: Vec, +} + +#[derive(Debug)] +pub enum ConverseContentBlock { + Text { text: String }, + Other, +} + +impl<'de> Deserialize<'de> for ConverseContentBlock { + fn deserialize>(deserializer: D) -> Result { + let value = Value::deserialize(deserializer)?; + let Some(text) = value.get("text") else { + return Ok(Self::Other); + }; + let text = text.as_str().ok_or_else(|| { + serde::de::Error::custom("invalid type for `text`, expected a string") + })?; + Ok(Self::Text { + text: text.to_owned(), + }) + } +} + +#[derive(Debug, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ConverseUsage { + pub input_tokens: u64, + pub output_tokens: u64, + #[serde(default)] + pub cache_read_input_tokens: u64, + #[serde(default)] + pub cache_write_input_tokens: u64, + pub total_tokens: Option, +} + +impl ConverseResponse { + pub fn content_text(&self) -> String { + self.output + .message + .content + .iter() + .filter_map(|block| match block { + ConverseContentBlock::Text { text } => Some(text.as_str()), + ConverseContentBlock::Other => None, + }) + .collect() + } + + pub fn message_content_is_non_text(&self) -> bool { + self.output + .message + .content + .iter() + .any(|block| matches!(block, ConverseContentBlock::Other)) + } +} diff --git a/litellm-rust/crates/llms-types/src/providers/bedrock/mod.rs b/litellm-rust/crates/llms-types/src/providers/bedrock/mod.rs new file mode 100644 index 00000000000..4b91879121f --- /dev/null +++ b/litellm-rust/crates/llms-types/src/providers/bedrock/mod.rs @@ -0,0 +1,5 @@ +mod constants; +mod converse; + +pub use constants::*; +pub use converse::*; diff --git a/litellm-rust/crates/llms-types/src/providers/minimax.rs b/litellm-rust/crates/llms-types/src/providers/minimax.rs deleted file mode 100644 index 7ddbf32af2c..00000000000 --- a/litellm-rust/crates/llms-types/src/providers/minimax.rs +++ /dev/null @@ -1,59 +0,0 @@ -use serde_json::{Map, Value}; - -use crate::formats::messages::CacheControl; - -#[macro_rules_attribute::apply(crate::wire_type)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum MinimaxMessagesContentBlock { - Image(MinimaxMediaBlock), - Video(MinimaxMediaBlock), - MidConvSystem { - text: String, - #[serde(flatten)] - extra: Map, - }, -} - -#[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(crate::wire_type)] -pub struct MinimaxMediaBlock { - pub source: MinimaxMediaSource, - pub cache_control: Option, - #[serde(flatten)] - pub extra: Map, -} - -#[macro_rules_attribute::apply(crate::wire_type)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum MinimaxMediaSource { - Base64 { - media_type: String, - data: String, - #[serde(flatten)] - options: MinimaxMediaOptions, - }, - Url { - url: String, - #[serde(flatten)] - options: MinimaxMediaOptions, - }, -} - -#[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(crate::wire_type)] -#[derive(Default)] -pub struct MinimaxMediaOptions { - pub detail: Option, - pub fps: Option, - pub max_long_side_pixel: Option, - #[serde(flatten)] - pub extra: Map, -} - -#[macro_rules_attribute::apply(crate::wire_type)] -#[serde(rename_all = "snake_case")] -pub enum MinimaxMediaDetail { - Low, - Default, - High, -} diff --git a/litellm-rust/crates/llms-types/src/providers/minimax/AGENTS.md b/litellm-rust/crates/llms-types/src/providers/minimax/AGENTS.md new file mode 100644 index 00000000000..bed656a5a3c --- /dev/null +++ b/litellm-rust/crates/llms-types/src/providers/minimax/AGENTS.md @@ -0,0 +1,9 @@ +# rules + +- Typed MiniMax extensions of the Messages format: image, video and mid-conversation system content blocks, with `cache_control` on them + +# references + +- https://platform.minimax.io/docs/api-reference/text-chat-anthropic +- https://platform.minimax.io/docs/api-reference/text-chat-anthropic.md +- https://platform.minimax.io/docs/api-reference/anthropic-api-compatible-cache.md diff --git a/litellm-rust/crates/llms-types/src/providers/minimax/messages.rs b/litellm-rust/crates/llms-types/src/providers/minimax/messages.rs new file mode 100644 index 00000000000..dbecb82b8f8 --- /dev/null +++ b/litellm-rust/crates/llms-types/src/providers/minimax/messages.rs @@ -0,0 +1,146 @@ +use serde_json::{Map, Value}; + +use crate::formats::messages::CacheControl; + +#[macro_rules_attribute::apply(crate::wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum MinimaxMessagesContentBlock { + Image(MinimaxMediaBlock), + Video(MinimaxMediaBlock), + MidConvSystem { + text: String, + #[serde(flatten)] + extra: Map, + }, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(crate::wire_type)] +pub struct MinimaxMediaBlock { + pub source: MinimaxMediaSource, + pub cache_control: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(crate::wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum MinimaxMediaSource { + Base64 { + media_type: String, + data: String, + #[serde(flatten)] + options: MinimaxMediaOptions, + }, + Url { + url: String, + #[serde(flatten)] + options: MinimaxMediaOptions, + }, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(crate::wire_type)] +#[derive(Default)] +pub struct MinimaxMediaOptions { + pub detail: Option, + pub fps: Option, + pub max_long_side_pixel: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(crate::wire_type)] +#[serde(rename_all = "snake_case")] +pub enum MinimaxMediaDetail { + Low, + Default, + High, +} + +#[cfg(test)] +mod tests { + use crate::providers::minimax::{ + MinimaxMediaDetail, MinimaxMediaSource, MinimaxMessagesContentBlock, + }; + + use rstest::rstest; + use serde_json::{Value, json}; + + #[rstest] + #[case::image(json!({"type":"image","source":{"type":"base64","media_type":"image/png","data":"AA==","detail":"low"}}))] + #[case::video(json!({"type":"video","source":{"type":"url","url":"https://example.test/video","detail":"high","fps":1,"max_long_side_pixel":1024,"future":null},"cache_control":{"type":"ephemeral"}}))] + #[case::mid_conversation_system(json!({"type":"mid_conv_system","text":"instruction","future":null}))] + fn provider_content_blocks_round_trip(#[case] wire: Value) { + let block: MinimaxMessagesContentBlock = serde_json::from_value(wire.clone()).unwrap(); + match &block { + MinimaxMessagesContentBlock::Image(image) => { + let MinimaxMediaSource::Base64 { + media_type, + data, + options, + } = &image.source + else { + panic!("expected base64 image source"); + }; + assert_eq!((media_type.as_str(), data.as_str()), ("image/png", "AA==")); + assert_eq!(options.detail, Some(MinimaxMediaDetail::Low)); + assert!(options.extra.is_empty()); + } + MinimaxMessagesContentBlock::Video(video) => { + let MinimaxMediaSource::Url { url, options } = &video.source else { + panic!("expected URL video source"); + }; + assert_eq!(url, "https://example.test/video"); + assert_eq!(options.detail, Some(MinimaxMediaDetail::High)); + assert_eq!(options.fps, Some(1.into())); + assert_eq!(options.max_long_side_pixel, Some(1024)); + assert_eq!(options.extra["future"], Value::Null); + assert_eq!( + video.cache_control.as_ref().unwrap().cache_type.as_deref(), + Some("ephemeral") + ); + } + MinimaxMessagesContentBlock::MidConvSystem { text, extra } => { + assert_eq!(text, "instruction"); + assert_eq!(extra["future"], Value::Null); + } + } + assert_eq!(serde_json::to_value(block).unwrap(), wire); + } + + #[rstest] + #[case::missing_source(json!({"type":"video"}))] + #[case::bad_source_tag(json!({"type":"image","source":{"type":"future"}}))] + #[case::url_without_url(json!({"type":"video","source":{"type":"url"}}))] + #[case::base64_without_data(json!({"type":"image","source":{"type":"base64","media_type":"image/png"}}))] + #[case::bad_detail(json!({"type":"image","source":{"type":"url","url":"u","detail":7}}))] + #[case::bad_fps(json!({"type":"video","source":{"type":"url","url":"u","fps":"fast"}}))] + #[case::missing_text(json!({"type":"mid_conv_system"}))] + fn provider_content_rejects_malformed_fields(#[case] wire: Value) { + assert!(serde_json::from_value::(wire).is_err()); + } + + #[rstest] + fn partial_media_options_omit_null_optionals() { + let block: MinimaxMessagesContentBlock = serde_json::from_value(json!({ + "type":"video", + "source":{"type":"url","url":"u","detail":null,"fps":null,"future":null}, + "cache_control":null + })) + .unwrap(); + let MinimaxMessagesContentBlock::Video(video) = &block else { + panic!("expected video") + }; + let MinimaxMediaSource::Url { options, .. } = &video.source else { + panic!("expected URL source"); + }; + assert!(options.detail.is_none()); + assert!(options.fps.is_none()); + assert!(video.cache_control.is_none()); + assert_eq!( + serde_json::to_value(block).unwrap(), + json!({"type":"video","source":{"type":"url","url":"u","future":null}}) + ); + } +} diff --git a/litellm-rust/crates/llms-types/src/providers/minimax/mod.rs b/litellm-rust/crates/llms-types/src/providers/minimax/mod.rs new file mode 100644 index 00000000000..299ee6fcabf --- /dev/null +++ b/litellm-rust/crates/llms-types/src/providers/minimax/mod.rs @@ -0,0 +1,3 @@ +mod messages; + +pub use messages::*; diff --git a/litellm-rust/crates/llms-types/src/providers/mod.rs b/litellm-rust/crates/llms-types/src/providers/mod.rs index c9eafe964ef..35fea5fe81d 100644 --- a/litellm-rust/crates/llms-types/src/providers/mod.rs +++ b/litellm-rust/crates/llms-types/src/providers/mod.rs @@ -1,2 +1,4 @@ pub mod anthropic; +pub mod bedrock; pub mod minimax; +pub mod vertex_ai; diff --git a/litellm-rust/crates/llms-types/src/providers/vertex_ai/AGENTS.md b/litellm-rust/crates/llms-types/src/providers/vertex_ai/AGENTS.md new file mode 100644 index 00000000000..03b04d87af1 --- /dev/null +++ b/litellm-rust/crates/llms-types/src/providers/vertex_ai/AGENTS.md @@ -0,0 +1,8 @@ +# rules + +- Shared across Vertex AI's Messages and OCR adapters: the global host, the global and default locations, the `rawPredict` methods and the Anthropic API version Vertex expects in the body +- Location validation and host selection are in `llms/src/vertex_ai/common_utils.rs` + +# references + +- https://cloud.google.com/vertex-ai/docs/general/locations diff --git a/litellm-rust/crates/llms-types/src/providers/vertex_ai/constants.rs b/litellm-rust/crates/llms-types/src/providers/vertex_ai/constants.rs new file mode 100644 index 00000000000..fccf7cdefe8 --- /dev/null +++ b/litellm-rust/crates/llms-types/src/providers/vertex_ai/constants.rs @@ -0,0 +1,7 @@ +pub const GLOBAL_API_BASE: &str = "https://aiplatform.googleapis.com"; +pub const GLOBAL_LOCATION: &str = "global"; +pub const DEFAULT_LOCATION: &str = "us-central1"; + +pub const RAW_PREDICT: &str = "rawPredict"; +pub const STREAM_RAW_PREDICT: &str = "streamRawPredict"; +pub const ANTHROPIC_VERSION: &str = "vertex-2023-10-16"; diff --git a/litellm-rust/crates/llms-types/src/providers/vertex_ai/mod.rs b/litellm-rust/crates/llms-types/src/providers/vertex_ai/mod.rs new file mode 100644 index 00000000000..27aa6cccc5e --- /dev/null +++ b/litellm-rust/crates/llms-types/src/providers/vertex_ai/mod.rs @@ -0,0 +1,3 @@ +mod constants; + +pub use constants::*; diff --git a/litellm-rust/crates/llms-types/src/recognized.rs b/litellm-rust/crates/llms-types/src/recognized.rs index 4fe5f66b31b..58e8dd7a2e7 100644 --- a/litellm-rust/crates/llms-types/src/recognized.rs +++ b/litellm-rust/crates/llms-types/src/recognized.rs @@ -18,11 +18,10 @@ impl Recognized { #[cfg(test)] mod tests { + use super::*; use rstest::rstest; use serde_json::json; - use super::*; - #[rstest] #[case::known(json!(7), Recognized::Known(7))] #[case::wrong_type(json!("7"), Recognized::Unrecognized(json!("7")))] diff --git a/litellm-rust/crates/llms-types/src/serde_compat.rs b/litellm-rust/crates/llms-types/src/serde_compat.rs index ffa86b7aec8..d32ca80f862 100644 --- a/litellm-rust/crates/llms-types/src/serde_compat.rs +++ b/litellm-rust/crates/llms-types/src/serde_compat.rs @@ -111,3 +111,94 @@ fn integral_float(value: f64) -> Option { && value < -(i64::MIN as f64)) .then_some(value as i64) } + +#[cfg(test)] +mod tests { + use crate::serde_compat::{FiniteF64, LaxI64}; + + use rstest::rstest; + use serde::{Deserialize, Serialize}; + use serde_json::{Value, json}; + use serde_with::serde_as; + + #[serde_as] + #[derive(Debug, Deserialize, Serialize, PartialEq)] + struct Numbers { + #[serde_as(deserialize_as = "Option>")] + integers: Option>, + #[serde_as(deserialize_as = "Option")] + float: Option, + } + + #[rstest] + fn adapters_compose_and_serialize_as_numbers() { + let numbers: Numbers = serde_json::from_value(json!({ + "integers": ["9007199254740993.0", "1_000", " +2.000 ", 3.0, true], + "float": " 1.5 " + })) + .unwrap(); + assert_eq!( + serde_json::to_value(numbers).unwrap(), + json!({"integers": [9_007_199_254_740_993_i64, 1000, 2, 3, 1], "float": 1.5}) + ); + } + + #[rstest] + #[case::missing(json!({}))] + #[case::null(json!({"integers": null, "float": null}))] + fn optional_adapters_accept_missing_and_null_fields(#[case] input: Value) { + assert_eq!( + serde_json::from_value::(input).unwrap(), + Numbers { + integers: None, + float: None + } + ); + } + + #[rstest] + #[case::minimum(json!(i64::MIN), i64::MIN)] + #[case::maximum(json!(i64::MAX), i64::MAX)] + #[case::maximum_string(json!(i64::MAX.to_string()), i64::MAX)] + fn integers_preserve_bounds(#[case] input: Value, #[case] expected: i64) { + let numbers: Numbers = serde_json::from_value(json!({"integers": [input]})).unwrap(); + assert_eq!(numbers.integers, Some(vec![expected])); + } + + #[rstest] + #[case::unsigned_maximum(json!(u64::MAX))] + #[case::above_maximum(json!(9_223_372_036_854_775_808_u64))] + #[case::float_above_maximum(json!(9_223_372_036_854_775_808.0))] + #[case::below_minimum(json!("-9223372036854775809"))] + #[case::precise_fraction(json!("1.0000000000000001"))] + #[case::exponent(json!("1e3"))] + #[case::missing_fraction(json!("2."))] + #[case::missing_integer(json!(".0"))] + #[case::leading_separator(json!("_2"))] + #[case::repeated_separator(json!("2__0"))] + #[case::fraction(json!(2.5))] + #[case::null(json!(null))] + #[case::object(json!({}))] + fn integers_reject_invalid_values(#[case] input: Value) { + assert!(serde_json::from_value::(json!({"integers": [input]})).is_err()); + } + + #[rstest] + #[case::nan(json!("NaN"))] + #[case::positive_infinity(json!("inf"))] + #[case::negative_infinity(json!("-inf"))] + #[case::overflow(json!("1e999"))] + #[case::array(json!([]))] + fn floats_reject_nonfinite_and_invalid_values(#[case] input: Value) { + assert!(serde_json::from_value::(json!({"float": input})).is_err()); + } + + #[rstest] + #[case::integer(json!(2), 2.0)] + #[case::float(json!(2.5), 2.5)] + #[case::boolean(json!(true), 1.0)] + fn floats_accept_finite_numbers(#[case] input: Value, #[case] expected: f64) { + let numbers: Numbers = serde_json::from_value(json!({"float": input})).unwrap(); + assert_eq!(numbers.float, Some(expected)); + } +} diff --git a/litellm-rust/crates/llms-types/src/test_support.rs b/litellm-rust/crates/llms-types/src/test_support.rs new file mode 100644 index 00000000000..ca28afb38ec --- /dev/null +++ b/litellm-rust/crates/llms-types/src/test_support.rs @@ -0,0 +1,11 @@ +use serde::{Serialize, de::DeserializeOwned}; +use serde_json::Value; + +pub(crate) fn round_trip(wire: Value) -> T +where + T: DeserializeOwned + Serialize, +{ + let parsed: T = serde_json::from_value(wire.clone()).unwrap(); + assert_eq!(serde_json::to_value(&parsed).unwrap(), wire); + parsed +} diff --git a/litellm-rust/crates/llms-types/tests/chat.rs b/litellm-rust/crates/llms-types/tests/chat.rs deleted file mode 100644 index 76aa791b510..00000000000 --- a/litellm-rust/crates/llms-types/tests/chat.rs +++ /dev/null @@ -1,163 +0,0 @@ -use litellm_llms_types::formats::chat_completions::{ - ChatContentPart, ChatLogprobs, ChatMediaUrl, PromptCacheMode, -}; -use litellm_llms_types::formats::messages::ContentSource; -use rstest::rstest; -use serde_json::{Value, json}; - -#[rstest] -#[case::parts(json!([ - {"type":"text","text":"hello","cache_control":{"type":"ephemeral"},"prompt_cache_breakpoint":{"mode":"explicit"}}, - {"type":"image_url","image_url":{"url":"https://example.test/image","detail":"high","format":"image/png"}}, - {"type":"video_url","video_url":"https://example.test/video"}, - {"type":"input_audio","input_audio":{"data":"AA==","format":"wav"}}, - {"type":"file","file":{"file_id":"file_1","file_data":"JVBE","filename":"a.mp4","format":"video/mp4","detail":"low","video_metadata":{"fps":1,"start_offset":"1s","end_offset":"2s"}}}, - {"type":"document","source":{"type":"text","media_type":"text/plain","data":"doc"},"title":"T","context":"C","citations":{"enabled":true}}, - {"type":"refusal","refusal":"refused"} -]))] -fn content_parts_round_trip(#[case] wire: Value) { - let parts: Vec = serde_json::from_value(wire.clone()).unwrap(); - let [ - ChatContentPart::Text { - text, - cache_control, - prompt_cache_breakpoint: Some(breakpoint), - extra: text_extra, - }, - ChatContentPart::ImageUrl { - image_url: ChatMediaUrl::Parameters(image), - prompt_cache_breakpoint: None, - .. - }, - ChatContentPart::VideoUrl { - video_url: ChatMediaUrl::Url(video), - .. - }, - ChatContentPart::InputAudio { input_audio, .. }, - ChatContentPart::File { file, .. }, - ChatContentPart::Document { - source, - title: Some(title), - context: Some(context), - citations: Some(citations), - .. - }, - ChatContentPart::Refusal { refusal, .. }, - ] = parts.as_slice() - else { - panic!("expected typed content parts"); - }; - assert_eq!(text, "hello"); - assert_eq!( - cache_control.as_ref().unwrap().cache_type.as_deref(), - Some("ephemeral") - ); - assert_eq!(breakpoint.mode, PromptCacheMode::Explicit); - assert!(breakpoint.extra.is_empty()); - assert!(text_extra.is_empty()); - assert_eq!(image.url, "https://example.test/image"); - assert_eq!(image.detail.as_deref(), Some("high")); - assert_eq!(image.format.as_deref(), Some("image/png")); - assert!(image.extra.is_empty()); - assert_eq!(video, "https://example.test/video"); - assert_eq!(input_audio.data, "AA=="); - assert_eq!(input_audio.format, "wav"); - assert_eq!(file.file_id.as_deref(), Some("file_1")); - assert_eq!(file.file_data.as_deref(), Some("JVBE")); - assert_eq!(file.filename.as_deref(), Some("a.mp4")); - assert_eq!(file.format.as_deref(), Some("video/mp4")); - assert_eq!(file.detail.as_deref(), Some("low")); - assert!(file.extra.is_empty()); - let metadata = file.video_metadata.as_ref().unwrap(); - assert_eq!(metadata.fps, Some(1.into())); - assert_eq!(metadata.start_offset.as_deref(), Some("1s")); - assert_eq!(metadata.end_offset.as_deref(), Some("2s")); - assert!(metadata.extra.is_empty()); - let ContentSource::Text { - data, media_type, .. - } = source.as_ref() - else { - panic!("expected document text source"); - }; - assert_eq!(data, "doc"); - assert_eq!(media_type, "text/plain"); - assert_eq!(title, "T"); - assert_eq!(context, "C"); - assert_eq!(citations.enabled, Some(true)); - assert_eq!(refusal, "refused"); - assert_eq!(serde_json::to_value(parts).unwrap(), wire); -} - -#[rstest] -fn logprobs_round_trip_with_tokens_and_alternatives() { - let wire = json!({ - "content":[{"token":"hi","logprob":-1,"bytes":[104,105],"top_logprobs":[{"token":"hey","logprob":-2.5,"bytes":[104]}]}], - "refusal":[{"token":"refused","logprob":-3}] - }); - let logprobs: ChatLogprobs = serde_json::from_value(wire.clone()).unwrap(); - let token = &logprobs.content.as_ref().unwrap()[0]; - assert_eq!(token.token, "hi"); - assert_eq!(token.logprob, (-1).into()); - assert_eq!(token.bytes.as_deref(), Some([104, 105].as_slice())); - let alternative = &token.top_logprobs.as_ref().unwrap()[0]; - assert_eq!(alternative.token, "hey"); - assert_eq!(alternative.bytes.as_deref(), Some([104].as_slice())); - assert_eq!(logprobs.refusal.as_ref().unwrap()[0].token, "refused"); - assert!(logprobs.refusal.as_ref().unwrap()[0].top_logprobs.is_none()); - assert_eq!(serde_json::to_value(logprobs).unwrap(), wire); -} - -#[rstest] -#[case::text(json!({"type":"text","text":false}))] -#[case::missing_image(json!({"type":"image_url"}))] -#[case::audio_shape(json!({"type":"input_audio","input_audio":{"data":7,"format":"wav"}}))] -#[case::audio_missing_format(json!({"type":"input_audio","input_audio":{"data":"AA=="}}))] -#[case::image_missing_url(json!({"type":"image_url","image_url":{"detail":"high"}}))] -#[case::file_metadata(json!({"type":"file","file":{"video_metadata":{"fps":"fast"}}}))] -#[case::document_source(json!({"type":"document","source":{"type":"url","url":false}}))] -#[case::refusal_missing_text(json!({"type":"refusal"}))] -#[case::file_missing_file(json!({"type":"file"}))] -#[case::document_missing_source(json!({"type":"document"}))] -#[case::video_missing_url(json!({"type":"video_url"}))] -#[case::cache_control_shape(json!({"type":"text","text":"t","cache_control":"ephemeral"}))] -#[case::breakpoint_missing_mode(json!({"type":"text","text":"t","prompt_cache_breakpoint":{}}))] -#[case::breakpoint_unknown_mode(json!({"type":"text","text":"t","prompt_cache_breakpoint":{"mode":"auto"}}))] -#[case::unknown_tag(json!({"type":"future"}))] -fn content_parts_reject_malformed_typed_fields(#[case] wire: Value) { - assert!(serde_json::from_value::(wire).is_err()); -} - -#[rstest] -fn partial_file_preserves_extensions_and_omits_null_optionals() { - let wire = json!({"type":"file","file":{"file_id":null,"filename":"a.pdf","extension":[1,null]},"future":true}); - let part: ChatContentPart = serde_json::from_value(wire).unwrap(); - let ChatContentPart::File { - file, - prompt_cache_breakpoint: None, - extra, - } = &part - else { - panic!("expected file") - }; - assert!(file.file_id.is_none()); - assert!(file.file_data.is_none()); - assert_eq!(file.filename.as_deref(), Some("a.pdf")); - assert_eq!(extra["future"], json!(true)); - assert_eq!( - serde_json::to_value(part).unwrap(), - json!({"type":"file","file":{"filename":"a.pdf","extension":[1,null]},"future":true}) - ); -} - -#[rstest] -#[case::missing_token(json!({"logprob":-1}))] -#[case::missing_logprob(json!({"token":"hi"}))] -#[case::wrong_bytes(json!({"token":"hi","logprob":-1,"bytes":[256]}))] -fn token_logprobs_reject_malformed_fields(#[case] wire: Value) { - assert!( - serde_json::from_value::( - wire - ) - .is_err() - ); -} diff --git a/litellm-rust/crates/llms-types/tests/json_schema.rs b/litellm-rust/crates/llms-types/tests/json_schema.rs deleted file mode 100644 index a626304910c..00000000000 --- a/litellm-rust/crates/llms-types/tests/json_schema.rs +++ /dev/null @@ -1,148 +0,0 @@ -use litellm_llms_types::json_schema::{ - JsonSchema, JsonSchemaItems, JsonSchemaObject, JsonSchemaType, -}; -use rstest::rstest; -use serde::{Serialize, de::DeserializeOwned}; -use serde_json::{Value, json}; - -fn round_trip(wire: Value) -> T -where - T: DeserializeOwned + Serialize, -{ - let parsed: T = serde_json::from_value(wire.clone()).unwrap(); - assert_eq!(serde_json::to_value(&parsed).unwrap(), wire); - parsed -} - -#[rstest] -#[case::boolean(json!(false))] -#[case::nested(json!({ - "type":["object","null"], - "properties":{"nested":{"type":"array","items":{"$ref":"#/$defs/item"}},"free":true}, - "additionalProperties":{"type":"string"}, - "$defs":{"item":{"anyOf":[{"type":"string"},false]}}, - "required":["nested"], - "enum":[{"custom":[1,null]}], - "future":null -}))] -fn recursive_schema_round_trips(#[case] wire: Value) { - let schema = round_trip::(wire); - match schema { - JsonSchema::Boolean(allowed) => assert!(!allowed), - JsonSchema::Object(schema) => { - assert_eq!( - schema.schema_type, - Some(JsonSchemaType::Names(vec!["object".into(), "null".into()])) - ); - let properties = schema.properties.as_ref().unwrap(); - let JsonSchema::Object(nested) = &properties["nested"] else { - panic!("expected nested schema") - }; - let Some(JsonSchemaItems::Schema(items)) = &nested.items else { - panic!("expected single items schema") - }; - let JsonSchema::Object(items) = items.as_ref() else { - panic!("expected items schema") - }; - assert_eq!(items.reference.as_deref(), Some("#/$defs/item")); - assert_eq!(properties["free"], JsonSchema::Boolean(true)); - let JsonSchema::Object(definition) = &schema.defs.as_ref().unwrap()["item"] else { - panic!("expected schema definition") - }; - assert_eq!( - definition.any_of.as_ref().unwrap()[1], - JsonSchema::Boolean(false) - ); - assert_eq!(schema.extra["enum"], json!([{"custom":[1,null]}])); - } - } -} - -#[rstest] -fn schema_maps_keep_wire_key_order() { - let wire = r#"{"type":"object","properties":{"zeta":{"type":"string"},"alpha":{"type":"integer"}},"$defs":{"y":true,"b":false}}"#; - let schema: JsonSchemaObject = serde_json::from_str(wire).unwrap(); - let names: Vec<&str> = schema - .properties - .as_ref() - .unwrap() - .keys() - .map(String::as_str) - .collect(); - assert_eq!(names, ["zeta", "alpha"]); - assert_eq!(serde_json::to_string(&schema).unwrap(), wire); -} - -#[rstest] -fn schema_object_round_trips_supported_keywords() { - let schema = round_trip::(json!({ - "type":"object", - "properties":{"name":{"type":"string"}}, - "required":["name"], - "additionalProperties":false, - "$ref":"#/$defs/value", - "strict":true, - "extension":{"nested":[1,null]} - })); - assert_eq!( - schema.schema_type, - Some(JsonSchemaType::Name("object".into())) - ); - assert_eq!(schema.required.as_deref(), Some(["name".into()].as_slice())); - assert_eq!( - schema.additional_properties.as_deref(), - Some(&JsonSchema::Boolean(false)) - ); - assert_eq!(schema.reference.as_deref(), Some("#/$defs/value")); - assert_eq!(schema.strict, Some(true)); -} - -#[rstest] -#[case::scalar(json!(7))] -#[case::properties_shape(json!({"properties":[]}))] -#[case::nested_schema(json!({"properties":{"name":7}}))] -#[case::schema_type(json!({"type":["string",7]}))] -#[case::items_shape(json!({"items":7}))] -#[case::tuple_member(json!({"items":[{"type":"string"},7]}))] -#[case::prefix_items_shape(json!({"prefixItems":{"type":"string"}}))] -#[case::composite_shape(json!({"anyOf":["string"]}))] -fn schemas_reject_malformed_known_keywords(#[case] wire: Value) { - assert!(serde_json::from_value::(wire).is_err()); -} - -#[rstest] -#[case::draft_07_tuple(json!({"type":"array","items":[{"type":"string"},true]}))] -#[case::draft_2020_12_tuple(json!({"type":"array","prefixItems":[{"type":"string"},true],"items":false}))] -fn tuple_schemas_expose_positional_members(#[case] wire: Value) { - let schema = round_trip::(wire); - let members = match (&schema.items, &schema.prefix_items) { - (Some(JsonSchemaItems::Tuple(members)), None) => members, - (Some(JsonSchemaItems::Schema(rest)), Some(members)) => { - assert_eq!(rest.as_ref(), &JsonSchema::Boolean(false)); - members - } - _ => panic!("expected positional members"), - }; - let [JsonSchema::Object(first), JsonSchema::Boolean(true)] = members.as_slice() else { - panic!("expected string schema then true"); - }; - assert_eq!( - first.schema_type, - Some(JsonSchemaType::Name("string".into())) - ); - assert!(schema.extra.is_empty()); -} - -#[rstest] -fn empty_schema_omits_null_optionals_and_preserves_arbitrary_keywords() { - let schema: JsonSchemaObject = serde_json::from_value( - json!({"type":null,"properties":null,"const":{"arbitrary":[1,null]},"future":null}), - ) - .unwrap(); - assert!(schema.schema_type.is_none()); - assert!(schema.properties.is_none()); - assert_eq!( - serde_json::to_value(schema).unwrap(), - json!({"const":{"arbitrary":[1,null]},"future":null}) - ); -} diff --git a/litellm-rust/crates/llms-types/tests/messages_request.rs b/litellm-rust/crates/llms-types/tests/messages_request.rs deleted file mode 100644 index 4825932bc02..00000000000 --- a/litellm-rust/crates/llms-types/tests/messages_request.rs +++ /dev/null @@ -1,79 +0,0 @@ -use litellm_llms_types::formats::messages::{ContentBlock, ContentBlockType}; -use rstest::rstest; -use serde_json::{Value, json}; - -#[rstest] -#[case::null(json!(null))] -#[case::number(json!(1))] -#[case::boolean(json!(true))] -#[case::array(json!(["tool_use"]))] -#[case::object(json!({"type": "tool_use"}))] -fn content_block_type_rejects_non_string_json(#[case] value: Value) { - assert!(serde_json::from_value::(value).is_err()); -} - -#[cfg(feature = "schema")] -#[rstest] -fn content_block_type_schema_remains_a_string() { - let schema = schemars::schema_for!(ContentBlockType).to_value(); - assert_eq!(schema.get("type"), Some(&json!("string"))); - assert_eq!( - schema.get("title"), - Some(&json!(stringify!(ContentBlockType))) - ); -} - -#[rstest] -#[case::text("text", ContentBlockType::Text)] -#[case::thinking("thinking", ContentBlockType::Thinking)] -#[case::redacted_thinking("redacted_thinking", ContentBlockType::RedactedThinking)] -#[case::tool_use("tool_use", ContentBlockType::ToolUse)] -#[case::server_tool_use("server_tool_use", ContentBlockType::ServerToolUse)] -#[case::tool_result("tool_result", ContentBlockType::ToolResult)] -#[case::compaction("compaction", ContentBlockType::Compaction)] -#[case::advisor_result("advisor_tool_result", ContentBlockType::AdvisorToolResult)] -#[case::web_search_result("web_search_tool_result", ContentBlockType::WebSearchToolResult)] -#[case::image("image", ContentBlockType::Other("image".into()))] -#[case::document("document", ContentBlockType::Other("document".into()))] -#[case::tool_addition("tool_addition", ContentBlockType::Other("tool_addition".into()))] -#[case::tool_removal("tool_removal", ContentBlockType::Other("tool_removal".into()))] -#[case::advisor("advisor_result", ContentBlockType::Other("advisor_result".into()))] -#[case::web_search_error( - "web_search_tool_result_error", - ContentBlockType::Other("web_search_tool_result_error".into()) -)] -#[case::future_block("future_block", ContentBlockType::Other("future_block".into()))] -#[case::case_sensitive("Tool_Use", ContentBlockType::Other("Tool_Use".into()))] -#[case::empty("", ContentBlockType::Other(String::new()))] -fn block_type_is_typed_and_round_trips_with_extra_fields( - #[case] wire: &str, - #[case] expected: ContentBlockType, -) { - let input = json!({"type": wire, "future_field": {"nested": [1, null]}}); - let block: ContentBlock = serde_json::from_value(input.clone()).unwrap(); - assert_eq!(block.block_type.as_ref(), Some(&expected)); - assert_eq!(serde_json::to_value(block).unwrap(), input); -} - -#[rstest] -#[case::number(json!(1))] -#[case::boolean(json!(true))] -#[case::array(json!(["text"]))] -#[case::enum_object(json!({"text": null}))] -fn block_type_rejects_non_string_values(#[case] value: Value) { - assert!(serde_json::from_value::(json!({"type": value})).is_err()); -} - -#[rstest] -#[case::same_type(json!({"type": "tool_use"}), ContentBlockType::ToolUse, true)] -#[case::other_type(json!({"type": "tool_result"}), ContentBlockType::ToolUse, false)] -#[case::unknown_type(json!({"type": "future_tool"}), ContentBlockType::ToolUse, false)] -#[case::no_type(json!({"text": "x"}), ContentBlockType::Text, false)] -fn is_type_matches_the_exact_block_type( - #[case] block: Value, - #[case] block_type: ContentBlockType, - #[case] expected: bool, -) { - let block: ContentBlock = serde_json::from_value(block).unwrap(); - assert_eq!(block.is_type(block_type), expected); -} diff --git a/litellm-rust/crates/llms-types/tests/messages_streaming.rs b/litellm-rust/crates/llms-types/tests/messages_streaming.rs deleted file mode 100644 index 5aebb1c052c..00000000000 --- a/litellm-rust/crates/llms-types/tests/messages_streaming.rs +++ /dev/null @@ -1,37 +0,0 @@ -use litellm_llms_types::formats::messages::streaming::MessagesStreamEvent; -use rstest::rstest; -use serde_json::{Value, json}; - -#[rstest] -#[case::start(json!({ - "type": "message_start", - "message": { - "id": "msg", "type": "message", "role": "assistant", "model": "test-model", - "content": [], "stop_reason": null, "stop_sequence": null, - "usage": {"input_tokens": 3, "future_usage": {"count": 9}}, - "future_message": [1, 2] - } -}))] -#[case::block(json!({ - "type": "content_block_start", "index": 0, - "content_block": {"type": "future", "payload": {"keep": true}} -}))] -#[case::delta(json!({ - "type": "message_delta", "delta": {"stop_reason": "end_turn", "future_delta": 42}, - "usage": {"output_tokens": 7, "future_usage": true} -}))] -#[case::error(json!({ - "type": "error", "error": {"type": "future_error", "message": "failed", "future": 42} -}))] -fn stream_events_preserve_extensible_fields(#[case] wire: Value) { - let event: MessagesStreamEvent = serde_json::from_value(wire.clone()).unwrap(); - assert_eq!(serde_json::to_value(event).unwrap(), wire); -} - -#[rstest] -#[case::invalid_index(json!({"type": "content_block_stop", "index": "zero"}))] -#[case::missing_delta(json!({"type": "content_block_delta", "index": 0}))] -#[case::unknown_event(json!({"type": "future_event"}))] -fn malformed_or_unrecognized_events_remain_rejected(#[case] wire: Value) { - assert!(serde_json::from_value::(wire).is_err()); -} diff --git a/litellm-rust/crates/llms-types/tests/messages_types.rs b/litellm-rust/crates/llms-types/tests/messages_types.rs deleted file mode 100644 index 51785a54c98..00000000000 --- a/litellm-rust/crates/llms-types/tests/messages_types.rs +++ /dev/null @@ -1,1512 +0,0 @@ -use indexmap::IndexMap; -use litellm_llms_types::formats::messages::{ - AdvisorTool, AdvisorToolName, AdvisorToolResultContent, AllowedCaller, BashCodeExecutionOutput, - BashCodeExecutionToolResultContent, BashToolName, BlockContent, BrowserStateChange, - BrowserToolsetConfigs, BuiltinMessagesTool, CacheControl, CacheMissReason, Citation, - CitationsConfig, ClientTool, CodeExecutionOutput, CodeExecutionToolName, - CodeExecutionToolResultContent, ComputerTool, ComputerTool20251124, ComputerToolName, - ComputerToolsetConfigs, ContainerReference, ContentSource, ContextManagementResponse, - ContextTrigger, CustomTool, CustomToolType, FallbackTrigger, McpListedTool, McpServer, - McpToolResultContent, McpToolResultText, McpToolset, MemoryToolName, MessageRole, MessageType, - MessagesCompaction, MessagesContainer, MessagesContentPart, MessagesDiagnostics, - MessagesDiagnosticsParam, MessagesMetadata, MessagesToolParam, MessagesUsage, OutputFormat, - ResponseInclusion, Safeguard, ServerTool, SkillType, StopDetails, StopDetailsType, StopReason, - StrReplaceBasedEditToolName, StrReplaceEditorName, TextEditorCodeExecutionToolResultContent, - TextEditorFileType, TextEditorTool20250728, ToolCaller, ToolChange, ToolChangeTarget, - ToolChoice, ToolChoiceType, ToolResultUrlSource, ToolSearchBm25ToolName, ToolSearchReference, - ToolSearchRegexToolName, ToolSearchToolResultContent, Toolset, ToolsetToolConfig, - UrlSourceToolReference, UsageIterationType, UserInputUrlSource, UserLocationType, - WebFetchDocument, WebFetchTool, WebFetchTool20260309, WebFetchTool20260318, WebFetchToolName, - WebFetchToolResultContent, WebFetchUrlSources, WebSearchTool, WebSearchTool20260318, - WebSearchToolName, WebSearchToolResultContent, WebSearchUserLocation, -}; -use litellm_llms_types::json_schema::{JsonSchema, JsonSchemaObject, JsonSchemaType}; -use litellm_llms_types::recognized::Recognized; -use rstest::rstest; -use serde::{Serialize, de::DeserializeOwned}; -use serde_json::{Map, Value, json}; - -fn round_trip(wire: Value) -> T -where - T: DeserializeOwned + Serialize, -{ - let parsed: T = serde_json::from_value(wire.clone()).unwrap(); - assert_eq!(serde_json::to_value(&parsed).unwrap(), wire); - parsed -} - -#[rstest] -fn metadata_contracts_round_trip() { - let metadata = round_trip::(json!({"user_id":"user_1","future":true})); - assert_eq!(metadata.user_id.as_deref(), Some("user_1")); - assert_eq!(metadata.extra.get("future"), Some(&json!(true))); - let output_format = round_trip::(json!({ - "type":"json_schema", - "schema":{"type":"object","properties":{"name":{"type":"string"}}}, - "strict":true - })); - assert_eq!(output_format.strict, Some(true)); - assert!(output_format.extra.is_empty()); - let JsonSchema::Object(schema) = &output_format.schema else { - panic!("expected output object schema"); - }; - assert!(schema.properties.as_ref().unwrap().contains_key("name")); - let compaction = - round_trip::(json!({"type":"summarize","instructions":"briefly"})); - assert_eq!(compaction.instructions.as_deref(), Some("briefly")); - assert!(compaction.extra.is_empty()); - let container = round_trip::(json!({ - "id":"container_1", - "expires_at":"2026-01-01T00:00:00Z", - "skills":[{"type":"custom","skill_id":"skill_1","version":"1"}] - })); - assert_eq!(container.id.as_deref(), Some("container_1")); - assert!(container.extra.is_empty()); - let [skill] = container.skills.as_ref().unwrap().as_slice() else { - panic!("expected container skill"); - }; - assert_eq!(skill.skill_type, SkillType::Custom); - assert_eq!(skill.skill_id, "skill_1"); - assert_eq!(skill.version.as_deref(), Some("1")); - assert!(skill.extra.is_empty()); - let reference = round_trip::(json!({"id":"container_1"})); - let ContainerReference::Parameters(parameters) = reference else { - panic!("expected container parameters"); - }; - assert_eq!(parameters.id.as_deref(), container.id.as_deref()); - assert!(parameters.skills.is_none()); - round_trip::( - json!({"id":"container_1","skills":[{"type":"anthropic","skill_id":"pptx"}]}), - ); - let server = round_trip::(json!({ - "type":"url", - "url":"https://example.test/mcp", - "name":"search", - "authorization_token":"token", - "tool_configuration":{"allowed_tools":["search"],"enabled":true} - })); - assert_eq!(server.url, "https://example.test/mcp"); - assert_eq!(server.name, "search"); - assert_eq!(server.authorization_token.as_deref(), Some("token")); - assert!(server.extra.is_empty()); - let configuration = server.tool_configuration.as_ref().unwrap(); - assert_eq!(configuration.enabled, Some(true)); - assert_eq!( - configuration.allowed_tools.as_deref(), - Some([String::from("search")].as_slice()) - ); - assert!(configuration.extra.is_empty()); - let context = round_trip::(json!({ - "applied_edits":[ - {"type":"clear_tool_uses_20250919","cleared_input_tokens":7,"cleared_tool_uses":2}, - {"type":"clear_thinking_20251015","cleared_input_tokens":3,"cleared_thinking_turns":1} - ] - })); - let [tool_uses, thinking] = context.applied_edits.as_ref().unwrap().as_slice() else { - panic!("expected two applied edits"); - }; - assert_eq!( - tool_uses.edit_type.as_deref(), - Some("clear_tool_uses_20250919") - ); - assert_eq!( - (tool_uses.cleared_input_tokens, tool_uses.cleared_tool_uses), - (Some(7), Some(2)) - ); - assert!(tool_uses.cleared_thinking_turns.is_none()); - assert_eq!( - ( - thinking.cleared_input_tokens, - thinking.cleared_thinking_turns - ), - (Some(3), Some(1)) - ); - assert!(tool_uses.extra.is_empty() && thinking.extra.is_empty()); - let safeguard = round_trip::( - json!({"type":"classifier","classifier_context":{"source":"test"}}), - ); - assert_eq!(safeguard.safeguard_type, "classifier"); - assert_eq!( - safeguard.classifier_context.as_ref().unwrap().get("source"), - Some(&json!("test")) - ); - assert!(safeguard.extra.is_empty()); -} - -#[rstest] -fn stop_details_expose_refusal_and_keep_unknown_types() { - let refusal = round_trip::( - json!({"type":"refusal","category":"cyber","explanation":"blocked"}), - ); - assert_eq!( - refusal.detail_type, - Recognized::Known(StopDetailsType::Refusal) - ); - assert_eq!(refusal.category.as_deref(), Some("cyber")); - assert_eq!(refusal.explanation.as_deref(), Some("blocked")); - assert!(refusal.extra.is_empty()); - let safeguard = - round_trip::(json!({"type":"safeguard","safeguard_types":["classifier"]})); - assert_eq!( - safeguard.detail_type, - Recognized::Unrecognized(json!("safeguard")) - ); - assert_eq!(safeguard.extra["safeguard_types"], json!(["classifier"])); -} - -#[rstest] -fn container_rejects_skill_without_id() { - assert!( - serde_json::from_value::( - json!({"skills":[{"type":"custom","version":"1"}]}) - ) - .is_err() - ); -} - -#[rstest] -#[case::without_url(json!({"type":"url","name":"search"}))] -#[case::without_name(json!({"type":"url","url":"https://example.test/mcp"}))] -fn mcp_server_rejects_missing_required_fields(#[case] wire: Value) { - assert!(serde_json::from_value::(wire).is_err()); -} - -#[rstest] -fn output_format_rejects_missing_schema() { - assert!(serde_json::from_value::(json!({"type":"json_schema"})).is_err()); -} - -#[rstest] -#[case::user(json!("user"))] -#[case::assistant(json!("assistant"))] -#[case::system(json!("system"))] -#[case::future(json!("future_role"))] -fn message_roles_round_trip(#[case] wire: Value) { - round_trip::(wire); -} - -#[rstest] -#[case::message(json!("message"))] -#[case::future(json!("future_type"))] -fn message_types_round_trip(#[case] wire: Value) { - round_trip::(wire); -} - -#[rstest] -#[case::end_turn(json!("end_turn"))] -#[case::refusal(json!("refusal"))] -#[case::compaction(json!("compaction"))] -#[case::future(json!("future_reason"))] -fn stop_reasons_round_trip(#[case] wire: Value) { - round_trip::(wire); -} - -#[rstest] -fn tool_choice_round_trips() { - let choice = round_trip::( - json!({"type":"tool","name":"search","disable_parallel_tool_use":true}), - ); - assert_eq!(choice.choice_type, ToolChoiceType::Tool); - assert_eq!(choice.name.as_deref(), Some("search")); - assert_eq!(choice.disable_parallel_tool_use, Some(true)); - assert!(choice.extra.is_empty()); -} - -#[rstest] -fn usage_contracts_round_trip() { - let usage = round_trip::(json!({ - "input_tokens":10, - "output_tokens":4, - "server_tool_use":{"web_search_requests":2,"web_fetch_requests":1}, - "cache_creation":{"ephemeral_1h_input_tokens":3,"ephemeral_5m_input_tokens":1}, - "output_tokens_details":{"thinking_tokens":2}, - "iterations":[ - {"type":"compaction","input_tokens":7,"output_tokens":1}, - {"type":"message","input_tokens":3,"output_tokens":3} - ], - "service_tier":"priority", - "inference_geo":"global", - "speed":"fast" - })); - assert_eq!(usage.inference_geo.as_deref(), Some("global")); - assert_eq!(usage.input_tokens, Some(10)); - assert_eq!(usage.output_tokens, Some(4)); - assert!(usage.extra.is_empty()); - let server = usage.server_tool_use.as_ref().unwrap(); - assert_eq!(server.web_search_requests, Some(2)); - assert_eq!(server.web_fetch_requests, Some(1)); - let cache = usage.cache_creation.as_ref().unwrap(); - assert_eq!(cache.ephemeral_1h_input_tokens, Some(3)); - assert_eq!(cache.ephemeral_5m_input_tokens, Some(1)); - assert_eq!( - usage - .output_tokens_details - .as_ref() - .unwrap() - .thinking_tokens, - Some(2) - ); - let [compaction, message] = usage.iterations.as_ref().unwrap().as_slice() else { - panic!("expected usage iterations"); - }; - assert_eq!( - compaction.iteration_type, - Recognized::Known(UsageIterationType::Compaction) - ); - assert_eq!( - message.iteration_type, - Recognized::Known(UsageIterationType::Message) - ); - assert_eq!( - compaction.input_tokens.unwrap() + message.input_tokens.unwrap(), - usage.input_tokens.unwrap() - ); - assert_eq!( - compaction.output_tokens.unwrap() + message.output_tokens.unwrap(), - usage.output_tokens.unwrap() - ); -} - -#[rstest] -fn usage_iterations_expose_advisor_and_fallback_models() { - let usage = round_trip::(json!({ - "iterations":[ - {"type":"advisor_message","model":"claude-opus-5-5","input_tokens":5,"output_tokens":2, - "cache_creation_input_tokens":4,"cache_read_input_tokens":1, - "cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":4}}, - {"type":"fallback_message","model":"claude-sonnet-5-5","input_tokens":6,"output_tokens":3, - "cache_creation_input_tokens":0,"cache_read_input_tokens":2}, - {"type":"future_iteration","input_tokens":1} - ] - })); - let [advisor, fallback, future] = usage.iterations.as_deref().unwrap() else { - panic!("expected three iterations"); - }; - assert_eq!( - advisor.iteration_type, - Recognized::Known(UsageIterationType::AdvisorMessage) - ); - assert_eq!(advisor.model.as_deref(), Some("claude-opus-5-5")); - assert_eq!( - ( - advisor.cache_creation_input_tokens, - advisor.cache_read_input_tokens - ), - (Some(4), Some(1)) - ); - assert_eq!( - advisor - .cache_creation - .as_ref() - .unwrap() - .ephemeral_5m_input_tokens, - advisor.cache_creation_input_tokens - ); - assert!(advisor.extra.is_empty()); - assert_eq!( - fallback.iteration_type, - Recognized::Known(UsageIterationType::FallbackMessage) - ); - assert_eq!(fallback.model.as_deref(), Some("claude-sonnet-5-5")); - assert_eq!(fallback.cache_read_input_tokens, Some(2)); - assert!(fallback.extra.is_empty()); - assert_eq!( - future.iteration_type, - Recognized::Unrecognized(json!("future_iteration")) - ); - assert_eq!(future.input_tokens, Some(1)); -} - -#[rstest] -#[case::base64(json!({"type":"base64","media_type":"image/png","data":"AA=="}))] -#[case::url(json!({"type":"url","url":"https://example.test/image"}))] -#[case::file(json!({"type":"file","file_id":"file_1"}))] -#[case::text(json!({"type":"text","media_type":"text/plain","data":"document"}))] -#[case::content(json!({"type":"content","content":"nested"}))] -fn content_sources_round_trip(#[case] wire: Value) { - round_trip::(wire); -} - -fn client_tool(name: N) -> ClientTool { - ClientTool { - name, - allowed_callers: None, - cache_control: None, - defer_loading: None, - input_examples: None, - strict: None, - extra: Map::new(), - } -} - -fn server_tool(name: N) -> ServerTool { - ServerTool { - name, - allowed_callers: None, - cache_control: None, - defer_loading: None, - strict: None, - extra: Map::new(), - } -} - -fn computer_tool(display_width_px: u64, display_height_px: u64) -> ComputerTool { - ComputerTool { - name: ComputerToolName::Computer, - display_width_px, - display_height_px, - display_number: None, - allowed_callers: None, - cache_control: None, - defer_loading: None, - input_examples: None, - strict: None, - extra: Map::new(), - } -} - -fn web_search_tool() -> WebSearchTool { - WebSearchTool { - name: WebSearchToolName::WebSearch, - allowed_callers: None, - allowed_domains: None, - blocked_domains: None, - cache_control: None, - defer_loading: None, - max_uses: None, - strict: None, - user_location: None, - extra: Map::new(), - } -} - -fn web_fetch_tool() -> WebFetchTool { - WebFetchTool { - name: WebFetchToolName::WebFetch, - allowed_callers: None, - allowed_domains: None, - blocked_domains: None, - cache_control: None, - citations: None, - defer_loading: None, - max_content_tokens: None, - max_uses: None, - strict: None, - url_sources: None, - extra: Map::new(), - } -} - -fn web_fetch_url_sources() -> WebFetchUrlSources { - WebFetchUrlSources { - client_tool_results: None, - server_tool_results: None, - user_input: None, - extra: Map::new(), - } -} - -fn tool_reference(name: &str) -> UrlSourceToolReference { - UrlSourceToolReference::ToolReference { - name: String::from(name), - extra: Map::new(), - } -} - -fn toolset_config(enabled: Option, defer_loading: Option) -> ToolsetToolConfig { - ToolsetToolConfig { - defer_loading, - enabled, - extra: Map::new(), - } -} - -fn cache_control(cache_type: &str, ttl: &str) -> CacheControl { - CacheControl { - cache_type: Some(String::from(cache_type)), - ttl: Some(String::from(ttl)), - ..CacheControl::default() - } -} - -fn browser_toolset_configs() -> BrowserToolsetConfigs { - BrowserToolsetConfigs { - type_text: None, - close_tab: None, - double_click: None, - file_upload: None, - find: None, - form_input: None, - get_page_text: None, - hold_key: None, - hover: None, - javascript_exec: None, - key: None, - left_click: None, - left_click_drag: None, - left_mouse_down: None, - left_mouse_up: None, - list_tabs: None, - middle_click: None, - mouse_move: None, - navigate: None, - new_tab: None, - read_console: None, - read_network: None, - read_page: None, - right_click: None, - screenshot: None, - scroll: None, - scroll_to: None, - switch_tab: None, - triple_click: None, - wait: None, - zoom: None, - extra: Map::new(), - } -} - -fn computer_toolset_configs() -> ComputerToolsetConfigs { - ComputerToolsetConfigs { - type_text: None, - cursor_position: None, - double_click: None, - hold_key: None, - key: None, - left_click: None, - left_click_drag: None, - left_mouse_down: None, - left_mouse_up: None, - middle_click: None, - mouse_move: None, - right_click: None, - screenshot: None, - scroll: None, - triple_click: None, - wait: None, - zoom: None, - extra: Map::new(), - } -} - -#[rstest] -#[case::bash_20241022( - json!({"type":"bash_20241022","name":"bash","input_examples":[{"command":"ls"}]}), - BuiltinMessagesTool::Bash20241022(ClientTool { - input_examples: Some(vec![Map::from_iter([(String::from("command"), json!("ls"))])]), - ..client_tool(BashToolName::Bash) - }) -)] -#[case::bash_20250124( - json!({"type":"bash_20250124","name":"bash","allowed_callers":["direct","code_execution_20260521"]}), - BuiltinMessagesTool::Bash20250124(ClientTool { - allowed_callers: Some(vec![AllowedCaller::Direct, AllowedCaller::CodeExecution20260521]), - ..client_tool(BashToolName::Bash) - }) -)] -#[case::text_editor_20241022( - json!({"type":"text_editor_20241022","name":"str_replace_editor","defer_loading":true}), - BuiltinMessagesTool::TextEditor20241022(ClientTool { - defer_loading: Some(true), - ..client_tool(StrReplaceEditorName::StrReplaceEditor) - }) -)] -#[case::text_editor_20250124( - json!({"type":"text_editor_20250124","name":"str_replace_editor","strict":true}), - BuiltinMessagesTool::TextEditor20250124(ClientTool { - strict: Some(true), - ..client_tool(StrReplaceEditorName::StrReplaceEditor) - }) -)] -#[case::text_editor_20250429( - json!({"type":"text_editor_20250429","name":"str_replace_based_edit_tool","cache_control":{"type":"ephemeral","ttl":"1h"}}), - BuiltinMessagesTool::TextEditor20250429(ClientTool { - cache_control: Some(cache_control("ephemeral", "1h")), - ..client_tool(StrReplaceBasedEditToolName::StrReplaceBasedEditTool) - }) -)] -#[case::text_editor_20250728( - json!({"type":"text_editor_20250728","name":"str_replace_based_edit_tool","max_characters":10000}), - BuiltinMessagesTool::TextEditor20250728(TextEditorTool20250728 { - name: StrReplaceBasedEditToolName::StrReplaceBasedEditTool, - allowed_callers: None, - cache_control: None, - defer_loading: None, - input_examples: None, - max_characters: Some(10000), - strict: None, - extra: Map::new(), - }) -)] -#[case::memory_20250818( - json!({"type":"memory_20250818","name":"memory","allowed_callers":["code_execution_20250825"]}), - BuiltinMessagesTool::Memory20250818(ClientTool { - allowed_callers: Some(vec![AllowedCaller::CodeExecution20250825]), - ..client_tool(MemoryToolName::Memory) - }) -)] -#[case::computer_20241022( - json!({"type":"computer_20241022","name":"computer","display_width_px":1024,"display_height_px":768,"display_number":1}), - BuiltinMessagesTool::Computer20241022(ComputerTool { - display_number: Some(1), - ..computer_tool(1024, 768) - }) -)] -#[case::computer_20250124( - json!({"type":"computer_20250124","name":"computer","display_width_px":1280,"display_height_px":800}), - BuiltinMessagesTool::Computer20250124(computer_tool(1280, 800)) -)] -#[case::computer_20251124( - json!({"type":"computer_20251124","name":"computer","display_width_px":1920,"display_height_px":1080,"enable_zoom":true}), - BuiltinMessagesTool::Computer20251124(ComputerTool20251124 { - name: ComputerToolName::Computer, - display_width_px: 1920, - display_height_px: 1080, - display_number: None, - enable_zoom: Some(true), - allowed_callers: None, - cache_control: None, - defer_loading: None, - input_examples: None, - strict: None, - extra: Map::new(), - }) -)] -#[case::code_execution_20250522( - json!({"type":"code_execution_20250522","name":"code_execution","strict":false}), - BuiltinMessagesTool::CodeExecution20250522(ServerTool { - strict: Some(false), - ..server_tool(CodeExecutionToolName::CodeExecution) - }) -)] -#[case::code_execution_20250825( - json!({"type":"code_execution_20250825","name":"code_execution","defer_loading":false}), - BuiltinMessagesTool::CodeExecution20250825(ServerTool { - defer_loading: Some(false), - ..server_tool(CodeExecutionToolName::CodeExecution) - }) -)] -#[case::code_execution_20260120( - json!({"type":"code_execution_20260120","name":"code_execution","allowed_callers":["direct"]}), - BuiltinMessagesTool::CodeExecution20260120(ServerTool { - allowed_callers: Some(vec![AllowedCaller::Direct]), - ..server_tool(CodeExecutionToolName::CodeExecution) - }) -)] -#[case::code_execution_20260521( - json!({"type":"code_execution_20260521","name":"code_execution","allowed_callers":["code_execution_20260120"]}), - BuiltinMessagesTool::CodeExecution20260521(ServerTool { - allowed_callers: Some(vec![AllowedCaller::CodeExecution20260120]), - ..server_tool(CodeExecutionToolName::CodeExecution) - }) -)] -#[case::tool_search_regex_20251119( - json!({"type":"tool_search_tool_regex_20251119","name":"tool_search_tool_regex","defer_loading":true}), - BuiltinMessagesTool::ToolSearchRegex20251119(ServerTool { - defer_loading: Some(true), - ..server_tool(ToolSearchRegexToolName::ToolSearchToolRegex) - }) -)] -#[case::tool_search_regex( - json!({"type":"tool_search_tool_regex","name":"tool_search_tool_regex","strict":true}), - BuiltinMessagesTool::ToolSearchRegex(ServerTool { - strict: Some(true), - ..server_tool(ToolSearchRegexToolName::ToolSearchToolRegex) - }) -)] -#[case::tool_search_bm25_20251119( - json!({"type":"tool_search_tool_bm25_20251119","name":"tool_search_tool_bm25","defer_loading":false}), - BuiltinMessagesTool::ToolSearchBm2520251119(ServerTool { - defer_loading: Some(false), - ..server_tool(ToolSearchBm25ToolName::ToolSearchToolBm25) - }) -)] -#[case::tool_search_bm25( - json!({"type":"tool_search_tool_bm25","name":"tool_search_tool_bm25","strict":false}), - BuiltinMessagesTool::ToolSearchBm25(ServerTool { - strict: Some(false), - ..server_tool(ToolSearchBm25ToolName::ToolSearchToolBm25) - }) -)] -#[case::web_search_20250305( - json!({"type":"web_search_20250305","name":"web_search","max_uses":3,"allowed_domains":["example.test"], - "user_location":{"type":"approximate","city":"San Francisco","country":"US","timezone":"America/Los_Angeles"}}), - BuiltinMessagesTool::WebSearch20250305(WebSearchTool { - max_uses: Some(3), - allowed_domains: Some(vec![String::from("example.test")]), - user_location: Some(WebSearchUserLocation { - location_type: UserLocationType::Approximate, - city: Some(String::from("San Francisco")), - country: Some(String::from("US")), - region: None, - timezone: Some(String::from("America/Los_Angeles")), - extra: Map::new(), - }), - ..web_search_tool() - }) -)] -#[case::web_search_20260209( - json!({"type":"web_search_20260209","name":"web_search","blocked_domains":["blocked.test"]}), - BuiltinMessagesTool::WebSearch20260209(WebSearchTool { - blocked_domains: Some(vec![String::from("blocked.test")]), - ..web_search_tool() - }) -)] -#[case::web_search_20260318( - json!({"type":"web_search_20260318","name":"web_search","response_inclusion":"excluded"}), - BuiltinMessagesTool::WebSearch20260318(WebSearchTool20260318 { - name: WebSearchToolName::WebSearch, - allowed_callers: None, - allowed_domains: None, - blocked_domains: None, - cache_control: None, - defer_loading: None, - max_uses: None, - response_inclusion: Some(ResponseInclusion::Excluded), - strict: None, - user_location: None, - extra: Map::new(), - }) -)] -#[case::web_fetch_20250910( - json!({"type":"web_fetch_20250910","name":"web_fetch","max_content_tokens":5000,"citations":{"enabled":true}, - "url_sources":{"client_tool_results":{"type":"only","tools":[{"type":"tool_reference","name":"lookup"}]}, - "server_tool_results":{"type":"all"},"user_input":{"type":"none"}}}), - BuiltinMessagesTool::WebFetch20250910(WebFetchTool { - max_content_tokens: Some(5000), - citations: Some(CitationsConfig { - enabled: Some(true), - extra: Map::new(), - }), - url_sources: Some(WebFetchUrlSources { - client_tool_results: Some(ToolResultUrlSource::Only { - tools: vec![tool_reference("lookup")], - extra: Map::new(), - }), - server_tool_results: Some(ToolResultUrlSource::All { extra: Map::new() }), - user_input: Some(UserInputUrlSource::None { extra: Map::new() }), - extra: Map::new(), - }), - ..web_fetch_tool() - }) -)] -#[case::web_fetch_20260209( - json!({"type":"web_fetch_20260209","name":"web_fetch","url_sources":{"server_tool_results":{"type":"except","tools":[{"type":"tool_reference","name":"web_search"}]}}}), - BuiltinMessagesTool::WebFetch20260209(WebFetchTool { - url_sources: Some(WebFetchUrlSources { - server_tool_results: Some(ToolResultUrlSource::Except { - tools: vec![tool_reference("web_search")], - extra: Map::new(), - }), - ..web_fetch_url_sources() - }), - ..web_fetch_tool() - }) -)] -#[case::web_fetch_20260309( - json!({"type":"web_fetch_20260309","name":"web_fetch","use_cache":false}), - BuiltinMessagesTool::WebFetch20260309(WebFetchTool20260309 { - name: WebFetchToolName::WebFetch, - allowed_callers: None, - allowed_domains: None, - blocked_domains: None, - cache_control: None, - citations: None, - defer_loading: None, - max_content_tokens: None, - max_uses: None, - strict: None, - url_sources: None, - use_cache: Some(false), - extra: Map::new(), - }) -)] -#[case::web_fetch_20260318( - json!({"type":"web_fetch_20260318","name":"web_fetch","use_cache":true,"response_inclusion":"full"}), - BuiltinMessagesTool::WebFetch20260318(WebFetchTool20260318 { - name: WebFetchToolName::WebFetch, - allowed_callers: None, - allowed_domains: None, - blocked_domains: None, - cache_control: None, - citations: None, - defer_loading: None, - max_content_tokens: None, - max_uses: None, - response_inclusion: Some(ResponseInclusion::Full), - strict: None, - url_sources: None, - use_cache: Some(true), - extra: Map::new(), - }) -)] -#[case::advisor_20260301( - json!({"type":"advisor_20260301","name":"advisor","model":"claude-opus-5-5","max_tokens":2048,"caching":{"type":"ephemeral","ttl":"5m"}}), - BuiltinMessagesTool::Advisor20260301(AdvisorTool { - name: AdvisorToolName::Advisor, - model: String::from("claude-opus-5-5"), - allowed_callers: None, - cache_control: None, - caching: Some(cache_control("ephemeral", "5m")), - defer_loading: None, - max_tokens: Some(2048), - max_uses: None, - strict: None, - extra: Map::new(), - }) -)] -#[case::browser_toolset_20260801( - json!({"type":"browser_toolset_20260801","configs":{"type":{"enabled":false},"javascript_exec":{"defer_loading":true}}}), - BuiltinMessagesTool::BrowserToolset20260801(Toolset { - cache_control: None, - configs: Some(Box::new(BrowserToolsetConfigs { - type_text: Some(toolset_config(Some(false), None)), - javascript_exec: Some(toolset_config(None, Some(true))), - ..browser_toolset_configs() - })), - extra: Map::new(), - }) -)] -#[case::computer_toolset_20260801( - json!({"type":"computer_toolset_20260801","configs":{"zoom":{"enabled":false},"cursor_position":{"enabled":true}}}), - BuiltinMessagesTool::ComputerToolset20260801(Toolset { - cache_control: None, - configs: Some(Box::new(ComputerToolsetConfigs { - zoom: Some(toolset_config(Some(false), None)), - cursor_position: Some(toolset_config(Some(true), None)), - ..computer_toolset_configs() - })), - extra: Map::new(), - }) -)] -#[case::mcp_toolset( - json!({"type":"mcp_toolset","mcp_server_name":"kb","default_config":{"enabled":false}, - "configs":{"search":{"enabled":true,"defer_loading":true}}, - "tools":[{"name":"search","input_schema":{"type":"object"},"description":"Search"}]}), - BuiltinMessagesTool::McpToolset(McpToolset { - mcp_server_name: String::from("kb"), - cache_control: None, - configs: Some(IndexMap::from([(String::from("search"), toolset_config(Some(true), Some(true)))])), - default_config: Some(toolset_config(Some(false), None)), - tools: Some(vec![McpListedTool { - name: String::from("search"), - description: Some(String::from("Search")), - input_schema: JsonSchema::Object(Box::new(JsonSchemaObject { - schema_type: Some(JsonSchemaType::Name(String::from("object"))), - ..JsonSchemaObject::default() - })), - extra: Map::new(), - }]), - extra: Map::new(), - }) -)] -fn builtin_tools_decode_typed_definitions( - #[case] wire: Value, - #[case] expected: BuiltinMessagesTool, -) { - assert_eq!(round_trip::(wire), expected); -} - -#[rstest] -fn builtin_tools_preserve_unknown_fields() { - let tool = round_trip::( - json!({"type":"bash_20250124","name":"bash","extension":[1,null]}), - ); - let BuiltinMessagesTool::Bash20250124(bash) = tool else { - panic!("expected bash tool"); - }; - assert_eq!(bash.extra.get("extension"), Some(&json!([1, null]))); -} - -#[rstest] -#[case::input_tokens("input_tokens", false)] -#[case::tool_uses("tool_uses", true)] -fn context_trigger_exposes_typed_threshold(#[case] tag: &str, #[case] tool_uses: bool) { - let wire = json!({"type":tag,"value":1024,"extension":true}); - let trigger: ContextTrigger = serde_json::from_value(wire.clone()).unwrap(); - match &trigger { - ContextTrigger::InputTokens { value, extra } => { - assert!(!tool_uses); - assert_eq!(*value, 1024); - assert_eq!(extra.get("extension"), Some(&json!(true))); - } - ContextTrigger::ToolUses { value, extra } => { - assert!(tool_uses); - assert_eq!(*value, 1024); - assert_eq!(extra.get("extension"), Some(&json!(true))); - } - } - assert_eq!(serde_json::to_value(trigger).unwrap(), wire); -} - -#[rstest] -#[case::negative(json!({"type":"input_tokens","value":-1}))] -#[case::wrong_shape(json!({"type":"input_tokens","value":"1024"}))] -#[case::missing_value(json!({"type":"input_tokens"}))] -#[case::null_value(json!({"type":"input_tokens","value":null}))] -#[case::tool_uses_negative(json!({"type":"tool_uses","value":-1}))] -#[case::tool_uses_fractional(json!({"type":"tool_uses","value":1.5}))] -#[case::tool_uses_missing_value(json!({"type":"tool_uses"}))] -#[case::missing_discriminator(json!({"value":1}))] -#[case::unknown_discriminator(json!({"type":"other","value":1}))] -fn token_threshold_requires_unsigned_integer(#[case] wire: Value) { - assert!(serde_json::from_value::(wire).is_err()); -} - -#[rstest] -#[case::missing_discriminator(json!({"name":"bash"}))] -#[case::unknown_discriminator(json!({"type":"future_tool","name":"bash"}))] -#[case::null_discriminator(json!({"type":null,"name":"bash"}))] -#[case::bash_without_name(json!({"type":"bash_20250124"}))] -#[case::bash_wrong_name(json!({"type":"bash_20241022","name":"shell"}))] -#[case::legacy_editor_with_new_name(json!({"type":"text_editor_20250124","name":"str_replace_based_edit_tool"}))] -#[case::new_editor_with_legacy_name(json!({"type":"text_editor_20250728","name":"str_replace_editor"}))] -#[case::editor_20250429_with_legacy_name(json!({"type":"text_editor_20250429","name":"str_replace_editor"}))] -#[case::memory_without_name(json!({"type":"memory_20250818"}))] -#[case::computer_without_width(json!({"type":"computer_20250124","name":"computer","display_height_px":768}))] -#[case::computer_without_height(json!({"type":"computer_20241022","name":"computer","display_width_px":1024}))] -#[case::zoom_computer_without_display(json!({"type":"computer_20251124","name":"computer"}))] -#[case::code_execution_without_name(json!({"type":"code_execution_20250825"}))] -#[case::regex_search_with_bm25_name(json!({"type":"tool_search_tool_regex","name":"tool_search_tool_bm25"}))] -#[case::bm25_search_with_regex_name(json!({"type":"tool_search_tool_bm25_20251119","name":"tool_search_tool_regex"}))] -#[case::web_search_with_fetch_name(json!({"type":"web_search_20250305","name":"web_fetch"}))] -#[case::web_fetch_without_name(json!({"type":"web_fetch_20260318"}))] -#[case::advisor_without_model(json!({"type":"advisor_20260301","name":"advisor"}))] -#[case::advisor_without_name(json!({"type":"advisor_20260301","model":"claude-opus-5-5"}))] -#[case::mcp_toolset_without_server(json!({"type":"mcp_toolset"}))] -#[case::unknown_allowed_caller(json!({"type":"code_execution_20250825","name":"code_execution","allowed_callers":["code_execution_20250522"]}))] -#[case::unknown_response_inclusion(json!({"type":"web_search_20260318","name":"web_search","response_inclusion":"partial"}))] -#[case::user_input_only_filter(json!({"type":"web_fetch_20250910","name":"web_fetch","url_sources":{"user_input":{"type":"only","tools":[]}}}))] -#[case::negative_limit(json!({"type":"web_search_20250305","name":"web_search","max_uses":-1}))] -#[case::wrong_mcp_config(json!({"type":"mcp_toolset","mcp_server_name":"kb","configs":{"search":{"enabled":"yes"}}}))] -#[case::wrong_browser_config(json!({"type":"browser_toolset_20260801","configs":{"navigate":{"enabled":1}}}))] -fn builtin_tools_reject_malformed_fields(#[case] wire: Value) { - assert!(serde_json::from_value::(wire.clone()).is_err()); - assert!(serde_json::from_value::(wire).is_err()); -} - -#[rstest] -fn existing_content_blocks_preserve_opaque_nested_fields() { - let wire = - json!({"type":"tool_result","content":[{"type":"text","text":7}],"source":{"url":7}}); - let block: litellm_llms_types::formats::messages::ContentBlock = - serde_json::from_value(wire.clone()).unwrap(); - assert_eq!(block.content.as_ref(), wire.get("content")); - assert_eq!(block.extra.get("source"), wire.get("source")); - assert_eq!(serde_json::to_value(block).unwrap(), wire); -} - -fn part(wire: Value) -> MessagesContentPart { - round_trip::(wire) -} - -#[rstest] -fn text_blocks_expose_every_citation_location() { - let block = part(json!({ - "type":"text", - "text":"cited", - "citations":[ - {"type":"char_location","cited_text":"a","document_index":0,"start_char_index":1,"end_char_index":6,"file_id":"file_1"}, - {"type":"page_location","cited_text":"b","document_index":1,"start_page_number":1,"end_page_number":2}, - {"type":"content_block_location","cited_text":"c","document_index":2,"start_block_index":0,"end_block_index":1}, - {"type":"web_search_result_location","cited_text":"d","url":"https://example.test","encrypted_index":"opaque","title":"result"}, - {"type":"search_result_location","cited_text":"e","search_result_index":3,"source":"kb","start_block_index":0,"end_block_index":2} - ], - "cache_control":{"type":"ephemeral"} - })); - let MessagesContentPart::Text(text) = &block else { - panic!("expected text block"); - }; - assert_eq!(text.text, "cited"); - let [ - Citation::CharLocation(chars), - Citation::PageLocation(page), - Citation::ContentBlockLocation(blocks), - Citation::WebSearchResultLocation(search), - Citation::SearchResultLocation(result), - ] = text.citations.as_deref().unwrap() - else { - panic!("expected one citation of each location type"); - }; - assert_eq!((chars.start_char_index, chars.end_char_index), (1, 6)); - assert_eq!(chars.file_id.as_deref(), Some("file_1")); - assert_eq!((page.start_page_number, page.end_page_number), (1, 2)); - assert_eq!(blocks.document_index, 2); - assert_eq!(search.encrypted_index, "opaque"); - assert_eq!(result.search_result_index, 3); - assert_eq!(result.source, "kb"); -} - -#[rstest] -fn text_block_null_optionals_are_omitted() { - let block: MessagesContentPart = serde_json::from_value( - json!({"type":"text","text":"hi","citations":null,"cache_control":null,"future":null}), - ) - .unwrap(); - let MessagesContentPart::Text(text) = &block else { - panic!("expected text block"); - }; - assert!(text.citations.is_none()); - assert!(text.cache_control.is_none()); - assert_eq!(text.extra.get("future"), Some(&Value::Null)); - assert_eq!( - serde_json::to_value(block).unwrap(), - json!({"type":"text","text":"hi","future":null}) - ); -} - -#[rstest] -fn request_media_blocks_expose_sources() { - let MessagesContentPart::Image(image) = part(json!({ - "type":"image", - "source":{"type":"base64","media_type":"image/png","data":"AA=="}, - "transformations":{"oversized_image":"error"} - })) else { - panic!("expected image block"); - }; - assert!( - matches!(&image.source, ContentSource::Base64 { media_type, .. } if media_type == "image/png") - ); - assert_eq!(image.extra["transformations"]["oversized_image"], "error"); - let MessagesContentPart::Document(document) = part(json!({ - "type":"document", - "source":{"type":"content","content":[ - {"type":"text","text":"Section 1"}, - {"type":"image","source":{"type":"url","url":"https://example.test/chart.png"}} - ]}, - "title":"Q3 report", - "context":"quarterly", - "citations":{"enabled":true} - })) else { - panic!("expected document block"); - }; - assert_eq!(document.title.as_deref(), Some("Q3 report")); - assert_eq!(document.citations.as_ref().unwrap().enabled, Some(true)); - let ContentSource::Content { - content: BlockContent::Blocks(blocks), - .. - } = &document.source - else { - panic!("expected content-block source"); - }; - assert!(matches!( - blocks.as_slice(), - [MessagesContentPart::Text(_), MessagesContentPart::Image(_)] - )); - let MessagesContentPart::SearchResult(search) = part(json!({ - "type":"search_result", - "source":"https://example.test/result", - "title":"result", - "content":[{"type":"text","text":"found"}], - "citations":{"enabled":false} - })) else { - panic!("expected search result block"); - }; - assert_eq!(search.source, "https://example.test/result"); - assert!( - matches!(search.content.as_slice(), [MessagesContentPart::Text(text)] if text.text == "found") - ); -} - -#[rstest] -fn reasoning_and_tool_blocks_expose_required_fields() { - let MessagesContentPart::Thinking(thinking) = - part(json!({"type":"thinking","thinking":"plan","signature":"sig"})) - else { - panic!("expected thinking block"); - }; - assert_eq!( - (thinking.thinking.as_str(), thinking.signature.as_str()), - ("plan", "sig") - ); - let MessagesContentPart::RedactedThinking(redacted) = - part(json!({"type":"redacted_thinking","data":"opaque"})) - else { - panic!("expected redacted thinking block"); - }; - assert_eq!(redacted.data, "opaque"); - let MessagesContentPart::ToolUse(tool_use) = part(json!({ - "type":"tool_use", - "id":"toolu_1", - "name":"lookup", - "input":{"query":[1,null]}, - "caller":{"type":"code_execution_20260120","tool_id":"srvtoolu_1"}, - "toolset_name":"browser" - })) else { - panic!("expected tool use block"); - }; - assert_eq!(tool_use.input["query"], json!([1, null])); - assert!( - matches!(&tool_use.caller, Some(ToolCaller::CodeExecution20260120 { tool_id, .. }) if tool_id == "srvtoolu_1") - ); - assert_eq!(tool_use.toolset_name.as_deref(), Some("browser")); - let MessagesContentPart::ToolResult(result) = part(json!({ - "type":"tool_result", - "tool_use_id":"toolu_1", - "is_error":false, - "content":[ - {"type":"tool_reference","tool_name":"lookup"}, - {"type":"browser_state","tabs":[{"tab_id":"1","title":"","url":"","active":true}], - "state_changes":[{"type":"download_completed","download_id":"d1","url":"https://example.test/f","size_bytes":3}]} - ] - })) else { - panic!("expected tool result block"); - }; - let Some(BlockContent::Blocks(blocks)) = &result.content else { - panic!("expected nested result blocks"); - }; - let [ - MessagesContentPart::ToolReference(reference), - MessagesContentPart::BrowserState(browser), - ] = blocks.as_slice() - else { - panic!("expected tool reference and browser state"); - }; - assert_eq!(reference.tool_name, "lookup"); - assert_eq!(browser.tabs[0].active, Some(true)); - assert!(matches!( - browser.state_changes.as_deref(), - Some([BrowserStateChange::DownloadCompleted { - size_bytes: Some(3), - path: None, - .. - }]) - )); - let MessagesContentPart::ToolResult(text_result) = - part(json!({"type":"tool_result","tool_use_id":"toolu_2","content":"done"})) - else { - panic!("expected tool result block"); - }; - assert_eq!(text_result.content, Some(BlockContent::Text("done".into()))); - assert!(text_result.is_error.is_none()); -} - -#[rstest] -fn server_tool_results_expose_nested_result_unions() { - let MessagesContentPart::ServerToolUse(server) = part( - json!({"type":"server_tool_use","id":"srvtoolu_1","name":"web_search","input":{"query":"rust"}}), - ) else { - panic!("expected server tool use"); - }; - assert_eq!(server.name, "web_search"); - let MessagesContentPart::WebSearchToolResult(search) = part(json!({ - "type":"web_search_tool_result", - "tool_use_id":"srvtoolu_1", - "content":[{"type":"web_search_result","url":"https://example.test","title":"t","encrypted_content":"e","page_age":"1d"}] - })) else { - panic!("expected web search result"); - }; - let WebSearchToolResultContent::Results(results) = &search.content else { - panic!("expected search results"); - }; - assert_eq!(results[0].page_age.as_deref(), Some("1d")); - let MessagesContentPart::WebSearchToolResult(search_error) = part(json!({ - "type":"web_search_tool_result", - "tool_use_id":"srvtoolu_1", - "content":{"type":"web_search_tool_result_error","error_code":"max_uses_exceeded"} - })) else { - panic!("expected web search error"); - }; - assert!( - matches!(&search_error.content, WebSearchToolResultContent::Error(error) if error.error_code == "max_uses_exceeded") - ); - let MessagesContentPart::WebFetchToolResult(fetch) = part(json!({ - "type":"web_fetch_tool_result", - "tool_use_id":"srvtoolu_2", - "content":{"type":"web_fetch_result","url":"https://example.test","retrieved_at":"2026-01-01T00:00:00Z", - "content":{"type":"document","source":{"type":"text","media_type":"text/plain","data":"page"}}} - })) else { - panic!("expected web fetch result"); - }; - let WebFetchToolResultContent::WebFetchResult(fetched) = &fetch.content else { - panic!("expected fetched page"); - }; - let WebFetchDocument::Document(document) = &fetched.content; - assert!(matches!(&document.source, ContentSource::Text { data, .. } if data == "page")); - assert!(fetched.extra.is_empty()); - let MessagesContentPart::CodeExecutionToolResult(code) = part(json!({ - "type":"code_execution_tool_result", - "tool_use_id":"srvtoolu_3", - "content":{"type":"encrypted_code_execution_result","encrypted_stdout":"opaque","stderr":"","return_code":-1, - "content":[{"type":"code_execution_output","file_id":"file_1"}]} - })) else { - panic!("expected code execution result"); - }; - let CodeExecutionToolResultContent::EncryptedCodeExecutionResult(encrypted) = &code.content - else { - panic!("expected encrypted execution result"); - }; - assert_eq!(encrypted.return_code, -1); - assert!( - matches!(encrypted.content.as_slice(), [CodeExecutionOutput::CodeExecutionOutput { file_id, .. }] if file_id == "file_1") - ); - let MessagesContentPart::BashCodeExecutionToolResult(bash) = part(json!({ - "type":"bash_code_execution_tool_result", - "tool_use_id":"srvtoolu_7", - "content":{"type":"bash_code_execution_result","stdout":"ok","stderr":"","return_code":0, - "content":[{"type":"bash_code_execution_output","file_id":"file_2"}]} - })) else { - panic!("expected bash code execution result"); - }; - let BashCodeExecutionToolResultContent::BashCodeExecutionResult(ran) = &bash.content else { - panic!("expected bash execution result"); - }; - assert_eq!((ran.stdout.as_str(), ran.return_code), ("ok", 0)); - assert!( - matches!(ran.content.as_slice(), [BashCodeExecutionOutput::BashCodeExecutionOutput { file_id, .. }] if file_id == "file_2") - ); - assert!(ran.extra.is_empty()); - let MessagesContentPart::TextEditorCodeExecutionToolResult(editor) = part(json!({ - "type":"text_editor_code_execution_tool_result", - "tool_use_id":"srvtoolu_4", - "content":{"type":"text_editor_code_execution_view_result","content":"fn main() {}","file_type":"text","num_lines":1} - })) else { - panic!("expected text editor result"); - }; - let TextEditorCodeExecutionToolResultContent::TextEditorCodeExecutionViewResult(view) = - &editor.content - else { - panic!("expected view result"); - }; - assert_eq!(view.file_type, TextEditorFileType::Text); - assert_eq!(view.num_lines, Some(1)); - assert!(view.start_line.is_none()); - let MessagesContentPart::ToolSearchToolResult(tool_search) = part(json!({ - "type":"tool_search_tool_result", - "tool_use_id":"srvtoolu_5", - "content":{"type":"tool_search_tool_result_error","error_code":"unavailable","error_message":"down"} - })) else { - panic!("expected tool search result"); - }; - assert!( - matches!(&tool_search.content, ToolSearchToolResultContent::ToolSearchToolResultError(error) if error.error_message.as_deref() == Some("down")) - ); - let MessagesContentPart::ToolSearchToolResult(found) = part(json!({ - "type":"tool_search_tool_result", - "tool_use_id":"srvtoolu_8", - "content":{"type":"tool_search_tool_search_result","tool_references":[{"type":"tool_reference","tool_name":"lookup"}]} - })) else { - panic!("expected tool search result"); - }; - let ToolSearchToolResultContent::ToolSearchToolSearchResult(result) = &found.content else { - panic!("expected tool search hits"); - }; - let [ToolSearchReference::ToolReference(reference)] = result.tool_references.as_slice() else { - panic!("expected one tool reference"); - }; - assert_eq!(reference.tool_name, "lookup"); - assert!(reference.extra.is_empty() && result.extra.is_empty()); - let MessagesContentPart::AdvisorToolResult(advisor) = part(json!({ - "type":"advisor_tool_result", - "tool_use_id":"srvtoolu_6", - "content":{"type":"advisor_result","text":"advice"} - })) else { - panic!("expected advisor result"); - }; - assert!( - matches!(&advisor.content, AdvisorToolResultContent::AdvisorResult { text, stop_reason: None, .. } if text == "advice") - ); -} - -#[rstest] -fn beta_blocks_expose_mcp_compaction_and_fallback_fields() { - let MessagesContentPart::McpToolUse(mcp) = part( - json!({"type":"mcp_tool_use","id":"mcptoolu_1","name":"search","server_name":"kb","input":{}}), - ) else { - panic!("expected MCP tool use"); - }; - assert_eq!(mcp.server_name, "kb"); - let MessagesContentPart::McpToolResult(mcp_result) = part( - json!({"type":"mcp_tool_result","tool_use_id":"mcptoolu_1","is_error":true,"content":"failed"}), - ) else { - panic!("expected MCP tool result"); - }; - assert_eq!(mcp_result.is_error, Some(true)); - assert_eq!( - mcp_result.content, - Some(McpToolResultContent::Text("failed".into())) - ); - let MessagesContentPart::McpToolResult(text_result) = part(json!({ - "type":"mcp_tool_result", - "tool_use_id":"mcptoolu_3", - "content":[{"type":"text","text":"found"}] - })) else { - panic!("expected MCP tool result"); - }; - let Some(McpToolResultContent::Blocks(blocks)) = &text_result.content else { - panic!("expected MCP text blocks"); - }; - let [McpToolResultText::Text(text)] = blocks.as_slice() else { - panic!("expected one text block"); - }; - assert_eq!(text.text, "found"); - let MessagesContentPart::McpToolResult(empty_result) = - part(json!({"type":"mcp_tool_result","tool_use_id":"mcptoolu_2"})) - else { - panic!("expected MCP tool result"); - }; - assert!(empty_result.content.is_none()); - assert!(empty_result.extra.is_empty()); - let MessagesContentPart::McpToolListing(listing) = part(json!({ - "type":"mcp_tool_listing", - "mcp_server_name":"kb", - "tools":[{"name":"search","input_schema":{"type":"object"}}] - })) else { - panic!("expected MCP tool listing"); - }; - assert!(listing.tools[0].description.is_none()); - let MessagesContentPart::Compaction(compaction) = part(json!({ - "type":"compaction", - "content":"summary", - "encrypted_content":"opaque", - "tool_changes":[ - {"type":"tool_addition","tool":{"type":"tool_definition","definition":{"name":"lookup","input_schema":{"type":"object"}}}}, - {"type":"tool_removal","tool":{"type":"mcp_tool_reference","server_name":"kb","name":"search"}} - ] - })) else { - panic!("expected compaction block"); - }; - let [ - ToolChange::ToolAddition(addition), - ToolChange::ToolRemoval(removal), - ] = compaction.tool_changes.as_deref().unwrap() - else { - panic!("expected tool addition and removal"); - }; - let ToolChangeTarget::ToolDefinition { definition, .. } = &addition.tool else { - panic!("expected inline tool definition"); - }; - assert!( - matches!(definition.as_ref(), MessagesToolParam::Custom(tool) if tool.name == "lookup") - ); - assert!( - matches!(&removal.tool, ToolChangeTarget::McpToolReference { server_name, .. } if server_name == "kb") - ); - let failed: MessagesContentPart = - serde_json::from_value(json!({"type":"compaction","content":null})).unwrap(); - assert_eq!(failed, MessagesContentPart::Compaction(Default::default())); - let MessagesContentPart::Fallback(fallback) = part(json!({ - "type":"fallback", - "from":{"model":"claude-opus-5-5"}, - "to":{"model":"claude-sonnet-5-5"}, - "trigger":{"type":"refusal","category":"cyber"} - })) else { - panic!("expected fallback block"); - }; - assert_eq!(fallback.from.model, "claude-opus-5-5"); - assert_eq!(fallback.to.model, "claude-sonnet-5-5"); - let Some(FallbackTrigger::Refusal { category, extra }) = &fallback.trigger else { - panic!("expected refusal trigger"); - }; - assert_eq!(category.as_deref(), Some("cyber")); - assert!(extra.is_empty() && fallback.extra.is_empty()); - let MessagesContentPart::Fallback(untriggered) = part(json!({ - "type":"fallback", - "from":{"model":"claude-opus-5-5"}, - "to":{"model":"claude-sonnet-5-5"} - })) else { - panic!("expected fallback block"); - }; - assert!(untriggered.trigger.is_none()); -} - -#[rstest] -#[case::missing_tag(json!({"text":"hello"}))] -#[case::unknown_tag(json!({"type":"future_block","text":"hello"}))] -#[case::text_without_text(json!({"type":"text"}))] -#[case::wrong_text(json!({"type":"text","text":7}))] -#[case::citation_missing_location(json!({"type":"text","text":"a","citations":[{"type":"char_location","cited_text":"a","document_index":0}]}))] -#[case::unknown_citation(json!({"type":"text","text":"a","citations":[{"type":"future_location"}]}))] -#[case::thinking_without_signature(json!({"type":"thinking","thinking":"plan"}))] -#[case::tool_use_without_id(json!({"type":"tool_use","name":"lookup","input":{}}))] -#[case::tool_use_array_input(json!({"type":"tool_use","id":"t","name":"lookup","input":[]}))] -#[case::tool_result_without_id(json!({"type":"tool_result","content":"done"}))] -#[case::malformed_nested_block(json!({"type":"tool_result","tool_use_id":"t","content":[{"type":"text","text":7}]}))] -#[case::malformed_source(json!({"type":"document","source":{"type":"url","url":7}}))] -#[case::image_without_source(json!({"type":"image"}))] -#[case::search_result_without_title(json!({"type":"search_result","source":"s","content":[]}))] -#[case::web_search_result_without_url(json!({"type":"web_search_tool_result","tool_use_id":"t","content":[{"type":"web_search_result","title":"t","encrypted_content":"e"}]}))] -#[case::unknown_fetch_result(json!({"type":"web_fetch_tool_result","tool_use_id":"t","content":{"type":"future"}}))] -#[case::fractional_return_code(json!({"type":"bash_code_execution_tool_result","tool_use_id":"t","content":{"type":"bash_code_execution_result","stdout":"","stderr":"","return_code":0.5,"content":[]}}))] -#[case::unknown_file_type(json!({"type":"text_editor_code_execution_tool_result","tool_use_id":"t","content":{"type":"text_editor_code_execution_view_result","content":"","file_type":"video"}}))] -#[case::negative_line_count(json!({"type":"text_editor_code_execution_tool_result","tool_use_id":"t","content":{"type":"text_editor_code_execution_str_replace_result","new_lines":-1}}))] -#[case::unknown_tool_change(json!({"type":"compaction","tool_changes":[{"type":"tool_addition","tool":{"type":"future"}}]}))] -#[case::tool_search_non_reference(json!({"type":"tool_search_tool_result","tool_use_id":"t","content":{"type":"tool_search_tool_search_result","tool_references":[{"type":"text","text":"lookup"}]}}))] -#[case::web_fetch_text_content(json!({"type":"web_fetch_tool_result","tool_use_id":"t","content":{"type":"web_fetch_result","url":"https://example.test","content":{"type":"text","text":"page"}}}))] -#[case::bash_result_with_code_output(json!({"type":"bash_code_execution_tool_result","tool_use_id":"t","content":{"type":"bash_code_execution_result","stdout":"","stderr":"","return_code":0,"content":[{"type":"code_execution_output","file_id":"f"}]}}))] -#[case::code_result_with_bash_output(json!({"type":"code_execution_tool_result","tool_use_id":"t","content":{"type":"code_execution_result","stdout":"","stderr":"","return_code":0,"content":[{"type":"bash_code_execution_output","file_id":"f"}]}}))] -#[case::fallback_unknown_trigger(json!({"type":"fallback","from":{"model":"a"},"to":{"model":"b"},"trigger":{"type":"overload"}}))] -#[case::unknown_state_change(json!({"type":"browser_state","tabs":[],"state_changes":[{"type":"tab_closed","tab_id":"1"}]}))] -fn content_parts_reject_malformed_known_fields(#[case] wire: Value) { - assert!(serde_json::from_value::(wire).is_err()); -} - -#[rstest] -fn custom_tool_exposes_schema_and_preserves_extensions() { - let tool = round_trip::(json!({ - "type":"custom", - "name":"lookup", - "input_schema":{"type":"object","properties":{"query":{"type":"string"}}}, - "strict":false, - "defer_loading":true, - "allowed_callers":["direct","code_execution_20260120"], - "extension":{"nested":[1,null]} - })); - assert_eq!( - tool.allowed_callers.as_deref(), - Some([AllowedCaller::Direct, AllowedCaller::CodeExecution20260120].as_slice()) - ); - assert_eq!(tool.tool_type, Some(CustomToolType::Custom)); - assert_eq!(tool.name, "lookup"); - assert_eq!((tool.strict, tool.defer_loading), (Some(false), Some(true))); - let JsonSchema::Object(schema) = &tool.input_schema else { - panic!("expected an object schema"); - }; - assert!(schema.properties.as_ref().unwrap().contains_key("query")); - assert_eq!(tool.extra["extension"], json!({"nested":[1,null]})); -} - -#[rstest] -fn custom_tool_omits_null_discriminator() { - let tool: CustomTool = - serde_json::from_value(json!({"type":null,"name":"lookup","input_schema":true})).unwrap(); - assert!(tool.tool_type.is_none()); - assert!(tool.description.is_none()); - assert_eq!( - serde_json::to_value(tool).unwrap(), - json!({"name":"lookup","input_schema":true}) - ); -} - -#[rstest] -#[case::missing_name(json!({"input_schema":{}}))] -#[case::missing_schema(json!({"name":"lookup"}))] -#[case::wrong_name_shape(json!({"name":7,"input_schema":{}}))] -#[case::builtin_tool(json!({"name":"lookup","input_schema":{},"type":"bash_20250124"}))] -#[case::unknown_discriminator(json!({"name":"lookup","input_schema":{},"type":"future_tool"}))] -#[case::unknown_allowed_caller(json!({"name":"lookup","input_schema":{},"allowed_callers":["anyone"]}))] -fn custom_tool_rejects_invalid_shapes(#[case] wire: Value) { - assert!(serde_json::from_value::(wire).is_err()); -} - -#[rstest] -#[case::builtin(json!({"type":"web_search_20250305","name":"web_search","max_uses":2}), true)] -#[case::builtin_toolset(json!({"type":"mcp_toolset","mcp_server_name":"kb"}), true)] -#[case::custom(json!({"name":"lookup","input_schema":{"type":"object"}}), false)] -fn tool_params_dispatch_on_discriminator(#[case] wire: Value, #[case] builtin: bool) { - let tool = round_trip::(wire); - assert_eq!(matches!(tool, MessagesToolParam::Builtin(_)), builtin); -} - -#[rstest] -fn tool_params_reject_unknown_tool_types() { - assert!( - serde_json::from_value::( - json!({"type":"future_tool","name":"lookup","input_schema":{}}) - ) - .is_err() - ); -} - -#[rstest] -#[case::image(json!([{"type":"image","source":{"type":"url","url":"https://example.test/a.png"}}]))] -#[case::tool_reference(json!([{"type":"tool_reference","tool_name":"lookup"}]))] -#[case::untagged(json!([{"text":"found"}]))] -fn mcp_tool_result_content_rejects_non_text_blocks(#[case] content: Value) { - assert!( - serde_json::from_value::( - json!({"type":"mcp_tool_result","tool_use_id":"mcptoolu_1","content":content}) - ) - .is_err() - ); -} - -#[rstest] -#[case::model(json!({"type":"model_changed","cache_missed_input_tokens":12}), Some(12))] -#[case::system(json!({"type":"system_changed","cache_missed_input_tokens":3}), Some(3))] -#[case::tools(json!({"type":"tools_changed","cache_missed_input_tokens":0}), Some(0))] -#[case::messages(json!({"type":"messages_changed","cache_missed_input_tokens":9}), Some(9))] -#[case::not_found(json!({"type":"previous_message_not_found"}), None)] -#[case::unavailable(json!({"type":"unavailable"}), None)] -fn diagnostics_expose_cache_miss_reason(#[case] reason: Value, #[case] missed: Option) { - let diagnostics = round_trip::(json!({"cache_miss_reason":reason})); - let Some(Recognized::Known(reason)) = &diagnostics.cache_miss_reason else { - panic!("expected known cache miss reason"); - }; - let actual = match reason { - CacheMissReason::ModelChanged(tokens) - | CacheMissReason::SystemChanged(tokens) - | CacheMissReason::ToolsChanged(tokens) - | CacheMissReason::MessagesChanged(tokens) => { - assert!(tokens.extra.is_empty()); - Some(tokens.cache_missed_input_tokens) - } - CacheMissReason::PreviousMessageNotFound { extra } - | CacheMissReason::Unavailable { extra } => { - assert!(extra.is_empty()); - None - } - }; - assert_eq!(actual, missed); - assert!(diagnostics.extra.is_empty()); -} - -#[rstest] -#[case::model(json!({"type":"model_changed","cache_missed_input_tokens":12}), "model_changed")] -#[case::system(json!({"type":"system_changed","cache_missed_input_tokens":12}), "system_changed")] -#[case::tools(json!({"type":"tools_changed","cache_missed_input_tokens":12}), "tools_changed")] -#[case::messages(json!({"type":"messages_changed","cache_missed_input_tokens":12}), "messages_changed")] -fn cache_miss_reason_variants_keep_their_tags(#[case] reason: Value, #[case] tag: &str) { - let parsed: CacheMissReason = serde_json::from_value(reason).unwrap(); - assert_eq!(serde_json::to_value(parsed).unwrap()["type"], json!(tag)); -} - -#[rstest] -#[case::pending(json!({"cache_miss_reason":null}))] -#[case::future_reason(json!({"cache_miss_reason":{"type":"future_changed","cache_missed_input_tokens":1}}))] -#[case::missing_tokens(json!({"cache_miss_reason":{"type":"model_changed"}}))] -fn diagnostics_keep_pending_and_unmodeled_reasons(#[case] wire: Value) { - let diagnostics: MessagesDiagnostics = serde_json::from_value(wire.clone()).unwrap(); - match &diagnostics.cache_miss_reason { - None => assert_eq!(serde_json::to_value(&diagnostics).unwrap(), json!({})), - Some(Recognized::Unrecognized(kept)) => { - assert_eq!(kept, &wire["cache_miss_reason"]); - assert_eq!(serde_json::to_value(&diagnostics).unwrap(), wire); - } - Some(Recognized::Known(reason)) => panic!("unexpected known reason {reason:?}"), - } -} - -#[rstest] -#[case::previous(json!({"previous_message_id":"msg_1"}), Some(Some("msg_1")))] -#[case::first_turn(json!({"previous_message_id":null}), Some(None))] -#[case::absent(json!({}), None)] -fn diagnostics_param_distinguishes_null_from_absent_previous_message( - #[case] wire: Value, - #[case] expected: Option>, -) { - let param = round_trip::(wire); - assert_eq!( - param.previous_message_id.as_ref().map(Option::as_deref), - expected - ); - assert!(param.extra.is_empty()); -} - -#[rstest] -fn diagnostics_param_rejects_non_string_previous_message() { - assert!( - serde_json::from_value::(json!({"previous_message_id":7})) - .is_err() - ); -} - -#[rstest] -fn cache_miss_reason_rejects_non_numeric_tokens() { - assert!( - serde_json::from_value::( - json!({"type":"model_changed","cache_missed_input_tokens":"many"}) - ) - .is_err() - ); -} diff --git a/litellm-rust/crates/llms-types/tests/minimax.rs b/litellm-rust/crates/llms-types/tests/minimax.rs deleted file mode 100644 index a7bd46f0934..00000000000 --- a/litellm-rust/crates/llms-types/tests/minimax.rs +++ /dev/null @@ -1,82 +0,0 @@ -use litellm_llms_types::providers::minimax::{ - MinimaxMediaDetail, MinimaxMediaSource, MinimaxMessagesContentBlock, -}; -use rstest::rstest; -use serde_json::{Value, json}; - -#[rstest] -#[case::image(json!({"type":"image","source":{"type":"base64","media_type":"image/png","data":"AA==","detail":"low"}}))] -#[case::video(json!({"type":"video","source":{"type":"url","url":"https://example.test/video","detail":"high","fps":1,"max_long_side_pixel":1024,"future":null},"cache_control":{"type":"ephemeral"}}))] -#[case::mid_conversation_system(json!({"type":"mid_conv_system","text":"instruction","future":null}))] -fn provider_content_blocks_round_trip(#[case] wire: Value) { - let block: MinimaxMessagesContentBlock = serde_json::from_value(wire.clone()).unwrap(); - match &block { - MinimaxMessagesContentBlock::Image(image) => { - let MinimaxMediaSource::Base64 { - media_type, - data, - options, - } = &image.source - else { - panic!("expected base64 image source"); - }; - assert_eq!((media_type.as_str(), data.as_str()), ("image/png", "AA==")); - assert_eq!(options.detail, Some(MinimaxMediaDetail::Low)); - assert!(options.extra.is_empty()); - } - MinimaxMessagesContentBlock::Video(video) => { - let MinimaxMediaSource::Url { url, options } = &video.source else { - panic!("expected URL video source"); - }; - assert_eq!(url, "https://example.test/video"); - assert_eq!(options.detail, Some(MinimaxMediaDetail::High)); - assert_eq!(options.fps, Some(1.into())); - assert_eq!(options.max_long_side_pixel, Some(1024)); - assert_eq!(options.extra["future"], Value::Null); - assert_eq!( - video.cache_control.as_ref().unwrap().cache_type.as_deref(), - Some("ephemeral") - ); - } - MinimaxMessagesContentBlock::MidConvSystem { text, extra } => { - assert_eq!(text, "instruction"); - assert_eq!(extra["future"], Value::Null); - } - } - assert_eq!(serde_json::to_value(block).unwrap(), wire); -} - -#[rstest] -#[case::missing_source(json!({"type":"video"}))] -#[case::bad_source_tag(json!({"type":"image","source":{"type":"future"}}))] -#[case::url_without_url(json!({"type":"video","source":{"type":"url"}}))] -#[case::base64_without_data(json!({"type":"image","source":{"type":"base64","media_type":"image/png"}}))] -#[case::bad_detail(json!({"type":"image","source":{"type":"url","url":"u","detail":7}}))] -#[case::bad_fps(json!({"type":"video","source":{"type":"url","url":"u","fps":"fast"}}))] -#[case::missing_text(json!({"type":"mid_conv_system"}))] -fn provider_content_rejects_malformed_fields(#[case] wire: Value) { - assert!(serde_json::from_value::(wire).is_err()); -} - -#[rstest] -fn partial_media_options_omit_null_optionals() { - let block: MinimaxMessagesContentBlock = serde_json::from_value(json!({ - "type":"video", - "source":{"type":"url","url":"u","detail":null,"fps":null,"future":null}, - "cache_control":null - })) - .unwrap(); - let MinimaxMessagesContentBlock::Video(video) = &block else { - panic!("expected video") - }; - let MinimaxMediaSource::Url { options, .. } = &video.source else { - panic!("expected URL source"); - }; - assert!(options.detail.is_none()); - assert!(options.fps.is_none()); - assert!(video.cache_control.is_none()); - assert_eq!( - serde_json::to_value(block).unwrap(), - json!({"type":"video","source":{"type":"url","url":"u","future":null}}) - ); -} diff --git a/litellm-rust/crates/llms-types/tests/ocr.rs b/litellm-rust/crates/llms-types/tests/ocr.rs deleted file mode 100644 index 03e8a586a09..00000000000 --- a/litellm-rust/crates/llms-types/tests/ocr.rs +++ /dev/null @@ -1,140 +0,0 @@ -use litellm_llms_types::formats::ocr::{LiteLLMOcrResponse, OcrBoundingBox, OcrDocument, OcrPage}; -use rstest::rstest; -use serde_json::{Map, Value, json}; - -#[rstest] -#[case::missing_page_fields(json!({"pages": [{}]}))] -#[case::invalid_markdown(json!({"pages": [{"index": 0, "markdown": false}]}))] -#[case::invalid_image_bounds(json!({"pages": [{"index": 0, "markdown": "", "images": [{"bbox": []}]}]}))] -#[case::fractional_page_count(json!({"usage_info": {"pages_processed": 1.5}}))] -#[case::invalid_table(json!({"tables": [false]}))] -#[case::invalid_key_value_pair(json!({"keyValuePairs": [[]]}))] -#[case::invalid_native_response(json!({"provider_native_response": []}))] -fn normalized_response_rejects_invalid_shared_fields(#[case] fields: Value) { - let payload: Map = json!({"model": "model", "pages": []}) - .as_object() - .unwrap() - .iter() - .chain(fields.as_object().unwrap()) - .map(|(key, value)| (key.clone(), value.clone())) - .collect(); - assert!(serde_json::from_value::(Value::Object(payload)).is_err()); -} - -#[rstest] -fn document_rejects_non_string_provider_fields() { - assert!( - serde_json::from_value::(json!({ - "type": "image_url", "image_url": "https://example.com/image", "detail": 42 - })) - .is_err() - ); -} - -#[rstest] -#[case::large_integer(json!("9007199254740993.0"), 9_007_199_254_740_993)] -#[case::signed_decimal(json!("+2.000"), 2)] -#[case::separator(json!("1_000"), 1000)] -#[case::boolean(json!(true), 1)] -#[case::integral_float(json!(2.0), 2)] -fn numeric_coercion_preserves_integer_precision(#[case] value: Value, #[case] expected: i64) { - let page: OcrPage = serde_json::from_value(json!({"index": value, "markdown": ""})).unwrap(); - assert_eq!(page.index, expected); - assert_eq!( - serde_json::to_value(page).unwrap()["index"], - json!(expected) - ); -} - -#[rstest] -#[case::exponent(json!("1e2"))] -#[case::missing_integer(json!(".0"))] -#[case::missing_fraction(json!("2."))] -#[case::leading_separator(json!("_2"))] -#[case::repeated_separator(json!("2__0"))] -#[case::fractional_float(json!(2.5))] -#[case::null(json!(null))] -fn page_index_rejects_invalid_integers(#[case] value: Value) { - assert!(serde_json::from_value::(json!({"index": value, "markdown": ""})).is_err()); -} - -#[rstest] -#[case::document_url("document_url", "document_name", "application/pdf")] -#[case::image_url("image_url", "detail", "image/png")] -fn document_variants_preserve_provider_fields_when_rewriting_sources( - #[case] kind: &str, - #[case] field: &str, - #[case] mime_type: &str, - #[values(json!("kept"), Value::Null)] extra: Value, -) { - let original = "https://example.com/input"; - let replacement = format!("data:{mime_type};base64,AA=="); - let document: OcrDocument = - serde_json::from_value(json!({"type": kind, kind: original, field: extra})).unwrap(); - assert_eq!(document.source(), original); - assert!(document.is_remote()); - let rewritten = document.with_source(replacement.clone()); - assert!(!rewritten.is_remote()); - assert_eq!( - serde_json::to_value(rewritten).unwrap(), - json!({"type": kind, kind: replacement, field: extra}) - ); -} - -#[rstest] -#[case::absent_native(None)] -#[case::present_native(Some(Map::from_iter([("native".into(), json!({"nested": [null, 1]}))])))] -fn response_serialization_preserves_extensions_and_native_presence( - #[case] native: Option>, -) { - let response = LiteLLMOcrResponse { - extra_fields: Map::from_iter([("provider_field".into(), json!("kept"))]), - provider_native_response: native.clone(), - ..LiteLLMOcrResponse::new("model", vec![]) - }; - let serialized = response.into_json(); - assert_eq!(serialized["provider_field"], "kept"); - assert_eq!( - serialized.get("provider_native_response").cloned(), - native.clone().map(Value::Object) - ); - let decoded: LiteLLMOcrResponse = serde_json::from_value(serialized.clone()).unwrap(); - assert_eq!(decoded.provider_native_response, native); - assert_eq!(decoded.into_json(), serialized); -} - -#[rstest] -fn bounding_box_exposes_corner_coordinates_and_keeps_extensions() { - let wire = json!({"top_left_x":1,"top_left_y":2.5,"bottom_right_x":30,"bottom_right_y":40,"future":true}); - let bounds: OcrBoundingBox = serde_json::from_value(wire.clone()).unwrap(); - assert_eq!(bounds.top_left_x, Some(1.into())); - assert_eq!( - bounds - .top_left_y - .as_ref() - .and_then(serde_json::Number::as_f64), - Some(2.5) - ); - assert_eq!(bounds.bottom_right_x, Some(30.into())); - assert_eq!(bounds.bottom_right_y, Some(40.into())); - assert_eq!(Value::Object(bounds.extra.clone()), json!({"future":true})); - assert_eq!(serde_json::to_value(bounds).unwrap(), wire); -} - -#[rstest] -fn partial_bounding_box_omits_null_corners() { - let bounds: OcrBoundingBox = - serde_json::from_value(json!({"top_left_x":null,"bottom_right_y":4})).unwrap(); - assert!(bounds.top_left_x.is_none()); - assert_eq!( - serde_json::to_value(bounds).unwrap(), - json!({"bottom_right_y":4}) - ); -} - -#[rstest] -#[case::string_corner(json!({"top_left_x":"1"}))] -#[case::array_corner(json!({"bottom_right_y":[4]}))] -fn bounding_box_rejects_non_numeric_corners(#[case] wire: Value) { - assert!(serde_json::from_value::(wire).is_err()); -} diff --git a/litellm-rust/crates/llms-types/tests/responses.rs b/litellm-rust/crates/llms-types/tests/responses.rs deleted file mode 100644 index 73cf5e15f6f..00000000000 --- a/litellm-rust/crates/llms-types/tests/responses.rs +++ /dev/null @@ -1,717 +0,0 @@ -use litellm_llms_types::formats::chat_completions::{PromptCacheBreakpoint, PromptCacheMode}; -use litellm_llms_types::formats::responses::{ - ResponsesAdditionalTools, ResponsesApplyPatchCall, ResponsesApplyPatchCallOutput, - ResponsesApplyPatchOperation, ResponsesCodeOutput, ResponsesCompaction, - ResponsesComputerAction, ResponsesComputerCall, ResponsesComputerCallOutput, - ResponsesComputerOutput, ResponsesContentPart, ResponsesCoordinate, ResponsesCustomToolCall, - ResponsesCustomToolCallOutput, ResponsesFunctionCall, ResponsesFunctionCallOutput, - ResponsesImageGenerationCall, ResponsesInputContent, ResponsesLocalShellAction, - ResponsesLocalShellCall, ResponsesLocalShellCallOutput, ResponsesMcpApprovalRequest, - ResponsesMcpApprovalResponse, ResponsesMcpError, ResponsesMcpErrorDetail, ResponsesOutputItem, - ResponsesProgram, ResponsesProgramOutput, ResponsesSafetyCheck, ResponsesShellAction, - ResponsesShellCall, ResponsesShellCallOutput, ResponsesShellEnvironment, ResponsesShellOutcome, - ResponsesShellOutputChunk, ResponsesToolCaller, ResponsesToolOutput, ResponsesToolSearchCall, - ResponsesToolSearchOutput, ResponsesWebSearchAction, - streaming_websocket::{ResponsesEventResponse, ResponsesWsEventType}, -}; -use litellm_llms_types::recognized::Recognized; -use rstest::rstest; -use serde::{Serialize, de::DeserializeOwned}; -use serde_json::{Value, json}; - -fn round_trip(wire: Value) -where - T: DeserializeOwned + Serialize, -{ - let parsed: T = serde_json::from_value(wire.clone()).unwrap(); - assert_eq!(serde_json::to_value(parsed).unwrap(), wire); -} - -#[rstest] -#[case::create("response.create", ResponsesWsEventType::ResponseCreate)] -#[case::created("response.created", ResponsesWsEventType::ResponseCreated)] -#[case::completed("response.completed", ResponsesWsEventType::ResponseCompleted)] -#[case::failed("response.failed", ResponsesWsEventType::ResponseFailed)] -#[case::incomplete("response.incomplete", ResponsesWsEventType::ResponseIncomplete)] -#[case::error("error", ResponsesWsEventType::Error)] -#[case::unknown( - "response.output_text.delta", - ResponsesWsEventType::Other("response.output_text.delta".to_string()) -)] -#[case::empty("", ResponsesWsEventType::Other(String::new()))] -#[case::case_sensitive( - "Response.Completed", - ResponsesWsEventType::Other("Response.Completed".into()) -)] -#[case::escaped("future\"\\\n", ResponsesWsEventType::Other("future\"\\\n".into()))] -fn websocket_event_type_round_trips(#[case] wire: &str, #[case] expected: ResponsesWsEventType) { - let serialized = serde_json::to_string(&expected).unwrap(); - assert_eq!(serialized, serde_json::to_string(wire).unwrap()); - assert_eq!( - serde_json::from_str::(&serialized).unwrap(), - expected - ); -} - -#[rstest] -#[case::number("17")] -#[case::boolean("true")] -#[case::null("null")] -#[case::array("[]")] -#[case::object("{}")] -fn websocket_event_type_rejects_non_strings(#[case] wire: &str) { - assert!(serde_json::from_str::(wire).is_err()); -} - -#[cfg(feature = "schema")] -#[rstest] -fn websocket_event_type_schema_is_open_string() { - let schema = schemars::schema_for!(ResponsesWsEventType); - assert_eq!( - schema.to_value().get("type"), - Some(&serde_json::json!("string")) - ); -} - -#[rstest] -#[case::message(json!({"type":"message","role":"assistant","content":[{"type":"output_text","text":"answer","annotations":[{"type":"url_citation","url":"https://example.test","start_index":0,"end_index":6}]}]}))] -#[case::function_call(json!({"type":"function_call","call_id":"call_1","name":"lookup","arguments":"{\"query\":\"q\"}"}))] -#[case::custom_tool(json!({"type":"custom_tool_call","name":"lookup","input":"q"}))] -#[case::reasoning(json!({"type":"reasoning","summary":[{"type":"summary_text","text":"summary"}],"encrypted_content":"opaque"}))] -#[case::web_search(json!({"type":"web_search_call","action":{"type":"search","queries":["q"],"sources":[{"type":"url","url":"https://example.test"}]}}))] -#[case::file_search(json!({"type":"file_search_call","queries":["q"],"results":[{"file_id":"file_1","score":1,"attributes":{"custom":[1,null]}}]}))] -#[case::code(json!({"type":"code_interpreter_call","outputs":[{"type":"logs","logs":"done"},{"type":"image","url":"https://example.test"}]}))] -#[case::image(json!({"type":"image_generation_call","result":"generated"}))] -#[case::mcp(json!({"type":"mcp_call","server_label":"server","name":"lookup","arguments":"{}","output":"done"}))] -fn output_items_round_trip(#[case] wire: Value) { - let item: ResponsesOutputItem = serde_json::from_value(wire.clone()).unwrap(); - match &item { - ResponsesOutputItem::Message(message) => { - assert_eq!(message.role.as_deref(), Some("assistant")); - let Some(content) = &message.content else { - panic!("expected content") - }; - let [ - ResponsesContentPart::OutputText { - text, - annotations: Some(annotations), - .. - }, - ] = content.as_slice() - else { - panic!("expected output text and annotations") - }; - assert_eq!(text, "answer"); - let [ - litellm_llms_types::formats::responses::ResponsesAnnotation::UrlCitation(citation), - ] = annotations.as_slice() - else { - panic!("expected URL citation") - }; - assert_eq!(citation.url.as_deref(), Some("https://example.test")); - assert_eq!(citation.start_index, Some(0)); - } - ResponsesOutputItem::FunctionCall(call) => { - assert_eq!(call.call_id.as_deref(), Some("call_1")); - assert_eq!(call.name.as_deref(), Some("lookup")); - assert_eq!(call.arguments.as_deref(), Some("{\"query\":\"q\"}")); - } - ResponsesOutputItem::CustomToolCall(call) => { - assert_eq!(call.name.as_deref(), Some("lookup")); - assert_eq!(call.input.as_deref(), Some("q")); - } - ResponsesOutputItem::Reasoning(reasoning) => { - assert_eq!(reasoning.encrypted_content.as_deref(), Some("opaque")); - let Some(summary) = &reasoning.summary else { - panic!("expected summary") - }; - let [ResponsesContentPart::SummaryText { text, .. }] = summary.as_slice() else { - panic!("expected summary text") - }; - assert_eq!(text, "summary"); - } - ResponsesOutputItem::WebSearchCall(call) => { - let Some(ResponsesWebSearchAction::Search { - queries: Some(queries), - sources: Some(sources), - .. - }) = &call.action - else { - panic!("expected search action") - }; - assert_eq!(queries, &["q"]); - assert_eq!(sources[0].url.as_deref(), Some("https://example.test")); - } - ResponsesOutputItem::FileSearchCall(call) => { - assert_eq!(call.queries.as_deref(), Some(["q".to_owned()].as_slice())); - let Some(results) = &call.results else { - panic!("expected search results") - }; - assert_eq!(results[0].file_id.as_deref(), Some("file_1")); - assert_eq!(results[0].score, Some(1.into())); - assert_eq!( - results[0].attributes.as_ref().unwrap()["custom"], - json!([1, null]) - ); - } - ResponsesOutputItem::CodeInterpreterCall(call) => { - let Some(outputs) = &call.outputs else { - panic!("expected code outputs") - }; - let [ - ResponsesCodeOutput::Logs { logs, .. }, - ResponsesCodeOutput::Image { url, .. }, - ] = outputs.as_slice() - else { - panic!("expected logs and image") - }; - assert_eq!(logs, "done"); - assert_eq!(url, "https://example.test"); - } - ResponsesOutputItem::ImageGenerationCall(call) => { - assert_eq!(call.result.as_deref(), Some("generated")) - } - ResponsesOutputItem::McpCall(call) => { - assert_eq!(call.server_label.as_deref(), Some("server")); - assert_eq!(call.name.as_deref(), Some("lookup")); - assert_eq!(call.arguments.as_deref(), Some("{}")); - assert_eq!(call.output.as_deref(), Some("done")); - } - other => panic!("unexpected variant {other:?}"), - } - assert_eq!(serde_json::to_value(item).unwrap(), wire); -} - -#[rstest] -fn nested_event_response_exposes_typed_output_and_preserves_extensions() { - let wire = json!({ - "id":"response_1", - "model":"example-model", - "status":"completed", - "output":[{"type":"function_call","call_id":"call_1","name":"lookup","arguments":"{}","extension":true}], - "extension":{"nested":[1,null]} - }); - let response: ResponsesEventResponse = serde_json::from_value(wire.clone()).unwrap(); - assert_eq!(response.id.as_deref(), Some("response_1")); - assert_eq!(response.model.as_deref(), Some("example-model")); - let Some(output) = &response.output else { - panic!("expected typed output"); - }; - let [Recognized::Known(ResponsesOutputItem::FunctionCall(call))] = output.as_slice() else { - panic!("expected function call"); - }; - assert_eq!(call.name.as_deref(), Some("lookup")); - assert_eq!(call.arguments.as_deref(), Some("{}")); - assert_eq!(serde_json::to_value(response).unwrap(), wire); -} - -#[rstest] -#[case::empty(json!({}))] -#[case::partial(json!({"id":"response_1","output":[]}))] -fn nested_event_response_accepts_partial_metadata(#[case] wire: Value) { - round_trip::(wire); -} - -#[rstest] -#[case::wrong_model(json!({"model":7}))] -#[case::wrong_status(json!({"status":false}))] -#[case::wrong_output(json!({"output":{}}))] -fn event_response_rejects_malformed_typed_fields(#[case] wire: Value) { - assert!(serde_json::from_value::(wire).is_err()); -} - -#[rstest] -#[case::unknown_type(json!({"type":"future_item","id":"item_1","payload":[1,null]}))] -#[case::missing_tag(json!({"id":"item_1"}))] -#[case::malformed_known(json!({"type":"message","content":[{"type":"output_text","text":7}]}))] -fn event_response_keeps_unmodeled_output_items_beside_typed_ones(#[case] item: Value) { - let wire = json!({"output":[{"type":"function_call","name":"lookup"}, item.clone()]}); - let response: ResponsesEventResponse = serde_json::from_value(wire.clone()).unwrap(); - let Some( - [ - Recognized::Known(ResponsesOutputItem::FunctionCall(call)), - Recognized::Unrecognized(kept), - ], - ) = response.output.as_deref() - else { - panic!("expected one typed item and one preserved item"); - }; - assert_eq!(call.name.as_deref(), Some("lookup")); - assert_eq!(kept, &item); - assert_eq!(serde_json::to_value(response).unwrap(), wire); -} - -#[rstest] -#[case::find_in_page( - json!({"type":"find_in_page","url":"https://example.test","pattern":"needle"}), - json!({"type":"find_in_page","url":"https://example.test","pattern":"needle"}) -)] -#[case::legacy_find( - json!({"type":"find","url":"https://example.test","pattern":"needle"}), - json!({"type":"find_in_page","url":"https://example.test","pattern":"needle"}) -)] -fn web_search_find_action_exposes_url_and_pattern(#[case] wire: Value, #[case] serialized: Value) { - let action: ResponsesWebSearchAction = serde_json::from_value(wire).unwrap(); - let ResponsesWebSearchAction::FindInPage { - url, - pattern, - extra, - } = &action - else { - panic!("expected find_in_page action"); - }; - assert_eq!(url, "https://example.test"); - assert_eq!(pattern, "needle"); - assert!(extra.is_empty()); - assert_eq!(serde_json::to_value(action).unwrap(), serialized); -} - -#[rstest] -#[case::with_url(json!({"type":"open_page","url":"https://example.test"}), Some("https://example.test"))] -#[case::without_url(json!({"type":"open_page"}), None)] -fn web_search_open_page_url_is_optional(#[case] wire: Value, #[case] expected: Option<&str>) { - let action: ResponsesWebSearchAction = serde_json::from_value(wire.clone()).unwrap(); - let ResponsesWebSearchAction::OpenPage { url, .. } = &action else { - panic!("expected open_page action"); - }; - assert_eq!(url.as_deref(), expected); - assert_eq!(serde_json::to_value(action).unwrap(), wire); -} - -#[rstest] -#[case::find_pattern(json!({"type":"find_in_page","url":"u","pattern":false}))] -#[case::find_missing_url(json!({"type":"find_in_page","pattern":"p"}))] -#[case::open_page_url(json!({"type":"open_page","url":7}))] -fn web_search_action_rejects_malformed_fields(#[case] wire: Value) { - assert!(serde_json::from_value::(wire).is_err()); -} - -#[rstest] -fn event_response_optional_fields_omit_missing_and_null() { - let response: ResponsesEventResponse = serde_json::from_value(json!({ - "id":null,"model":null,"status":null,"output":null,"future":null - })) - .unwrap(); - assert!(response.id.is_none()); - assert!(response.model.is_none()); - assert!(response.status.is_none()); - assert!(response.output.is_none()); - assert_eq!( - serde_json::to_value(response).unwrap(), - json!({"future":null}) - ); -} - -#[rstest] -#[case::refusal(json!({"type":"refusal","refusal":"refused","future":null}), ResponsesContentPart::Refusal { refusal:"refused".into(), extra:serde_json::Map::from_iter([("future".into(), Value::Null)]) })] -#[case::reasoning(json!({"type":"reasoning_text","text":"reason"}), ResponsesContentPart::ReasoningText { text:"reason".into(), extra:Default::default() })] -fn content_parts_expose_typed_variants( - #[case] wire: Value, - #[case] expected: ResponsesContentPart, -) { - let content: ResponsesContentPart = serde_json::from_value(wire.clone()).unwrap(); - assert_eq!(content, expected); - assert_eq!(serde_json::to_value(content).unwrap(), wire); -} - -#[rstest] -fn mcp_discovery_exposes_tools_and_nested_schemas() { - let wire = json!({ - "type":"mcp_list_tools","id":"item_1","server_label":"tools", - "tools":[{"name":"lookup","description":"Look up a value","input_schema":{"type":"object","properties":{"query":{"type":"string"}}},"annotations":{"read_only":false},"future":null}], - "extension":[1,null] - }); - let item: ResponsesOutputItem = serde_json::from_value(wire.clone()).unwrap(); - let ResponsesOutputItem::McpListTools(discovery) = &item else { - panic!("expected discovery") - }; - assert_eq!(discovery.server_label.as_deref(), Some("tools")); - let tool = &discovery.tools.as_ref().unwrap()[0]; - assert_eq!(tool.name.as_deref(), Some("lookup")); - let Some(litellm_llms_types::json_schema::JsonSchema::Object(schema)) = &tool.input_schema - else { - panic!("expected tool schema") - }; - assert!(schema.properties.as_ref().unwrap().contains_key("query")); - assert_eq!(tool.annotations, Some(json!({"read_only":false}))); - assert_eq!(tool.extra["future"], Value::Null); - assert_eq!(serde_json::to_value(item).unwrap(), wire); -} - -#[rstest] -#[case::partial(json!({"server_label":"tools","tools":[]}))] -#[case::failure(json!({"server_label":"tools","error":"unavailable"}))] -fn mcp_discovery_accepts_partial_payloads(#[case] wire: Value) { - let discovery: litellm_llms_types::formats::responses::ResponsesMcpListTools = - serde_json::from_value(wire.clone()).unwrap(); - assert_eq!(discovery.server_label.as_deref(), Some("tools")); - assert!(discovery.id.is_none()); - assert_eq!(serde_json::to_value(discovery).unwrap(), wire); -} - -#[rstest] -#[case::message(json!("unavailable"), ResponsesMcpError::Message("unavailable".into()))] -#[case::protocol(json!({"type":"mcp_protocol_error","code":-32600,"message":"invalid","extension":null}), ResponsesMcpError::Detail(ResponsesMcpErrorDetail::McpProtocolError {code:-32600,message:"invalid".into(),extra:serde_json::Map::from_iter([("extension".into(), Value::Null)])}))] -#[case::http(json!({"type":"http_error","code":503,"message":"unavailable"}), ResponsesMcpError::Detail(ResponsesMcpErrorDetail::HttpError {code:503,message:"unavailable".into(),extra:Default::default()}))] -#[case::tool(json!({"type":"mcp_tool_execution_error","content":{"arbitrary":[1,null]}}), ResponsesMcpError::Detail(ResponsesMcpErrorDetail::McpToolExecutionError {content:json!({"arbitrary":[1,null]}),extra:Default::default()}))] -fn mcp_call_errors_expose_typed_variants(#[case] wire: Value, #[case] expected: ResponsesMcpError) { - let error: ResponsesMcpError = serde_json::from_value(wire.clone()).unwrap(); - assert_eq!(error, expected); - assert_eq!(serde_json::to_value(&error).unwrap(), wire); - let call: ResponsesOutputItem = - serde_json::from_value(json!({"type":"mcp_call","error":wire})).unwrap(); - let ResponsesOutputItem::McpCall(call) = call else { - panic!("expected call") - }; - assert_eq!(call.error, Some(error)); -} - -#[rstest] -#[case::tools_not_array(json!({"type":"mcp_list_tools","tools":{}}))] -#[case::invalid_name(json!({"type":"mcp_list_tools","tools":[{"name":7}]}))] -#[case::invalid_schema(json!({"type":"mcp_list_tools","tools":[{"input_schema":{"properties":{"query":7}}}]}))] -#[case::invalid_error_code(json!({"type":"mcp_call","error":{"type":"http_error","code":"503","message":"unavailable"}}))] -#[case::missing_error_content(json!({"type":"mcp_call","error":{"type":"mcp_tool_execution_error"}}))] -#[case::unknown_error_tag(json!({"type":"mcp_call","error":{"type":"future"}}))] -fn mcp_output_rejects_malformed_known_fields(#[case] wire: Value) { - assert!(serde_json::from_value::(wire).is_err()); -} - -fn text(value: &str) -> Option { - Some(value.to_owned()) -} - -fn program_caller() -> Option { - Some(ResponsesToolCaller::Program { - caller_id: "prog_1".into(), - extra: Default::default(), - }) -} - -fn direct_caller() -> Option { - Some(ResponsesToolCaller::Direct { - extra: Default::default(), - }) -} - -fn safety_check() -> ResponsesSafetyCheck { - ResponsesSafetyCheck { - id: text("sc_1"), - code: text("malicious_instructions"), - message: text("check"), - ..Default::default() - } -} - -#[rstest] -#[case::function_call( - json!({"type":"function_call","id":"fc_1","status":"completed","call_id":"call_1","name":"lookup","arguments":"{}","namespace":"ns","async":true,"caller":{"type":"program","caller_id":"prog_1"}}), - ResponsesOutputItem::FunctionCall(ResponsesFunctionCall { - id: text("fc_1"), status: text("completed"), call_id: text("call_1"), name: text("lookup"), - arguments: text("{}"), namespace: text("ns"), r#async: Some(true), caller: program_caller(), ..Default::default() - }) -)] -#[case::custom_tool_call( - json!({"type":"custom_tool_call","call_id":"call_1","name":"lookup","input":"q","namespace":"ns","async":false,"caller":{"type":"direct"}}), - ResponsesOutputItem::CustomToolCall(ResponsesCustomToolCall { - call_id: text("call_1"), name: text("lookup"), input: text("q"), namespace: text("ns"), - r#async: Some(false), caller: direct_caller(), ..Default::default() - }) -)] -#[case::image_generation_call( - json!({"type":"image_generation_call","id":"ig_1","status":"completed","result":"b64","action":"edit","background":"opaque","output_format":"webp","quality":"high","revised_prompt":"a cat","size":"1536x864"}), - ResponsesOutputItem::ImageGenerationCall(ResponsesImageGenerationCall { - id: text("ig_1"), status: text("completed"), result: text("b64"), action: text("edit"), background: text("opaque"), - output_format: text("webp"), quality: text("high"), revised_prompt: text("a cat"), size: text("1536x864"), ..Default::default() - }) -)] -#[case::function_call_output_text( - json!({"type":"function_call_output","id":"fco_1","status":"completed","call_id":"call_1","output":"done","caller":{"type":"direct"},"created_by":"user_1","name":"lookup","namespace":"ns"}), - ResponsesOutputItem::FunctionCallOutput(ResponsesFunctionCallOutput { - id: text("fco_1"), status: text("completed"), call_id: text("call_1"), output: Some(ResponsesToolOutput::Text("done".into())), - caller: direct_caller(), created_by: text("user_1"), name: text("lookup"), namespace: text("ns"), ..Default::default() - }) -)] -#[case::function_call_output_content( - json!({"type":"function_call_output","call_id":"call_1","output":[ - {"type":"input_text","text":"t","prompt_cache_breakpoint":{"mode":"explicit"}}, - {"type":"input_image","detail":"low","file_id":"file_1","image_url":"https://example.test/i.png"}, - {"type":"input_file","detail":"high","file_data":"data","file_id":"file_2","file_url":"https://example.test/f","filename":"f.pdf"} - ]}), - ResponsesOutputItem::FunctionCallOutput(ResponsesFunctionCallOutput { - call_id: text("call_1"), - output: Some(ResponsesToolOutput::Content(vec![ - ResponsesInputContent::InputText { - text: "t".into(), - prompt_cache_breakpoint: Some(PromptCacheBreakpoint { mode: PromptCacheMode::Explicit, extra: Default::default() }), - extra: Default::default(), - }, - ResponsesInputContent::InputImage { - detail: "low".into(), file_id: text("file_1"), image_url: text("https://example.test/i.png"), - prompt_cache_breakpoint: None, extra: Default::default(), - }, - ResponsesInputContent::InputFile { - detail: text("high"), file_data: text("data"), file_id: text("file_2"), file_url: text("https://example.test/f"), - filename: text("f.pdf"), prompt_cache_breakpoint: None, extra: Default::default(), - }, - ])), - ..Default::default() - }) -)] -#[case::custom_tool_call_output( - json!({"type":"custom_tool_call_output","id":"cto_1","status":"completed","call_id":"call_1","output":[{"type":"input_text","text":"t"}],"caller":{"type":"program","caller_id":"prog_1"},"created_by":"user_1"}), - ResponsesOutputItem::CustomToolCallOutput(ResponsesCustomToolCallOutput { - id: text("cto_1"), status: text("completed"), call_id: text("call_1"), - output: Some(ResponsesToolOutput::Content(vec![ResponsesInputContent::InputText { - text: "t".into(), prompt_cache_breakpoint: None, extra: Default::default(), - }])), - caller: program_caller(), created_by: text("user_1"), ..Default::default() - }) -)] -#[case::computer_call( - json!({"type":"computer_call","id":"cu_1","status":"completed","call_id":"call_1", - "pending_safety_checks":[{"id":"sc_1","code":"malicious_instructions","message":"check"}], - "action":{"type":"click","button":"left","x":1,"y":2,"keys":["shift"]}, - "actions":[ - {"type":"double_click","x":3,"y":4}, - {"type":"drag","path":[{"x":5,"y":6},{"x":7,"y":8}],"keys":["ctrl"]}, - {"type":"keypress","keys":["enter"]}, - {"type":"move","x":-1,"y":9}, - {"type":"screenshot"}, - {"type":"scroll","scroll_x":0,"scroll_y":-10,"x":11,"y":12}, - {"type":"type","text":"hello"}, - {"type":"wait"} - ]}), - ResponsesOutputItem::ComputerCall(ResponsesComputerCall { - id: text("cu_1"), status: text("completed"), call_id: text("call_1"), pending_safety_checks: Some(vec![safety_check()]), - action: Some(ResponsesComputerAction::Click { button: "left".into(), x: 1, y: 2, keys: Some(vec!["shift".into()]), extra: Default::default() }), - actions: Some(vec![ - ResponsesComputerAction::DoubleClick { x: 3, y: 4, keys: None, extra: Default::default() }, - ResponsesComputerAction::Drag { - path: vec![ - ResponsesCoordinate { x: 5, y: 6, extra: Default::default() }, - ResponsesCoordinate { x: 7, y: 8, extra: Default::default() }, - ], - keys: Some(vec!["ctrl".into()]), - extra: Default::default(), - }, - ResponsesComputerAction::Keypress { keys: vec!["enter".into()], extra: Default::default() }, - ResponsesComputerAction::Move { x: -1, y: 9, keys: None, extra: Default::default() }, - ResponsesComputerAction::Screenshot { extra: Default::default() }, - ResponsesComputerAction::Scroll { scroll_x: 0, scroll_y: -10, x: 11, y: 12, keys: None, extra: Default::default() }, - ResponsesComputerAction::Type { text: "hello".into(), extra: Default::default() }, - ResponsesComputerAction::Wait { extra: Default::default() }, - ]), - ..Default::default() - }) -)] -#[case::computer_call_output( - json!({"type":"computer_call_output","id":"cuo_1","status":"completed","call_id":"call_1","output":{"type":"computer_screenshot","file_id":"file_1","image_url":"https://example.test/s.png"},"acknowledged_safety_checks":[{"id":"sc_1","code":"malicious_instructions","message":"check"}],"created_by":"user_1"}), - ResponsesOutputItem::ComputerCallOutput(ResponsesComputerCallOutput { - id: text("cuo_1"), status: text("completed"), call_id: text("call_1"), - output: Some(ResponsesComputerOutput::ComputerScreenshot { file_id: text("file_1"), image_url: text("https://example.test/s.png"), extra: Default::default() }), - acknowledged_safety_checks: Some(vec![safety_check()]), created_by: text("user_1"), ..Default::default() - }) -)] -#[case::program( - json!({"type":"program","id":"prog_item","call_id":"prog_1","code":"run()","fingerprint":"fp"}), - ResponsesOutputItem::Program(ResponsesProgram { - id: text("prog_item"), call_id: text("prog_1"), code: text("run()"), fingerprint: text("fp"), ..Default::default() - }) -)] -#[case::program_output( - json!({"type":"program_output","id":"po_1","status":"completed","call_id":"prog_1","result":"42"}), - ResponsesOutputItem::ProgramOutput(ResponsesProgramOutput { - id: text("po_1"), status: text("completed"), call_id: text("prog_1"), result: text("42"), ..Default::default() - }) -)] -#[case::tool_search_call( - json!({"type":"tool_search_call","id":"ts_1","status":"completed","call_id":"call_1","arguments":{"query":["weather",null]},"execution":"server","created_by":"user_1"}), - ResponsesOutputItem::ToolSearchCall(ResponsesToolSearchCall { - id: text("ts_1"), status: text("completed"), call_id: text("call_1"), arguments: Some(json!({"query":["weather",null]})), - execution: text("server"), created_by: text("user_1"), ..Default::default() - }) -)] -#[case::tool_search_output( - json!({"type":"tool_search_output","id":"tso_1","status":"completed","call_id":"call_1","execution":"client","tools":[{"type":"function","name":"lookup","parameters":null,"strict":true}],"created_by":"user_1"}), - ResponsesOutputItem::ToolSearchOutput(ResponsesToolSearchOutput { - id: text("tso_1"), status: text("completed"), call_id: text("call_1"), execution: text("client"), - tools: Some(vec![serde_json::Map::from_iter([ - ("type".into(), json!("function")), ("name".into(), json!("lookup")), - ("parameters".into(), Value::Null), ("strict".into(), json!(true)), - ])]), - created_by: text("user_1"), ..Default::default() - }) -)] -#[case::additional_tools( - json!({"type":"additional_tools","id":"at_1","role":"developer","tools":[{"type":"local_shell"}]}), - ResponsesOutputItem::AdditionalTools(ResponsesAdditionalTools { - id: text("at_1"), role: text("developer"), - tools: Some(vec![serde_json::Map::from_iter([("type".into(), json!("local_shell"))])]), ..Default::default() - }) -)] -#[case::compaction( - json!({"type":"compaction","id":"cmp_1","encrypted_content":"opaque","created_by":"user_1"}), - ResponsesOutputItem::Compaction(ResponsesCompaction { - id: text("cmp_1"), encrypted_content: text("opaque"), created_by: text("user_1"), ..Default::default() - }) -)] -#[case::local_shell_call( - json!({"type":"local_shell_call","id":"ls_1","status":"completed","call_id":"call_1","action":{"type":"exec","command":["ls","-a"],"env":{"HOME":"/home/u"},"timeout_ms":1000,"user":"u","working_directory":"/tmp"}}), - ResponsesOutputItem::LocalShellCall(ResponsesLocalShellCall { - id: text("ls_1"), status: text("completed"), call_id: text("call_1"), - action: Some(ResponsesLocalShellAction::Exec { - command: vec!["ls".into(), "-a".into()], env: [("HOME".into(), "/home/u".into())].into(), - timeout_ms: Some(1000), user: text("u"), working_directory: text("/tmp"), extra: Default::default(), - }), - ..Default::default() - }) -)] -#[case::local_shell_call_output( - json!({"type":"local_shell_call_output","id":"lso_1","status":"completed","output":"{\"stdout\":\"x\"}"}), - ResponsesOutputItem::LocalShellCallOutput(ResponsesLocalShellCallOutput { - id: text("lso_1"), status: text("completed"), output: text("{\"stdout\":\"x\"}"), ..Default::default() - }) -)] -#[case::shell_call_local( - json!({"type":"shell_call","id":"sh_1","status":"in_progress","call_id":"call_1","action":{"commands":["ls"],"max_output_length":100,"timeout_ms":500},"environment":{"type":"local"},"caller":{"type":"direct"},"created_by":"user_1"}), - ResponsesOutputItem::ShellCall(ResponsesShellCall { - id: text("sh_1"), status: text("in_progress"), call_id: text("call_1"), - action: Some(ResponsesShellAction { commands: Some(vec!["ls".into()]), max_output_length: Some(100), timeout_ms: Some(500), ..Default::default() }), - environment: Some(ResponsesShellEnvironment::Local { extra: Default::default() }), - caller: direct_caller(), created_by: text("user_1"), ..Default::default() - }) -)] -#[case::shell_call_container( - json!({"type":"shell_call","call_id":"call_1","environment":{"type":"container_reference","container_id":"cntr_1"}}), - ResponsesOutputItem::ShellCall(ResponsesShellCall { - call_id: text("call_1"), - environment: Some(ResponsesShellEnvironment::ContainerReference { container_id: "cntr_1".into(), extra: Default::default() }), - ..Default::default() - }) -)] -#[case::shell_call_output( - json!({"type":"shell_call_output","id":"sho_1","status":"completed","call_id":"call_1","max_output_length":100,"output":[ - {"outcome":{"type":"exit","exit_code":2},"stdout":"out","stderr":"err","created_by":"user_1"}, - {"outcome":{"type":"timeout"},"stdout":"","stderr":""} - ],"caller":{"type":"program","caller_id":"prog_1"},"created_by":"user_1"}), - ResponsesOutputItem::ShellCallOutput(ResponsesShellCallOutput { - id: text("sho_1"), status: text("completed"), call_id: text("call_1"), max_output_length: Some(100), - output: Some(vec![ - ResponsesShellOutputChunk { - outcome: Some(ResponsesShellOutcome::Exit { exit_code: 2, extra: Default::default() }), - stdout: text("out"), stderr: text("err"), created_by: text("user_1"), ..Default::default() - }, - ResponsesShellOutputChunk { - outcome: Some(ResponsesShellOutcome::Timeout { extra: Default::default() }), - stdout: text(""), stderr: text(""), ..Default::default() - }, - ]), - caller: program_caller(), created_by: text("user_1"), ..Default::default() - }) -)] -#[case::apply_patch_create( - json!({"type":"apply_patch_call","id":"ap_1","status":"completed","call_id":"call_1","operation":{"type":"create_file","path":"a.txt","diff":"+a"},"caller":{"type":"direct"},"created_by":"user_1"}), - ResponsesOutputItem::ApplyPatchCall(ResponsesApplyPatchCall { - id: text("ap_1"), status: text("completed"), call_id: text("call_1"), - operation: Some(ResponsesApplyPatchOperation::CreateFile { path: "a.txt".into(), diff: "+a".into(), extra: Default::default() }), - caller: direct_caller(), created_by: text("user_1"), ..Default::default() - }) -)] -#[case::apply_patch_delete( - json!({"type":"apply_patch_call","call_id":"call_1","operation":{"type":"delete_file","path":"a.txt"}}), - ResponsesOutputItem::ApplyPatchCall(ResponsesApplyPatchCall { - call_id: text("call_1"), - operation: Some(ResponsesApplyPatchOperation::DeleteFile { path: "a.txt".into(), extra: Default::default() }), - ..Default::default() - }) -)] -#[case::apply_patch_update( - json!({"type":"apply_patch_call","call_id":"call_1","operation":{"type":"update_file","path":"a.txt","diff":"-a\n+b"}}), - ResponsesOutputItem::ApplyPatchCall(ResponsesApplyPatchCall { - call_id: text("call_1"), - operation: Some(ResponsesApplyPatchOperation::UpdateFile { path: "a.txt".into(), diff: "-a\n+b".into(), extra: Default::default() }), - ..Default::default() - }) -)] -#[case::apply_patch_call_output( - json!({"type":"apply_patch_call_output","id":"apo_1","status":"failed","call_id":"call_1","output":"conflict","caller":{"type":"direct"},"created_by":"user_1"}), - ResponsesOutputItem::ApplyPatchCallOutput(ResponsesApplyPatchCallOutput { - id: text("apo_1"), status: text("failed"), call_id: text("call_1"), output: text("conflict"), - caller: direct_caller(), created_by: text("user_1"), ..Default::default() - }) -)] -#[case::mcp_approval_request( - json!({"type":"mcp_approval_request","id":"apr_1","server_label":"s","name":"n","arguments":"{}"}), - ResponsesOutputItem::McpApprovalRequest(ResponsesMcpApprovalRequest { - id: text("apr_1"), server_label: text("s"), name: text("n"), arguments: text("{}"), ..Default::default() - }) -)] -#[case::mcp_approval_response( - json!({"type":"mcp_approval_response","id":"aprr_1","approval_request_id":"apr_1","approve":false,"reason":"denied"}), - ResponsesOutputItem::McpApprovalResponse(ResponsesMcpApprovalResponse { - id: text("aprr_1"), approval_request_id: text("apr_1"), approve: Some(false), reason: text("denied"), ..Default::default() - }) -)] -fn documented_output_items_parse_into_typed_variants( - #[case] wire: Value, - #[case] expected: ResponsesOutputItem, -) { - let item: ResponsesOutputItem = serde_json::from_value(wire.clone()).unwrap(); - assert_eq!(item, expected); - assert_eq!(serde_json::to_value(item).unwrap(), wire); -} - -#[rstest] -#[case::compaction(json!({"type":"compaction","id":"cmp_1","encrypted_content":"opaque"}))] -#[case::approval(json!({"type":"mcp_approval_request","id":"apr_1","server_label":"s","name":"n","arguments":"{}"}))] -#[case::shell(json!({"type":"shell_call","call_id":"call_1","action":{"commands":["ls"]}}))] -fn event_response_types_documented_output_items(#[case] item: Value) { - let wire = json!({"output":[item.clone()]}); - let response: ResponsesEventResponse = serde_json::from_value(wire.clone()).unwrap(); - let Some([Recognized::Known(known)]) = response.output.as_deref() else { - panic!("expected one typed item"); - }; - assert_eq!( - known, - &serde_json::from_value::(item).unwrap() - ); - assert_eq!(serde_json::to_value(response).unwrap(), wire); -} - -#[rstest] -#[case::unknown_caller(json!({"type":"function_call","caller":{"type":"future"}}))] -#[case::program_caller_missing_id(json!({"type":"shell_call","caller":{"type":"program"}}))] -#[case::untagged_caller(json!({"type":"custom_tool_call","caller":{}}))] -#[case::async_not_bool(json!({"type":"function_call","async":"yes"}))] -#[case::output_not_text_or_list(json!({"type":"function_call_output","output":{"text":"t"}}))] -#[case::unknown_input_content(json!({"type":"function_call_output","output":[{"type":"input_audio"}]}))] -#[case::input_text_missing_text(json!({"type":"custom_tool_call_output","output":[{"type":"input_text"}]}))] -#[case::input_image_missing_detail(json!({"type":"custom_tool_call_output","output":[{"type":"input_image","file_id":"f"}]}))] -#[case::bad_cache_mode(json!({"type":"custom_tool_call_output","output":[{"type":"input_text","text":"t","prompt_cache_breakpoint":{"mode":"implicit"}}]}))] -#[case::unknown_computer_action(json!({"type":"computer_call","action":{"type":"teleport"}}))] -#[case::click_missing_button(json!({"type":"computer_call","action":{"type":"click","x":1,"y":2}}))] -#[case::click_fractional_x(json!({"type":"computer_call","action":{"type":"click","button":"left","x":1.5,"y":2}}))] -#[case::drag_point_missing_y(json!({"type":"computer_call","actions":[{"type":"drag","path":[{"x":1}]}]}))] -#[case::keypress_keys_not_list(json!({"type":"computer_call","actions":[{"type":"keypress","keys":"enter"}]}))] -#[case::safety_check_not_object(json!({"type":"computer_call","pending_safety_checks":["sc_1"]}))] -#[case::unknown_screenshot_tag(json!({"type":"computer_call_output","output":{"type":"screenshot"}}))] -#[case::untagged_screenshot(json!({"type":"computer_call_output","output":{"file_id":"f"}}))] -#[case::tool_not_object(json!({"type":"tool_search_output","tools":["lookup"]}))] -#[case::unknown_local_shell_action(json!({"type":"local_shell_call","action":{"type":"spawn","command":[],"env":{}}}))] -#[case::local_shell_missing_env(json!({"type":"local_shell_call","action":{"type":"exec","command":["ls"]}}))] -#[case::local_shell_env_value(json!({"type":"local_shell_call","action":{"type":"exec","command":["ls"],"env":{"A":1}}}))] -#[case::shell_commands_not_list(json!({"type":"shell_call","action":{"commands":"ls"}}))] -#[case::unknown_shell_environment(json!({"type":"shell_call","environment":{"type":"container_auto"}}))] -#[case::container_missing_id(json!({"type":"shell_call","environment":{"type":"container_reference"}}))] -#[case::unknown_shell_outcome(json!({"type":"shell_call_output","output":[{"outcome":{"type":"killed"}}]}))] -#[case::exit_missing_code(json!({"type":"shell_call_output","output":[{"outcome":{"type":"exit"}}]}))] -#[case::negative_max_output(json!({"type":"shell_call_output","max_output_length":-1}))] -#[case::unknown_patch_operation(json!({"type":"apply_patch_call","operation":{"type":"rename_file","path":"a"}}))] -#[case::update_missing_diff(json!({"type":"apply_patch_call","operation":{"type":"update_file","path":"a"}}))] -#[case::approve_not_bool(json!({"type":"mcp_approval_response","approve":"true"}))] -#[case::compaction_content_not_string(json!({"type":"compaction","encrypted_content":7}))] -#[case::program_code_not_string(json!({"type":"program","code":["run()"]}))] -fn output_items_reject_malformed_typed_fields(#[case] wire: Value) { - assert!(serde_json::from_value::(wire).is_err()); -} diff --git a/litellm-rust/crates/llms-types/tests/serde_compat.rs b/litellm-rust/crates/llms-types/tests/serde_compat.rs deleted file mode 100644 index 76dba17a241..00000000000 --- a/litellm-rust/crates/llms-types/tests/serde_compat.rs +++ /dev/null @@ -1,86 +0,0 @@ -use litellm_llms_types::serde_compat::{FiniteF64, LaxI64}; -use rstest::rstest; -use serde::{Deserialize, Serialize}; -use serde_json::{Value, json}; -use serde_with::serde_as; - -#[serde_as] -#[derive(Debug, Deserialize, Serialize, PartialEq)] -struct Numbers { - #[serde_as(deserialize_as = "Option>")] - integers: Option>, - #[serde_as(deserialize_as = "Option")] - float: Option, -} - -#[rstest] -fn adapters_compose_and_serialize_as_numbers() { - let numbers: Numbers = serde_json::from_value(json!({ - "integers": ["9007199254740993.0", "1_000", " +2.000 ", 3.0, true], - "float": " 1.5 " - })) - .unwrap(); - assert_eq!( - serde_json::to_value(numbers).unwrap(), - json!({"integers": [9_007_199_254_740_993_i64, 1000, 2, 3, 1], "float": 1.5}) - ); -} - -#[rstest] -#[case::missing(json!({}))] -#[case::null(json!({"integers": null, "float": null}))] -fn optional_adapters_accept_missing_and_null_fields(#[case] input: Value) { - assert_eq!( - serde_json::from_value::(input).unwrap(), - Numbers { - integers: None, - float: None - } - ); -} - -#[rstest] -#[case::minimum(json!(i64::MIN), i64::MIN)] -#[case::maximum(json!(i64::MAX), i64::MAX)] -#[case::maximum_string(json!(i64::MAX.to_string()), i64::MAX)] -fn integers_preserve_bounds(#[case] input: Value, #[case] expected: i64) { - let numbers: Numbers = serde_json::from_value(json!({"integers": [input]})).unwrap(); - assert_eq!(numbers.integers, Some(vec![expected])); -} - -#[rstest] -#[case::unsigned_maximum(json!(u64::MAX))] -#[case::above_maximum(json!(9_223_372_036_854_775_808_u64))] -#[case::float_above_maximum(json!(9_223_372_036_854_775_808.0))] -#[case::below_minimum(json!("-9223372036854775809"))] -#[case::precise_fraction(json!("1.0000000000000001"))] -#[case::exponent(json!("1e3"))] -#[case::missing_fraction(json!("2."))] -#[case::missing_integer(json!(".0"))] -#[case::leading_separator(json!("_2"))] -#[case::repeated_separator(json!("2__0"))] -#[case::fraction(json!(2.5))] -#[case::null(json!(null))] -#[case::object(json!({}))] -fn integers_reject_invalid_values(#[case] input: Value) { - assert!(serde_json::from_value::(json!({"integers": [input]})).is_err()); -} - -#[rstest] -#[case::nan(json!("NaN"))] -#[case::positive_infinity(json!("inf"))] -#[case::negative_infinity(json!("-inf"))] -#[case::overflow(json!("1e999"))] -#[case::array(json!([]))] -fn floats_reject_nonfinite_and_invalid_values(#[case] input: Value) { - assert!(serde_json::from_value::(json!({"float": input})).is_err()); -} - -#[rstest] -#[case::integer(json!(2), 2.0)] -#[case::float(json!(2.5), 2.5)] -#[case::boolean(json!(true), 1.0)] -fn floats_accept_finite_numbers(#[case] input: Value, #[case] expected: f64) { - let numbers: Numbers = serde_json::from_value(json!({"float": input})).unwrap(); - assert_eq!(numbers.float, Some(expected)); -} diff --git a/litellm-rust/crates/llms-types/tests/wire_type.rs b/litellm-rust/crates/llms-types/tests/wire_type.rs deleted file mode 100644 index 75755b9f40a..00000000000 --- a/litellm-rust/crates/llms-types/tests/wire_type.rs +++ /dev/null @@ -1,32 +0,0 @@ -use litellm_llms_types::formats::chat_completions::ChatMessage; -use rstest::rstest; -use serde_json::json; - -#[rstest] -fn wire_type_preserves_serialization() { - let message = ChatMessage { - role: "user".to_owned(), - content: None, - name: None, - extra: Default::default(), - }; - - assert_eq!( - serde_json::to_value(message).unwrap(), - json!({"role": "user"}) - ); -} - -#[cfg(feature = "schema")] -#[rstest] -fn wire_type_supports_schema_generation() { - let schema = schemars::schema_for!(ChatMessage); - - assert!( - schema - .to_value() - .get("properties") - .and_then(serde_json::Value::as_object) - .is_some_and(|properties| properties.contains_key("role")) - ); -} diff --git a/litellm-rust/crates/llms/AGENTS.md b/litellm-rust/crates/llms/AGENTS.md index 494c3fa2d39..2dbc74e905c 100644 --- a/litellm-rust/crates/llms/AGENTS.md +++ b/litellm-rust/crates/llms/AGENTS.md @@ -1,39 +1,71 @@ -litellm-llms mirrors `litellm/llms/`: base config traits, provider transformations, and the OCR request handler in `base_llm/ocr/handler.rs`. Transport code (clients, media fetching, header helpers, transport errors) lives in `litellm-http`. See `../inference/AGENTS.md` for how the crates layer. +# rules -## Python/Rust transformation pairs +## Scope -Use the base OCR and Mistral OCR pairs as the reference when aligning transformations. Use `src///transformation.rs` for provider transformations. Python paths identify counterparts but do not dictate Rust module names +- This crate mirrors `litellm/llms/`: base config traits in `src/base_llm//`, provider transformations in `src///transformation.rs` +- Transport (clients, media fetching, header helpers, transport errors) lives in `litellm-http`. See `../inference/AGENTS.md` for crate layering -Keep corresponding operation names and parameter names when their responsibilities match. Rust types retain the Python semantic name with Rust acronym casing (`BaseOCRConfig` / `BaseOcrConfig`, `MistralOCRConfig` / `MistralOcrConfig`). Private Python helpers can drop their leading underscore. Give Rust adapter helpers distinct responsibility names rather than duplicating trait method names +## Layering -Order OCR config methods as supported parameters, credential metadata and connection resolution, health-check input, parameter mapping, environment validation, URL construction, request transformation, async request transformation, response transformation, async response transformation, and error conversion. Put constants and data types before the config, private helpers after it in operation order, and tests last. Rust-only trait hooks follow the corresponding Python methods +- `litellm-llms-types` owns API data contracts +- `src/base_llm//` owns the adapter contract and provider-independent machinery. It never imports a provider or embeds provider policy in trait defaults, normalization or context defaults +- `src///` owns that provider's implementation and policy +- `inference-` owns call orchestration +- These are the intended boundaries, not a claim that all code already satisfies them -Use trait defaults for unchanged inherited behavior and explicit delegation for shared provider behavior. Keep typed inputs, ownership, `Result`, and async I/O idiomatic. A matching path or symbol identifies the counterpart, not a claim of full behavioral parity +## Provider folders -Use named `#[rstest]` cases for independent input/output scenarios instead of loops or repeated calls in one test. Inject reusable setup with `#[fixture]` arguments and use `#[with(...)]` for fixture overrides. Keep assertions about the same result together +- `src//` holds provider-wide policy: credentials, auth, endpoints, model capabilities. `common_utils.rs` means shared across that provider's formats, not across providers +- Shared constants and wire types used by several of a provider's formats go in `litellm-llms-types/src/providers//` +- A provider may explicitly reuse another provider's helper when its policy applies to the backend (Bedrock, Vertex and Azure reuse `anthropic/messages` shaping for Claude). That does not make the policy format-wide +- Pure payload rewrites belong with transformations, not transport handlers, even in a file named `handler.rs` -Shared OCR document and response contracts live in `litellm-llms-types::formats::ocr`. `BaseOcrConfig` and decoding into adapter errors remain in `src/base_llm/ocr/transformation.rs`. Rust context/environment types support the runtime. `BaseOcrConfig::prepare_request` corresponds to Python's HTTP-handler preparation rather than a `BaseOCRConfig` method, and `validate_request_body` is a Rust-only hook. `src/base_llm/ocr/error.rs` and `src/base_llm/ocr/document.rs` are Rust-only: the OCR error taxonomy shared with the route, and inline-document helpers shared by several providers +## AGENTS.md convention -For Mistral, `async_transform_ocr_request` uses the base default in both languages. `resolve_headers` and `build_ocr_url` implement the respective environment and URL operations, and `normalize_response` implements the typed part of response transformation. Existing auth key/header handling and top-level response-extra preservation differ between languages. Layout refactors must preserve those behaviors and verify them with the existing tests +- Every file is `# rules` then `# references`, both concise bullets. Split a long `# rules` into `##` topic sections +- A folder gets an AGENTS.md only when it adds something: a deviation, an explicit reuse of another provider, or upstream docs no other file lists. A folder without one follows its nearest parent +- Every `src///` folder gets one, since it talks to its own upstream endpoint. `.greptile/files.json` lists them all +- A rule lives in the highest file where it holds. Never restate a parent's rule +- Each upstream URL appears in exactly one AGENTS.md, the one owning that contract. Others point to that file by path + - Format specs: `litellm-llms-types/src/formats//` + - Provider wire-type docs: `litellm-llms-types/src/providers//` + - Provider-wide docs (auth, errors, regions): `src//` + - One host's endpoint docs: `src///` -For non-OCR pairs, order corresponding methods as parameter support/mapping, environment validation, URL construction, request transformation, and response transformation, followed by Rust-only runtime hooks. Auth resolution remains split between configs and route preparation in the `litellm-inference-` crates. Chat `supported_openai_param_mappings` describes accepted OpenAI/provider name pairs, unlike Python's `get_supported_openai_params` name list. Audio `map_transcription_params` remains a Rust filtering helper +## Python pairs -Azure Messages maps to `llms/azure_ai/anthropic/messages_transformation.py`. Bedrock Converse maps to `llms/bedrock/chat/converse_transformation.py`. `AnthropicConfig`, `AmazonConverseConfig`, and the non-OCR base traits are partial ports. `OpenAiResponsesApiConfig` implements WebSocket transformations and a direct HTTP Responses path. Its HTTP path does not implement Python model-specific parameter rewriting or Responses-to-Chat emulation. Preserve their acceptance gates, passthrough behavior, and host fallback contracts when aligning layout +- Python paths identify counterparts but do not dictate Rust module names or class hierarchy. Preserve behavior and concepts, not structure +- Keep operation and parameter names when responsibilities match. Rust types keep the Python name with Rust acronym casing (`BaseOCRConfig` -> `BaseOcrConfig`). Private Python helpers drop the leading underscore +- Give Rust-only helpers distinct responsibility names instead of reusing trait method names +- Use trait defaults for unchanged inherited behavior and explicit delegation for shared provider behavior. Config traits represent real provider contracts, so do not recreate inheritance with extra traits +- Every provider format writes out its own `impl BaseConfig for Config`. Do not route several providers' methods through one cross-provider wrapper +- The base OCR and Mistral OCR pairs are the reference when aligning transformations -## Provider and format boundaries +## Method order -The same ownership rule applies to Messages, Responses, Chat Completions, OCR, and other API formats. `litellm-llms-types` owns shared API data contracts. `llms/src/base_llm//` owns provider adapter contracts and shared transformation machinery. `llms/src///` owns provider implementations and policy. `inference-` owns call orchestration. Repeating a format name identifies the API each layer handles, not duplicate ownership of its schema. These boundaries also apply between modules in the same crate +- OCR configs: supported params, credential metadata and connection resolution, health-check input, param mapping, env validation, URL, request, async request, response, async response, error conversion +- Other formats: param support and mapping, env validation, URL, request, response, then Rust-only runtime hooks +- Constants and data types before the config, private helpers after it in operation order, tests last -A provider adapter may explicitly reuse another provider's transformation helper when that policy applies to its backend, such as Bedrock's Claude adapter using Anthropic payload shaping. Reuse across hosts of the same model family does not make the policy format-wide. Keep provider policy out of shared trait defaults and generic normalization, and keep shared execution contexts limited to inputs the adapter contract actually needs. Pure payload rewrites belong with transformations, not transport handlers +## Known partial ports -- These are intended boundaries, not a claim that all existing code already satisfies them -- Preserve behavior and conceptual boundaries. Python names and layout are reference points, not requirements to reproduce its class hierarchy or helper structure -- Provider directories own provider behavior. API formats and their public data contracts are independent of the provider that originated them -- Shared `base_llm` contracts must not import provider implementations or provider-specific transformation policy -- Config traits represent actual provider contracts. Use composition and existing helpers instead of recreating inheritance with unnecessary traits or delegation layers -- Providers choose authentication and header policy. Shared auth and HTTP infrastructure apply those decisions -- Generic configuration lookup belongs in the existing settings utilities, not in a provider directory -- Closures are idiomatic Rust, but a `Vec -> Vec` helper is not automatically a useful abstraction -- Choose traversal for the operation: per-block mapping, filtering, or whole-message processing when blocks depend on one another -- Add an abstraction only when it clarifies a repeated responsibility -- Verify observable auth precedence, headers, serialization, passthrough, and transformations, not code structure +- `AnthropicConfig`, `AmazonConverseConfig` and the non-OCR base traits are partial. Preserve their acceptance gates, passthrough and host fallback contracts +- `OpenAiResponsesApiConfig` implements WebSocket transformations and direct HTTP Responses, without Python's model-specific param rewriting or Responses-to-Chat emulation +- Chat `supported_openai_param_mappings` lists OpenAI/provider name pairs, unlike Python's name-only list. Audio `map_transcription_params` is a Rust filtering helper +- Auth resolution is split between configs and route preparation in `inference-` + +## OCR specifics + +- `BaseOcrConfig::prepare_request` matches Python's HTTP-handler preparation. `validate_request_body` is Rust-only +- `src/base_llm/ocr/error.rs` and `document.rs` are Rust-only: the OCR error taxonomy shared with the route and inline-document helpers +- Mistral auth key handling and top-level response-extra preservation differ from Python. Layout refactors must keep those behaviors and their tests + +## Code + +- Providers choose auth and header policy, shared auth and HTTP infrastructure apply it. Generic config lookup uses the existing settings utilities +- Pick traversal by operation (per-block map, filter, or whole-message when blocks depend on each other). Add an abstraction only for a repeated responsibility + +## Tests + +- Named `#[rstest]` cases instead of loops, `#[fixture]` for setup, `#[with(...)]` for overrides +- Verify observable auth precedence, headers, serialization, passthrough and transformations, never code structure diff --git a/litellm-rust/crates/llms/src/anthropic/AGENTS.md b/litellm-rust/crates/llms/src/anthropic/AGENTS.md index a226d4b56ce..e62c4a185bd 100644 --- a/litellm-rust/crates/llms/src/anthropic/AGENTS.md +++ b/litellm-rust/crates/llms/src/anthropic/AGENTS.md @@ -1,8 +1,11 @@ -- This directory owns Anthropic provider behavior: credentials, OAuth policy, endpoints, beta requirements, model capabilities, and transformations -- `common_utils.rs` means shared across Anthropic operations, not shared across providers -- Put behavior specific to the Messages API in `messages/` -- Keep generic HTTP mechanics in `litellm-http`, configuration lookup in the existing settings utilities, and credential application in the shared auth layer -- Choose authentication policy and required headers here, then let shared infrastructure apply those decisions -- Consume shared API contracts from `litellm-llms-types`. Do not define public Messages protocol types under this provider -- Preserve Python's concepts and observable behavior where useful, without mechanically reproducing its class hierarchy, helpers, or file structure -- `ReplayedWebSearchResult` and `ReplayedWebSearchContent` are private partial models for replay flattening, not complete public protocol contracts. Keep them private while they serve that transformation +# rules + +- Owns Anthropic provider policy: credentials, OAuth handling, endpoint resolution, beta requirements, model capabilities and transformations +- Paths, header names and default headers come from `litellm_llms_types::providers::anthropic` +- Choose auth and required headers here. Shared infrastructure applies them +- `ReplayedWebSearchResult` and `ReplayedWebSearchContent` are private partial models for replay flattening. Keep them private while they only serve that transformation + +# references + +- https://platform.claude.com/docs/en/api/overview.md +- https://platform.claude.com/docs/en/api/errors.md diff --git a/litellm-rust/crates/llms/src/anthropic/batches/AGENTS.md b/litellm-rust/crates/llms/src/anthropic/batches/AGENTS.md index c3a3c234492..9dd8fc09a85 100644 --- a/litellm-rust/crates/llms/src/anthropic/batches/AGENTS.md +++ b/litellm-rust/crates/llms/src/anthropic/batches/AGENTS.md @@ -1 +1,7 @@ +# rules + +- Maps Anthropic Message Batches onto the shared batch format. `AnthropicMessageBatch` and the result records are adapter-only projections + +# references + - https://platform.claude.com/docs/en/api/http/beta/messages/batches/create diff --git a/litellm-rust/crates/llms/src/anthropic/batches/transformation.rs b/litellm-rust/crates/llms/src/anthropic/batches/transformation.rs index 209fc7a0058..b8ea701af99 100644 --- a/litellm-rust/crates/llms/src/anthropic/batches/transformation.rs +++ b/litellm-rust/crates/llms/src/anthropic/batches/transformation.rs @@ -1,5 +1,6 @@ use litellm_llms_types::formats::batches::{BatchRequestCounts, BatchResponse, BatchStatus}; use litellm_llms_types::formats::messages::MessagesResponse; +use litellm_llms_types::providers::anthropic::{BATCHES_PATH, MESSAGES_PATH}; use serde::{Deserialize, Serialize}; use serde_json::Value; use time::OffsetDateTime; @@ -7,8 +8,6 @@ use url::Url; use crate::{Error, anthropic::common_utils::resolve_anthropic_api_base}; -const BATCHES_PATH_SUFFIX: &str = "/v1/messages/batches"; - #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct AnthropicBatchRequestCounts { #[serde(default)] @@ -106,12 +105,12 @@ fn batches_base_url( ) -> Result { let api_base = resolve_anthropic_api_base(api_base, env_lookup); let api_base = api_base.trim_end_matches('/'); - let complete_url = if api_base.ends_with(BATCHES_PATH_SUFFIX) { + let complete_url = if api_base.ends_with(BATCHES_PATH) { api_base.to_string() - } else if let Some(base) = api_base.strip_suffix("/v1/messages") { - format!("{base}{BATCHES_PATH_SUFFIX}") + } else if let Some(base) = api_base.strip_suffix(MESSAGES_PATH) { + format!("{base}{BATCHES_PATH}") } else { - format!("{api_base}{BATCHES_PATH_SUFFIX}") + format!("{api_base}{BATCHES_PATH}") }; Url::parse(&complete_url).map_err(|error| { Error::InvalidRequest(crate::ErrorDetail::invalid("Anthropic API base", error)) @@ -187,7 +186,7 @@ impl AnthropicBatchesConfig for AnthropicBatchesTransformation { BatchResponse { id: response.id.clone(), object: "batch".into(), - endpoint: "/v1/messages".into(), + endpoint: MESSAGES_PATH.into(), input_file_id: "None".into(), completion_window: "24h".into(), status, diff --git a/litellm-rust/crates/llms/src/anthropic/beta_headers.rs b/litellm-rust/crates/llms/src/anthropic/beta_headers.rs index 3b83b12d727..3c18bdcf437 100644 --- a/litellm-rust/crates/llms/src/anthropic/beta_headers.rs +++ b/litellm-rust/crates/llms/src/anthropic/beta_headers.rs @@ -1,10 +1,8 @@ use litellm_http::request::{with_header, without_headers}; -use litellm_llms_types::providers::anthropic::{BetaProvider, BetaSet}; +use litellm_llms_types::providers::anthropic::{BETA_HEADER, BetaProvider, BetaSet}; use crate::{anthropic::common_utils::existing_betas, base_llm::auth::Headers}; -const BETA_HEADER: &str = "anthropic-beta"; - /// How the `anthropic-beta` header is treated before a request leaves for a host. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum BetaPolicy { @@ -34,3 +32,121 @@ impl BetaPolicy { with_header(headers, BETA_HEADER, accepted.to_string()) } } + +#[cfg(test)] +mod tests { + use crate::anthropic::beta_headers::BetaPolicy; + use litellm_llms_types::providers::anthropic::BetaProvider; + use rstest::rstest; + + fn headers(pairs: &[(&str, &str)]) -> Vec<(String, String)> { + pairs + .iter() + .map(|(name, value)| (name.to_string(), value.to_string())) + .collect() + } + + #[rstest] + fn forward_keeps_headers_as_is() { + let input = headers(&[ + ( + "Anthropic-Beta", + "example-beta-2099-01-01,web-search-2025-03-05", + ), + ("x-api-key", "k"), + ]); + assert_eq!(BetaPolicy::Forward.apply(input.clone()), input); + } + + #[rstest] + fn drop_removes_every_casing() { + assert_eq!( + BetaPolicy::Drop.apply(headers(&[ + ("anthropic-beta", "a"), + ("x-api-key", "k"), + ("ANTHROPIC-BETA", "b"), + ])), + headers(&[("x-api-key", "k")]) + ); + } + + #[rstest] + #[case::bedrock(BetaProvider::Bedrock, Some("tool-search-tool-2025-10-19"))] + #[case::bedrock_mantle(BetaProvider::BedrockMantle, Some("tool-search-tool-2025-10-19"))] + #[case::vertex_ai(BetaProvider::VertexAi, Some("tool-search-tool-2025-10-19"))] + #[case::anthropic(BetaProvider::Anthropic, Some("advanced-tool-use-2025-11-20"))] + #[case::azure_ai(BetaProvider::AzureAi, Some("advanced-tool-use-2025-11-20"))] + #[case::databricks(BetaProvider::Databricks, Some("advanced-tool-use-2025-11-20"))] + #[case::bedrock_converse(BetaProvider::BedrockConverse, None)] + fn filter_renames_per_host( + #[case] provider: BetaProvider, + #[case] expected_beta: Option<&str>, + ) { + let expected = match expected_beta { + Some(beta) => headers(&[("x-api-key", "k"), ("anthropic-beta", beta)]), + None => headers(&[("x-api-key", "k")]), + }; + assert_eq!( + BetaPolicy::Filter(provider).apply(headers(&[ + ("Anthropic-Beta", "advanced-tool-use-2025-11-20"), + ("x-api-key", "k"), + ])), + expected + ); + } + + #[rstest] + #[case::azure_ai(BetaProvider::AzureAi, "fast-mode-2026-02-01,example-beta-2099-01-01")] + #[case::anthropic(BetaProvider::Anthropic, "bash_20241022")] + #[case::vertex_ai(BetaProvider::VertexAi, " , ")] + fn filter_removes_header_when_nothing_survives( + #[case] provider: BetaProvider, + #[case] beta_value: &str, + ) { + assert_eq!( + BetaPolicy::Filter(provider).apply(headers(&[ + ("x-api-key", "k"), + ("anthropic-beta", beta_value) + ])), + headers(&[("x-api-key", "k")]) + ); + } + + #[rstest] + #[case::anthropic( + BetaProvider::Anthropic, + "web-search-2025-03-05", + "oauth-2025-04-20,web-search-2025-03-05,example-beta-2099-01-01", + "oauth-2025-04-20,web-search-2025-03-05" + )] + #[case::vertex_ai_renames_and_deduplicates( + BetaProvider::VertexAi, + "advanced-tool-use-2025-11-20", + "tool-search-tool-2025-10-19", + "tool-search-tool-2025-10-19" + )] + fn filter_deduplicates_and_sorts( + #[case] provider: BetaProvider, + #[case] first_beta: &str, + #[case] second_beta: &str, + #[case] expected_beta: &str, + ) { + assert_eq!( + BetaPolicy::Filter(provider).apply(headers(&[ + ("ANTHROPIC-BETA", first_beta), + ("x-api-key", "k"), + ("anthropic-beta", second_beta), + ])), + headers(&[("x-api-key", "k"), ("anthropic-beta", expected_beta)]) + ); + } + + #[rstest] + #[case::forward(BetaPolicy::Forward)] + #[case::drop(BetaPolicy::Drop)] + #[case::filter(BetaPolicy::Filter(BetaProvider::Bedrock))] + fn headers_without_a_beta_are_untouched(#[case] policy: BetaPolicy) { + let input = headers(&[("x-api-key", "k"), ("anthropic-version", "2023")]); + assert_eq!(policy.apply(input.clone()), input); + } +} diff --git a/litellm-rust/crates/llms/src/anthropic/chat/AGENTS.md b/litellm-rust/crates/llms/src/anthropic/chat/AGENTS.md new file mode 100644 index 00000000000..dab8abcc5ad --- /dev/null +++ b/litellm-rust/crates/llms/src/anthropic/chat/AGENTS.md @@ -0,0 +1,4 @@ +# rules + +- Chat Completions translated onto Anthropic Messages. Both format specs live under `llms-types/src/formats/` +- `top_k` is not forwarded, because Python gates it per model inside `transform_request` diff --git a/litellm-rust/crates/llms/src/anthropic/chat/transformation.rs b/litellm-rust/crates/llms/src/anthropic/chat/transformation.rs index e1fd1956167..ce759f6299e 100644 --- a/litellm-rust/crates/llms/src/anthropic/chat/transformation.rs +++ b/litellm-rust/crates/llms/src/anthropic/chat/transformation.rs @@ -3,8 +3,11 @@ use litellm_core_utils::{ core_helpers::{finish_reason_for, unix_now, usage_from_parts}, prompt_templates::factory::{Conversation, build_conversation}, }; -use litellm_llms_types::formats::chat_completions::{ - ChatCompletionsChoice, ChatCompletionsChoiceMessage, ChatCompletionsResponse, ChatMessage, +use litellm_llms_types::{ + formats::chat_completions::{ + ChatCompletionsChoice, ChatCompletionsChoiceMessage, ChatCompletionsResponse, ChatMessage, + }, + providers::anthropic::DEFAULT_HEADERS, }; use serde::Deserialize; use serde_json::{Map, Value, json}; @@ -199,10 +202,7 @@ impl BaseConfig for AnthropicConfig { } fn default_headers(&self) -> &'static [(&'static str, &'static str)] { - &[ - ("anthropic-version", "2023-06-01"), - ("content-type", "application/json"), - ] + DEFAULT_HEADERS } /// An OAuth bearer is the whole credential: Python's `validate_environment` @@ -263,3 +263,462 @@ fn anthropic_body( ); Value::Object(body) } + +#[cfg(test)] +mod tests { + use crate::{ + Error, + anthropic::chat::transformation::ANTHROPIC_CHAT_COMPLETIONS_CONFIG, + base_llm::{ + auth::AuthScheme, + chat::transformation::{BaseConfig, ProviderChatResponseData, Unsupported}, + }, + }; + use litellm_llms_types::formats::chat_completions::{ChatCompletionsResponse, ChatMessage}; + use rstest::rstest; + use serde_json::{Map, Value, json}; + + fn messages(value: Value) -> Vec { + serde_json::from_value(value).expect("valid messages") + } + + fn params(value: Value) -> Map { + match value { + Value::Object(map) => map, + other => panic!("params must be an object, got {other}"), + } + } + + fn transform(model: &str, msgs: Value, opts: Value) -> Value { + ANTHROPIC_CHAT_COMPLETIONS_CONFIG + .transform_request(model, messages(msgs), params(opts)) + .expect("request transforms") + .body + } + + fn transform_response(body: Value) -> Result { + ANTHROPIC_CHAT_COMPLETIONS_CONFIG + .transform_response("claude-sonnet-4-5", ProviderChatResponseData { body }) + } + + fn reason(msgs: Value, opts: Value) -> Option { + ANTHROPIC_CHAT_COMPLETIONS_CONFIG.unsupported_reason(&messages(msgs), ¶ms(opts)) + } + + #[test] + fn builds_the_messages_body_python_builds() { + let body = transform( + "claude-sonnet-4-5", + json!([ + {"role": "system", "content": "be terse"}, + {"role": "user", "content": "hi"} + ]), + json!({"max_tokens": 128, "temperature": 0.2}), + ); + assert_eq!( + body, + json!({ + "model": "claude-sonnet-4-5", + "messages": [ + {"role": "user", "content": [{"type": "text", "text": "hi"}]} + ], + "system": [{"type": "text", "text": "be terse"}], + "max_tokens": 128, + "temperature": 0.2 + }) + ); + } + + #[test] + fn omits_system_when_no_system_message_is_present() { + let body = transform( + "claude-sonnet-4-5", + json!([{"role": "user", "content": "hi"}]), + json!({"max_tokens": 16}), + ); + assert!(body.get("system").is_none()); + } + + #[test] + fn merges_consecutive_turns_and_wraps_every_text_in_a_block() { + let body = transform( + "claude-sonnet-4-5", + json!([ + {"role": "user", "content": "one"}, + {"role": "user", "content": [{"type": "text", "text": "two"}]}, + {"role": "assistant", "content": "ack"} + ]), + json!({"max_tokens": 16}), + ); + assert_eq!( + body["messages"], + json!([ + {"role": "user", "content": [ + {"type": "text", "text": "one"}, + {"type": "text", "text": "two"} + ]}, + {"role": "assistant", "content": [{"type": "text", "text": "ack"}]} + ]) + ); + } + + #[test] + fn right_strips_a_trailing_assistant_prefill_like_python() { + let body = transform( + "claude-sonnet-4-5", + json!([ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "Argentina "} + ]), + json!({"max_tokens": 16}), + ); + assert_eq!( + body["messages"][1]["content"][0]["text"], + json!("Argentina") + ); + } + + #[test] + fn passes_every_supported_param_through_untouched() { + let body = transform( + "claude-sonnet-4-5", + json!([{"role": "user", "content": "hi"}]), + json!({ + "max_tokens": 64, + "temperature": 0.1, + "top_p": 0.9, + "stop_sequences": ["STOP"] + }), + ); + assert_eq!(body["max_tokens"], json!(64)); + assert_eq!(body["temperature"], json!(0.1)); + assert_eq!(body["top_p"], json!(0.9)); + assert_eq!(body["stop_sequences"], json!(["STOP"])); + } + + #[test] + fn declines_top_k_because_python_gates_it_by_model_below_this_point() { + // `temperature` and `top_p` arrive already resolved, because + // `map_openai_params` applies `_apply_sampling_param` to them before the + // gate runs. `top_k` bypasses that and is gated inside `transform_request`, + // the function this route replaces, so forwarding it would send `top_k` to + // a model that removed sampling params and take a 400 after the call, where + // Python drops it and succeeds. + assert_eq!( + reason( + json!([{"role": "user", "content": "hi"}]), + json!({"top_k": 40}) + ), + Some(Unsupported("unrecognized request parameter")) + ); + } + + #[test] + fn declines_streaming_before_anything_else() { + assert_eq!( + reason( + json!([{"role": "user", "content": "hi"}]), + json!({"stream": true, "max_tokens": 16}) + ), + Some(Unsupported("streaming")) + ); + } + + #[test] + fn accepts_an_explicit_stream_false() { + assert_eq!( + reason( + json!([{"role": "user", "content": "hi"}]), + json!({"stream": false, "max_tokens": 16}) + ), + None + ); + } + + #[rstest] + #[case::tools(json!({"tools": []}))] + #[case::tool_choice(json!({"tool_choice": {"type": "auto"}}))] + #[case::thinking(json!({"thinking": {"type": "enabled"}}))] + #[case::system(json!({"system": "injected"}))] + #[case::metadata(json!({"metadata": {"user_id": "u1"}}))] + #[case::output_config(json!({"output_config": {"effort": "high"}}))] + fn declines_any_param_outside_the_allowlist(#[case] param: Value) { + assert_eq!( + reason(json!([{"role": "user", "content": "hi"}]), param.clone()), + Some(Unsupported("unrecognized request parameter")), + "expected {param} to decline" + ); + } + + #[test] + fn declines_tool_calls_tool_results_and_multimodal_content() { + assert_eq!( + reason( + json!([ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": null, "tool_calls": [ + {"id": "c1", "type": "function", + "function": {"name": "f", "arguments": "{}"}} + ]} + ]), + json!({}) + ), + Some(Unsupported("unrecognized message field")) + ); + assert_eq!( + reason( + json!([ + {"role": "user", "content": "hi"}, + {"role": "tool", "tool_call_id": "c1", "content": "ok"} + ]), + json!({}) + ), + Some(Unsupported("unrecognized message field")) + ); + assert_eq!( + reason( + json!([ + {"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": "https://x/y.png"}} + ]}]), + json!({}) + ), + Some(Unsupported("non-text message content")) + ); + assert_eq!( + reason( + json!([ + {"role": "user", "content": [ + {"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}} + ]}]), + json!({}) + ), + Some(Unsupported("non-text message content")) + ); + } + + #[test] + fn declines_a_message_whose_content_list_is_empty() { + // An empty list passes every per-part check, so without this it would reach + // the provider as an empty `content` array and fail after the call rather + // than declining to Python before it. + assert_eq!( + reason(json!([{"role": "user", "content": []}]), json!({})), + Some(Unsupported("message without content")) + ); + assert_eq!( + reason( + json!([{"role": "user", "content": [{"type": "text", "text": "hi"}]}]), + json!({}) + ), + None + ); + } + + #[test] + fn declines_a_conversation_that_does_not_open_on_a_user_turn() { + assert_eq!( + reason( + json!([ + {"role": "system", "content": "be terse"}, + {"role": "assistant", "content": "prefill"} + ]), + json!({}) + ), + Some(Unsupported("conversation does not open on a user turn")) + ); + } + + #[test] + fn accepts_a_plain_text_conversation() { + assert_eq!( + reason( + json!([ + {"role": "system", "content": "be terse"}, + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + {"role": "user", "content": [{"type": "text", "text": "again"}]} + ]), + json!({"max_tokens": 16, "temperature": 0.5}) + ), + None + ); + } + + #[test] + fn normalizes_a_text_response_into_openai_shape() { + let response = transform_response(json!({ + "id": "msg_123", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5-20260101", + "content": [{"type": "text", "text": "hello"}, {"type": "text", "text": " there"}], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": {"input_tokens": 11, "output_tokens": 4} + })) + .expect("response transforms"); + + assert_eq!(response.model, "claude-sonnet-4-5-20260101"); + assert_eq!(response.choices.len(), 1); + assert_eq!(response.choices[0].index, 0); + assert_eq!(response.choices[0].message.role, "assistant"); + assert_eq!( + response.choices[0].message.content.as_deref(), + Some("hello there") + ); + assert_eq!(response.choices[0].finish_reason, "stop"); + assert_eq!(response.usage.prompt_tokens, 11); + assert_eq!(response.usage.completion_tokens, 4); + assert_eq!(response.usage.total_tokens, 15); + } + + #[test] + fn folds_cache_tokens_into_prompt_tokens_like_python() { + let response = transform_response(json!({ + "model": "claude-sonnet-4-5", + "content": [{"type": "text", "text": "hi"}], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 10, + "output_tokens": 2, + "cache_read_input_tokens": 5, + "cache_creation_input_tokens": 3 + } + })) + .expect("response transforms"); + assert_eq!(response.usage.prompt_tokens, 18); + assert_eq!(response.usage.total_tokens, 20); + assert_eq!(response.usage.prompt_tokens_details.cached_tokens, 5); + assert_eq!( + response.usage.prompt_tokens_details.cache_creation_tokens, + 3 + ); + assert_eq!(response.usage.prompt_tokens_details.text_tokens, 10); + } + + #[test] + fn maps_max_tokens_stop_reason_to_length() { + let response = transform_response(json!({ + "model": "claude-sonnet-4-5", + "content": [{"type": "text", "text": "hi"}], + "stop_reason": "max_tokens", + "usage": {"input_tokens": 1, "output_tokens": 1} + })) + .expect("response transforms"); + assert_eq!(response.choices[0].finish_reason, "length"); + } + + #[test] + fn a_refusal_returns_the_completion_python_returns() { + // `refusal` is a stop_reason, not a content block type, so the content is + // ordinary text and this normalizes rather than declining. Python maps it + // to content_filter in _FINISH_REASON_MAP and returns the completion. + let response = transform_response(json!({ + "model": "claude-sonnet-4-5", + "content": [{"type": "text", "text": "I can't help with that."}], + "stop_reason": "refusal", + "usage": {"input_tokens": 9, "output_tokens": 6} + })) + .expect("a refusal still transforms"); + assert_eq!(response.choices[0].finish_reason, "content_filter"); + assert_eq!( + response.choices[0].message.content.as_deref(), + Some("I can't help with that.") + ); + } + + #[test] + fn reports_no_content_rather_than_an_empty_string() { + let response = transform_response(json!({ + "model": "claude-sonnet-4-5", + "content": [], + "stop_reason": "end_turn", + "usage": {"input_tokens": 1, "output_tokens": 0} + })) + .expect("response transforms"); + assert_eq!(response.choices[0].message.content, None); + } + + #[test] + fn response_carries_no_id_so_python_keeps_its_chatcmpl_id() { + let response = transform_response(json!({ + "id": "msg_should_not_leak", + "model": "claude-sonnet-4-5", + "content": [{"type": "text", "text": "hi"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 1, "output_tokens": 1} + })) + .expect("response transforms"); + let value = serde_json::to_value(response).expect("serializable"); + assert!( + value.get("id").is_none(), + "the rust response must not carry an id, got {value}" + ); + } + + #[test] + fn declines_a_response_carrying_a_non_text_block() { + let err = transform_response(json!({ + "model": "claude-sonnet-4-5", + "content": [{"type": "tool_use", "id": "t1", "name": "f", "input": {}}], + "stop_reason": "tool_use", + "usage": {"input_tokens": 1, "output_tokens": 1} + })) + .expect_err("non-text block"); + assert_eq!(err, Error::Unsupported("non-text response content block")); + } + + #[rstest::rstest] + #[case::not_an_object(json!("nope"))] + #[case::missing_content(json!({"model": "test-model", "usage": {"input_tokens": 1, "output_tokens": 1}}))] + #[case::missing_usage(json!({"model": "test-model", "content": []}))] + #[case::missing_model(json!({"content": [], "usage": {"input_tokens": 1, "output_tokens": 1}}))] + fn errors_on_a_response_missing_required_fields(#[case] body: Value) { + let error = transform_response(body).expect_err("invalid response"); + assert!(matches!(error, Error::InvalidResponse(_))); + let source = std::iter::successors(Some(&error as &dyn std::error::Error), |error| { + error.source() + }) + .find_map(|error| error.downcast_ref::()) + .expect("the JSON decoding source is preserved"); + assert_eq!( + error.to_string(), + format!("invalid response: invalid messages response: {source}") + ); + } + + #[test] + fn resolves_the_messages_url_and_x_api_key_auth() { + let config = &ANTHROPIC_CHAT_COMPLETIONS_CONFIG; + assert_eq!( + config + .get_complete_url(None, "claude-sonnet-4-5", &Map::new(), &|_| None) + .expect("url builds"), + "https://api.anthropic.com/v1/messages" + ); + let validated = config + .validate_environment( + Vec::new(), + Some("sk-x"), + "claude-sonnet-4-5", + &Map::new(), + &|_| None, + ) + .expect("auth resolves"); + assert!(matches!( + validated.auth, + AuthScheme::Credential { + placement: litellm_auth::CredentialPlacement::Header("x-api-key"), + ref secret + } if secret.expose() == "sk-x" + )); + assert_eq!( + config.default_headers(), + &[ + ("anthropic-version", "2023-06-01"), + ("content-type", "application/json"), + ] + ); + } +} diff --git a/litellm-rust/crates/llms/src/anthropic/common_utils.rs b/litellm-rust/crates/llms/src/anthropic/common_utils.rs index 335731fa001..7991a57353e 100644 --- a/litellm-rust/crates/llms/src/anthropic/common_utils.rs +++ b/litellm-rust/crates/llms/src/anthropic/common_utils.rs @@ -9,13 +9,15 @@ use litellm_llms_types::{ ContentBlock, ContentBlockType, EffortLevel, Message, MessageContent, MessagesTool, SystemPrompt, }, - providers::anthropic::{AnthropicBeta, BetaSet}, + providers::anthropic::{ + API_BASE, API_KEY_HEADER, AnthropicBeta, BETA_HEADER, BetaSet, + DIRECT_BROWSER_ACCESS_HEADER, MESSAGES_PATH, + }, recognized::Recognized, }; use serde::Deserialize; use serde_json::Value; -use crate::base_llm::messages::transformation::MESSAGES_PATH_SUFFIX; use crate::{ anthropic::ANTHROPIC_OAUTH_TOKEN_PREFIX, base_llm::auth::{AuthScheme, Headers}, @@ -25,14 +27,10 @@ pub const ANTHROPIC_API_KEY_ENV: &str = "ANTHROPIC_API_KEY"; pub const ANTHROPIC_AUTH_TOKEN_ENV: &str = "ANTHROPIC_AUTH_TOKEN"; pub const ENCRYPTED_REASONING_SIGNATURE_PREFIX: &str = "litellm_encrypted_reasoning:"; const THOUGHT_SIGNATURE_SEPARATOR: &str = "__thought__"; -const BETA_HEADER: &str = "anthropic-beta"; pub const ANTHROPIC_API_BASE_ENV: &str = "ANTHROPIC_API_BASE"; pub const ANTHROPIC_BASE_URL_ENV: &str = "ANTHROPIC_BASE_URL"; -pub const DEFAULT_ANTHROPIC_API_BASE: &str = "https://api.anthropic.com"; -pub const API_KEY_PLACEMENT: CredentialPlacement = CredentialPlacement::Header("x-api-key"); -const API_KEY_HEADER: &str = API_KEY_PLACEMENT.header_name(); +pub const API_KEY_PLACEMENT: CredentialPlacement = CredentialPlacement::Header(API_KEY_HEADER); const AUTHORIZATION: &str = CredentialPlacement::Bearer.header_name(); -const DIRECT_BROWSER_ACCESS_HEADER: &str = "anthropic-dangerous-direct-browser-access"; pub fn supports_effort_tier(capabilities: &MessagesModelCapabilities, level: EffortLevel) -> bool { match level { @@ -145,7 +143,7 @@ pub fn resolve_anthropic_api_base( env_lookup, &[ANTHROPIC_API_BASE_ENV, ANTHROPIC_BASE_URL_ENV], ) - .unwrap_or_else(|| DEFAULT_ANTHROPIC_API_BASE.to_string()) + .unwrap_or_else(|| API_BASE.to_string()) } pub fn complete_anthropic_url( @@ -155,10 +153,10 @@ pub fn complete_anthropic_url( let api_base = resolve_anthropic_api_base(api_base, env_lookup); let api_base = api_base.trim_end_matches('/'); - if api_base.ends_with(MESSAGES_PATH_SUFFIX) { + if api_base.ends_with(MESSAGES_PATH) { return api_base.to_string(); } - format!("{api_base}{MESSAGES_PATH_SUFFIX}") + format!("{api_base}{MESSAGES_PATH}") } pub fn existing_betas(headers: &[(String, String)]) -> BetaSet { diff --git a/litellm-rust/crates/llms/src/anthropic/count_tokens/AGENTS.md b/litellm-rust/crates/llms/src/anthropic/count_tokens/AGENTS.md index f08c0c6d017..6d45f2ae14d 100644 --- a/litellm-rust/crates/llms/src/anthropic/count_tokens/AGENTS.md +++ b/litellm-rust/crates/llms/src/anthropic/count_tokens/AGENTS.md @@ -1 +1,7 @@ +# rules + +- Builds the count-tokens request against the public Anthropic host. The request and response structs are adapter-only projections + +# references + - https://platform.claude.com/docs/en/api/http/messages/count_tokens diff --git a/litellm-rust/crates/llms/src/anthropic/count_tokens/transformation.rs b/litellm-rust/crates/llms/src/anthropic/count_tokens/transformation.rs index 4d892ca198c..9b2ee44516c 100644 --- a/litellm-rust/crates/llms/src/anthropic/count_tokens/transformation.rs +++ b/litellm-rust/crates/llms/src/anthropic/count_tokens/transformation.rs @@ -1,10 +1,14 @@ -use litellm_llms_types::formats::messages::{Message, SystemPrompt}; +use litellm_llms_types::{ + formats::messages::{Message, SystemPrompt}, + providers::anthropic::{ + API_BASE, API_KEY_HEADER, API_VERSION, BETA_HEADER, COUNT_TOKENS_PATH, VERSION_HEADER, + }, +}; use serde::{Deserialize, Serialize}; use serde_json::Value; use crate::{Error, anthropic::ANTHROPIC_OAUTH_TOKEN_PREFIX}; -const COUNT_TOKENS_ENDPOINT: &str = "https://api.anthropic.com/v1/messages/count_tokens"; const TOKEN_COUNTING_BETA: &str = "token-counting-2024-11-01"; #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] @@ -23,7 +27,7 @@ pub struct AnthropicCountTokensResponse { } pub trait AnthropicCountTokensConfig { - fn endpoint(&self) -> &'static str; + fn endpoint(&self) -> String; fn validate_request(&self, model: &str, messages: &[Message]) -> Result<(), Error>; @@ -44,8 +48,8 @@ pub const ANTHROPIC_COUNT_TOKENS_TRANSFORMATION: AnthropicCountTokensTransformat AnthropicCountTokensTransformation; impl AnthropicCountTokensConfig for AnthropicCountTokensTransformation { - fn endpoint(&self) -> &'static str { - COUNT_TOKENS_ENDPOINT + fn endpoint(&self) -> String { + format!("{API_BASE}{COUNT_TOKENS_PATH}") } fn transform_request( @@ -79,13 +83,13 @@ impl AnthropicCountTokensConfig for AnthropicCountTokensTransformation { let auth = if api_key.starts_with(ANTHROPIC_OAUTH_TOKEN_PREFIX) { ("authorization", format!("Bearer {api_key}")) } else { - ("x-api-key", api_key.to_string()) + (API_KEY_HEADER, api_key.to_string()) }; vec![ ("content-type", "application/json".to_string()), auth, - ("anthropic-version", "2023-06-01".to_string()), - ("anthropic-beta", TOKEN_COUNTING_BETA.to_string()), + (VERSION_HEADER, API_VERSION.to_string()), + (BETA_HEADER, TOKEN_COUNTING_BETA.to_string()), ] } } @@ -127,7 +131,7 @@ mod tests { ); assert_eq!( ANTHROPIC_COUNT_TOKENS_TRANSFORMATION.endpoint(), - COUNT_TOKENS_ENDPOINT + "https://api.anthropic.com/v1/messages/count_tokens" ); } diff --git a/litellm-rust/crates/llms/src/anthropic/messages/AGENTS.md b/litellm-rust/crates/llms/src/anthropic/messages/AGENTS.md index 48c8c342c80..05c3c9b61d8 100644 --- a/litellm-rust/crates/llms/src/anthropic/messages/AGENTS.md +++ b/litellm-rust/crates/llms/src/anthropic/messages/AGENTS.md @@ -1,9 +1,5 @@ -This directory owns Anthropic's implementation of the Messages adapter contract in `base_llm/messages`. Shared Messages API data contracts belong in `litellm-llms-types::formats::messages`, and call orchestration belongs in `inference-messages`. Sharing the `llms` crate with `base_llm/messages` does not erase this boundary +# rules -Payload shaping, metadata filtering, tool-ID rewriting, web-search replay handling, thinking translation, and beta selection are provider policy. Keep them here or in Anthropic helpers shared by its operations. Pure payload shaping belongs with transformations, even if an existing file is named `handler.rs` - -Bedrock and Azure adapters may explicitly reuse these helpers where Anthropic policy applies to their Claude backend. That reuse does not make the policy part of the shared Messages contract or a default for every provider. Shared `base_llm` code must never depend on this implementation - -`web_search_result`, `web_search_tool_result_error`, and encrypted-content fields are protocol data owned by `litellm-llms-types`. Keep those schemas separate from decisions about flattening, encrypted results, beta requirements, and model capabilities - -Protocol reference: [Messages API](https://platform.claude.com/docs/en/api/http/messages/create) +- Format spec lives in `llms-types/src/formats/messages/AGENTS.md` +- Payload shaping, metadata filtering, tool-ID rewriting, web-search replay, thinking translation and beta selection live here or in Anthropic-wide helpers +- Bedrock, Vertex, Azure and DeepSeek reuse these helpers for their Claude-compatible hosts. Keep the helpers free of those hosts' differences diff --git a/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs b/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs index f83b477471b..aa926662b7a 100644 --- a/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs @@ -3,7 +3,7 @@ use litellm_llms_types::{ formats::messages::{ ContextEdit, ContextManagement, Message, MessagesOptionalParams, MessagesRequest, Speed, }, - providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet}, + providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet, DEFAULT_HEADERS}, recognized::Recognized, }; use litellm_router_types::LitellmParams; @@ -26,11 +26,6 @@ use crate::{ }, }; -pub(crate) const DEFAULT_HEADERS: &[(&str, &str)] = &[ - ("anthropic-version", "2023-06-01"), - ("content-type", "application/json"), -]; - pub struct AnthropicMessagesConfig; pub const ANTHROPIC_MESSAGES_CONFIG: AnthropicMessagesConfig = AnthropicMessagesConfig; diff --git a/litellm-rust/crates/llms/src/aws_textract/ocr/AGENTS.md b/litellm-rust/crates/llms/src/aws_textract/ocr/AGENTS.md index 4913f89924a..ca851461d87 100644 --- a/litellm-rust/crates/llms/src/aws_textract/ocr/AGENTS.md +++ b/litellm-rust/crates/llms/src/aws_textract/ocr/AGENTS.md @@ -1,3 +1,9 @@ +# rules + +- Translates Textract DetectDocumentText and AnalyzeDocument output into the shared OCR format. Only synchronous single-page calls are supported + +# references + - https://docs.aws.amazon.com/textract/latest/APIReference/Welcome.md - https://docs.aws.amazon.com/textract/latest/APIReference/API_Operations.md - https://docs.aws.amazon.com/textract/latest/APIReference/API_DetectDocumentText.md diff --git a/litellm-rust/crates/llms/src/azure_ai/messages/AGENTS.md b/litellm-rust/crates/llms/src/azure_ai/messages/AGENTS.md index 2e25c26286d..e1f270378e0 100644 --- a/litellm-rust/crates/llms/src/azure_ai/messages/AGENTS.md +++ b/litellm-rust/crates/llms/src/azure_ai/messages/AGENTS.md @@ -1,3 +1,7 @@ -This directory owns Azure's Messages adapter: its endpoints, authentication policy, headers, and transformations. Implement the shared adapter contract from `base_llm/messages`, consume API data contracts from `litellm-llms-types::formats::messages`, and leave call orchestration to `inference-messages` +# rules -The Claude adapter may explicitly reuse payload policy from `anthropic/messages` when it applies to Azure's Claude backend. Keep Azure-specific differences here. Sharing that helper does not make Anthropic policy a format-wide default or justify a dependency from `base_llm/messages` on provider implementations +- Claude on Azure AI Foundry: reuses `anthropic/messages` shaping with the Azure `/anthropic` path, Azure credential env vars, and `x-api-key` or bearer auth + +# references + +- https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/how-to/use-foundry-models-claude diff --git a/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs b/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs index 3e6abf2d14e..9eaaa657f6f 100644 --- a/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs @@ -1,14 +1,18 @@ -use litellm_auth::{CredentialPlacement, SecretValue}; +use litellm_auth::SecretValue; use litellm_http::request::{has_bearer_auth, has_header}; -use litellm_llms_types::formats::messages::MessagesRequest; +use litellm_llms_types::{ + formats::messages::MessagesRequest, + providers::anthropic::{DEFAULT_HEADERS, MESSAGES_PATH}, +}; use litellm_router_types::LitellmParams; use crate::{ Error, - anthropic::messages::{ - handler::shape_anthropic_messages_request, - transformation::{ - DEFAULT_HEADERS, transform_messages_request, update_headers_with_anthropic_beta, + anthropic::{ + common_utils::API_KEY_PLACEMENT, + messages::{ + handler::shape_anthropic_messages_request, + transformation::{transform_messages_request, update_headers_with_anthropic_beta}, }, }, azure_ai::common_utils::{ @@ -19,12 +23,11 @@ use crate::{ messages::{ context::MessagesTransformContext, normalization::{normalize_system_role_messages, strip_cache_control_scope}, - transformation::{BaseMessagesConfig, MESSAGES_PATH_SUFFIX}, + transformation::BaseMessagesConfig, }, }, }; -const API_KEY_PLACEMENT: CredentialPlacement = CredentialPlacement::Header("x-api-key"); const ANTHROPIC_PATH_SEGMENT: &str = "/anthropic"; pub struct AzureAnthropicMessagesConfig; @@ -113,7 +116,7 @@ pub fn complete_azure_anthropic_url( let api_base = api_base.trim_end_matches('/'); - if api_base.ends_with(MESSAGES_PATH_SUFFIX) { + if api_base.ends_with(MESSAGES_PATH) { return Ok(api_base.to_string()); } @@ -121,7 +124,7 @@ pub fn complete_azure_anthropic_url( Some((prefix, _)) => format!("{prefix}{ANTHROPIC_PATH_SEGMENT}"), None => format!("{api_base}{ANTHROPIC_PATH_SEGMENT}"), }; - Ok(format!("{with_anthropic}{MESSAGES_PATH_SUFFIX}")) + Ok(format!("{with_anthropic}{MESSAGES_PATH}")) } #[cfg(test)] diff --git a/litellm-rust/crates/llms/src/azure_ai/ocr/AGENTS.md b/litellm-rust/crates/llms/src/azure_ai/ocr/AGENTS.md new file mode 100644 index 00000000000..413c85a054b --- /dev/null +++ b/litellm-rust/crates/llms/src/azure_ai/ocr/AGENTS.md @@ -0,0 +1,9 @@ +# rules + +- `transformation.rs` sends the Mistral OCR request to Azure AI's `/providers/mistral/azure/ocr` path +- `cohere_parse_transformation.rs` reuses `cohere/ocr` against Azure AI's `/providers/cohere/v2/parse` path +- `document_intelligence/` translates Azure Document Intelligence's async analyze operation into the shared OCR format + +# references + +- https://learn.microsoft.com/en-us/rest/api/aiservices/document-models/analyze-document diff --git a/litellm-rust/crates/llms/src/base_llm/messages/AGENTS.md b/litellm-rust/crates/llms/src/base_llm/messages/AGENTS.md index e7a00c3bb47..eb1d798af49 100644 --- a/litellm-rust/crates/llms/src/base_llm/messages/AGENTS.md +++ b/litellm-rust/crates/llms/src/base_llm/messages/AGENTS.md @@ -1,5 +1,7 @@ -This directory owns the shared Messages provider adapter contract, its execution inputs such as `MessagesTransformContext`, and provider-independent transformation machinery. Public request, response, content-block, and event schemas belong in `litellm-llms-types::formats::messages`. Call orchestration belongs in `inference-messages`, and provider implementations belong in `llms/src//messages` +# rules -Do not import provider implementations or embed their policy in shared trait defaults, normalization, or context defaults. A context carries inputs the shared adapter contract needs, not every provider's settings. Thinking-budget choices and model-specific restrictions do not become format rules merely because several providers host Claude - -Shared normalization must implement LiteLLM's provider-independent Messages input contract. Provider-specific metadata filtering, tool-ID rewriting, web-search replay policy, beta selection, and thinking translation belong in the provider implementation. Let each adapter explicitly opt into applicable shared provider helpers +- Owns the Messages adapter contract, its execution inputs such as `MessagesTransformContext`, and provider-independent machinery +- A context carries only inputs the contract needs, not every provider's settings +- Shared normalization implements LiteLLM's provider-independent Messages input contract +- Metadata filtering, tool-ID rewriting, web-search replay, beta selection, thinking translation and thinking budgets are provider policy. Adapters opt into shared provider helpers explicitly +- Restrictions common to Claude hosts are still provider policy, not format rules diff --git a/litellm-rust/crates/llms/src/base_llm/messages/normalization.rs b/litellm-rust/crates/llms/src/base_llm/messages/normalization.rs index 3ae4145ae32..7e24da2b5eb 100644 --- a/litellm-rust/crates/llms/src/base_llm/messages/normalization.rs +++ b/litellm-rust/crates/llms/src/base_llm/messages/normalization.rs @@ -103,3 +103,61 @@ pub fn strip_cache_control_scope(request: MessagesRequest) -> MessagesRequest { ..request } } + +#[cfg(test)] +mod tests { + use crate::base_llm::messages::normalization::normalize_system_role_messages; + use litellm_llms_types::formats::messages::MessagesRequest; + use rstest::rstest; + use serde_json::{Value, json}; + + #[rstest] + #[case::text_system(json!("existing"), json!([{"type": "text", "text": "existing"}]))] + #[case::block_system(json!([{ "type": "future", "payload": 7 }]), json!([{ "type": "future", "payload": 7 }]))] + #[case::no_system(Value::Null, json!([]))] + fn hoisting_preserves_block_fields_order_and_unrelated_request_fields( + #[case] system: Value, + #[case] initial_blocks: Value, + ) { + let cache_control = json!({"type": "ephemeral", "scope": "global", "future": true}); + let leading_block = + json!({"type": "text", "text": "second", "cache_control": cache_control}); + let user = json!({"role": "user", "content": "hello", "future_message": 42}); + let later_turn = json!({"role": "system", "content": "mid-conversation reminder"}); + let request: MessagesRequest = serde_json::from_value(json!({ + "model": "test-model", + "max_tokens": 64, + "system": system, + "messages": [ + {"role": "system", "content": "first"}, + {"role": "system", "content": [leading_block]}, + user, + later_turn + ], + "future_request": {"nested": true} + })) + .unwrap(); + let normalized = normalize_system_role_messages(request, true); + let expected_blocks: Vec = initial_blocks + .as_array() + .unwrap() + .iter() + .cloned() + .chain([json!({"type": "text", "text": "first"}), leading_block]) + .collect(); + assert_eq!( + serde_json::to_value(&normalized).unwrap(), + json!({ + "model": "test-model", + "max_tokens": 64, + "system": expected_blocks, + "messages": [user, later_turn], + "future_request": {"nested": true} + }) + ); + assert_eq!( + normalize_system_role_messages(normalized.clone(), true), + normalized + ); + } +} diff --git a/litellm-rust/crates/llms/src/base_llm/messages/transformation.rs b/litellm-rust/crates/llms/src/base_llm/messages/transformation.rs index f42adcdcfe6..61398fe4ea9 100644 --- a/litellm-rust/crates/llms/src/base_llm/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/base_llm/messages/transformation.rs @@ -7,8 +7,6 @@ use super::context::MessagesTransformContext; pub use crate::base_llm::auth::{Headers, ValidatedEnvironment}; use crate::{Error, base_llm::messages::streaming::StreamDecoder}; -pub const MESSAGES_PATH_SUFFIX: &str = "/v1/messages"; - pub trait BaseMessagesConfig: Sync { fn shape_request( &self, diff --git a/litellm-rust/crates/llms/src/base_llm/ocr/handler.rs b/litellm-rust/crates/llms/src/base_llm/ocr/handler.rs index a4c2d465cbd..2cf955ff0c6 100644 --- a/litellm-rust/crates/llms/src/base_llm/ocr/handler.rs +++ b/litellm-rust/crates/llms/src/base_llm/ocr/handler.rs @@ -303,7 +303,13 @@ pub fn body_document(body: &Value) -> Result { #[cfg(test)] mod tests { + use crate::base_llm::ocr::{error::Error, handler::read_response_bytes}; + use rstest::rstest; use std::time::Duration; + use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + }; use super::*; @@ -328,4 +334,78 @@ mod tests { )); server.abort(); } + + /// Answers one request with raw `response` bytes and then holds the connection open, so a + /// read that waits for the rest of an oversized body hangs instead of passing. + async fn read_bounded(response: String, limit: usize) -> Result { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = [0; 4096]; + assert!(socket.read(&mut request).await.unwrap() > 0); + socket.write_all(response.as_bytes()).await.unwrap(); + std::future::pending::<()>().await; + }); + let response = litellm_http::Client::plain_for_test() + .get(format!("http://{address}")) + .send() + .await + .unwrap(); + let result = + tokio::time::timeout(Duration::from_secs(2), read_response_bytes(response, limit)) + .await; + server.abort(); + result.expect("bounded reads must finish without waiting for the rest of an oversized body") + } + + #[rstest] + #[case::declared("HTTP/1.1 200 OK\r\nContent-Length: 8\r\n\r\nabcdefgh")] + #[case::chunked( + "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nabcd\r\n4\r\nefgh\r\n0\r\n\r\n" + )] + #[tokio::test] + async fn a_body_of_exactly_the_limit_is_read(#[case] response: &str) { + assert_eq!(read_bounded(response.into(), 8).await.unwrap(), "abcdefgh"); + } + + #[rstest] + #[case::declared("HTTP/1.1 200 OK\r\nContent-Length: 9\r\n\r\n")] + #[case::chunked( + "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nabcd\r\n5\r\nefghi\r\n" + )] + #[tokio::test] + async fn a_body_over_the_limit_is_rejected(#[case] response: &str) { + assert!(matches!( + read_bounded(response.into(), 8).await, + Err(Error::TooLarge { limit: 8 }) + )); + } + + #[rstest] + #[case::declared("Content-Length: 1000000")] + #[case::chunked("Transfer-Encoding: chunked")] + #[tokio::test] + async fn an_oversized_error_keeps_its_status_and_a_bounded_body_without_draining( + #[case] headers: &str, + ) { + let prefix = "x".repeat(4096); + let body = match headers.starts_with("Transfer") { + true => format!("{:x}\r\n{prefix}\r\n", prefix.len()), + false => prefix.clone(), + }; + + let error = read_bounded( + format!("HTTP/1.1 429 Too Many Requests\r\n{headers}\r\n\r\n{body}"), + prefix.len(), + ) + .await + .unwrap_err(); + + let Error::Transport(litellm_http::transport::Error::Http { status, body }) = error else { + panic!("unexpected error: {error}"); + }; + assert_eq!(status, 429); + assert_eq!(body, prefix); + } } diff --git a/litellm-rust/crates/llms/src/bedrock/AGENTS.md b/litellm-rust/crates/llms/src/bedrock/AGENTS.md new file mode 100644 index 00000000000..2b012558e02 --- /dev/null +++ b/litellm-rust/crates/llms/src/bedrock/AGENTS.md @@ -0,0 +1,9 @@ +# rules + +- Owns Bedrock-wide policy: region resolution, the runtime endpoint template, and SigV4 or bearer-token auth through `litellm-auth-aws` +- Paths and wire types shared by Bedrock's formats come from `litellm_llms_types::providers::bedrock` + +# references + +- https://docs.aws.amazon.com/general/latest/gr/bedrock.html +- https://docs.aws.amazon.com/bedrock/latest/userguide/api-keys.html diff --git a/litellm-rust/crates/llms/src/bedrock/audio_transcription/AGENTS.md b/litellm-rust/crates/llms/src/bedrock/audio_transcription/AGENTS.md new file mode 100644 index 00000000000..3891321845e --- /dev/null +++ b/litellm-rust/crates/llms/src/bedrock/audio_transcription/AGENTS.md @@ -0,0 +1,8 @@ +# rules + +- Transcription over Converse: the audio goes in as a Converse content block and the transcript is the response text +- Non-text response blocks are an error, never silently dropped + +# references + +- https://docs.aws.amazon.com/bedrock/latest/userguide/conversation-inference.html diff --git a/litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs b/litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs index 307e7a09498..61dd264336e 100644 --- a/litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs +++ b/litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs @@ -5,7 +5,10 @@ use litellm_auth_aws::{ resolve_bedrock_region, }; use litellm_core_utils::core_helpers::json_type_name; -use litellm_llms_types::formats::audio_transcription::AudioTranscriptionResponseData; +use litellm_llms_types::{ + formats::audio_transcription::AudioTranscriptionResponseData, + providers::bedrock::ConverseResponse, +}; use serde::Deserialize; use serde_json::{Map, Value, json}; use strum::IntoStaticStr; @@ -19,7 +22,6 @@ use crate::{ }, auth::AuthScheme, }, - bedrock::chat::converse_transformation::ConverseResponse, }; const SUPPORTED_PARAMS: &[&str] = &["language", "prompt", "temperature", "response_format"]; diff --git a/litellm-rust/crates/llms/src/bedrock/chat/AGENTS.md b/litellm-rust/crates/llms/src/bedrock/chat/AGENTS.md new file mode 100644 index 00000000000..8627e3f73ef --- /dev/null +++ b/litellm-rust/crates/llms/src/bedrock/chat/AGENTS.md @@ -0,0 +1,5 @@ +# rules + +- Chat Completions over Converse. Converse wire docs live in `llms-types/src/providers/bedrock/AGENTS.md` +- `topK` is not forwarded, because Python's placement depends on the model catalog this crate cannot see +- `invoke_handler.rs` decodes InvokeModel event streams for Claude, reusing `anthropic/chat` response handling diff --git a/litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs b/litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs index 73e02d4b44e..caa345a7024 100644 --- a/litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs +++ b/litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs @@ -9,11 +9,13 @@ use litellm_core_utils::{ core_helpers::{finish_reason_for, unix_now, usage_from_parts}, prompt_templates::factory::{Conversation, TurnRole, build_conversation}, }; -use litellm_llms_types::formats::chat_completions::{ - ChatCompletionsChoice, ChatCompletionsChoiceMessage, ChatCompletionsResponse, - ChatCompletionsUsage, ChatMessage, ChatMessageContent, +use litellm_llms_types::{ + formats::chat_completions::{ + ChatCompletionsChoice, ChatCompletionsChoiceMessage, ChatCompletionsResponse, + ChatCompletionsUsage, ChatMessage, ChatMessageContent, + }, + providers::bedrock::{CONVERSE_PATH, ConverseResponse}, }; -use serde::Deserialize; use serde_json::{Map, Value, json}; use crate::{ @@ -62,61 +64,6 @@ const CONFIG_PARAMS: &[&str] = &[ AWS_BEDROCK_RUNTIME_ENDPOINT, ]; -const CONVERSE_PATH_SUFFIX: &str = "/converse"; - -#[derive(Deserialize)] -pub(crate) struct ConverseResponse { - output: ConverseOutput, - // Converse always reports usage, but the transcription route tolerates its - // absence; the chat transform checks for the field itself. - #[serde(default)] - usage: ConverseUsage, - #[serde(rename = "stopReason")] - stop_reason: Option, -} - -#[derive(Deserialize)] -struct ConverseOutput { - message: ConverseMessage, -} - -#[derive(Deserialize)] -struct ConverseMessage { - content: Vec, -} - -enum ConverseContentBlock { - Text { text: String }, - Other(serde::de::IgnoredAny), -} - -impl<'de> Deserialize<'de> for ConverseContentBlock { - fn deserialize>(deserializer: D) -> Result { - let value = Value::deserialize(deserializer)?; - if let Some(text) = value.get("text") { - let text = text.as_str().ok_or_else(|| { - serde::de::Error::custom("invalid type for `text`, expected a string") - })?; - return Ok(Self::Text { - text: text.to_owned(), - }); - } - Ok(Self::Other(serde::de::IgnoredAny)) - } -} - -#[derive(Default, Deserialize)] -#[serde(rename_all = "camelCase")] -struct ConverseUsage { - input_tokens: u64, - output_tokens: u64, - #[serde(default)] - cache_read_input_tokens: u64, - #[serde(default)] - cache_write_input_tokens: u64, - total_tokens: Option, -} - enum ConverseStopReason { Value(String), Unknown(String), @@ -141,28 +88,6 @@ impl ConverseStopReason { } } -impl ConverseResponse { - pub(crate) fn content_text(&self) -> String { - self.output - .message - .content - .iter() - .filter_map(|block| match block { - ConverseContentBlock::Text { text } => Some(text.as_str()), - ConverseContentBlock::Other(_) => None, - }) - .collect() - } - - pub(crate) fn message_content_is_non_text(&self) -> bool { - self.output - .message - .content - .iter() - .any(|block| matches!(block, ConverseContentBlock::Other(_))) - } -} - pub struct AmazonConverseConfig; pub const BEDROCK_CHAT_COMPLETIONS_CONFIG: AmazonConverseConfig = AmazonConverseConfig; @@ -205,10 +130,10 @@ impl BaseConfig for AmazonConverseConfig { // A host that already built the full Converse URL (LiteLLM's Python // path encodes the model id itself) passes it through untouched, the // way the Anthropic config leaves a complete `/v1/messages` URL alone. - if endpoint.ends_with(CONVERSE_PATH_SUFFIX) { + if endpoint.ends_with(CONVERSE_PATH) { return Ok(endpoint.to_string()); } - Ok(format!("{endpoint}/model/{model_id}{CONVERSE_PATH_SUFFIX}")) + Ok(format!("{endpoint}/model/{model_id}{CONVERSE_PATH}")) } fn transform_request( @@ -426,3 +351,624 @@ fn has_blank_text(message: &ChatMessage) -> bool { }), } } + +#[cfg(test)] +mod tests { + use crate::{ + Error, + base_llm::{ + auth::AuthScheme, + chat::transformation::{BaseConfig, ProviderChatResponseData, Unsupported}, + }, + bedrock::chat::converse_transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG, + }; + use litellm_auth::CredentialPlacement; + use litellm_llms_types::formats::chat_completions::{ChatCompletionsResponse, ChatMessage}; + use rstest::rstest; + use serde_json::{Map, Value, json}; + + fn messages(value: Value) -> Vec { + serde_json::from_value(value).expect("valid messages") + } + + fn params(value: Value) -> Map { + match value { + Value::Object(map) => map, + other => panic!("params must be an object, got {other}"), + } + } + + fn transform(msgs: Value, opts: Value) -> Value { + BEDROCK_CHAT_COMPLETIONS_CONFIG + .transform_request( + "anthropic.claude-sonnet-4-5-v1:0", + messages(msgs), + params(opts), + ) + .expect("request transforms") + .body + } + + fn transform_response(body: Value) -> Result { + BEDROCK_CHAT_COMPLETIONS_CONFIG.transform_response( + "anthropic.claude-sonnet-4-5-v1:0", + ProviderChatResponseData { body }, + ) + } + + fn reason(msgs: Value, opts: Value) -> Option { + BEDROCK_CHAT_COMPLETIONS_CONFIG.unsupported_reason(&messages(msgs), ¶ms(opts)) + } + + #[test] + fn builds_the_converse_body_python_builds() { + let body = transform( + json!([ + {"role": "system", "content": "be terse"}, + {"role": "user", "content": "hi"} + ]), + json!({"maxTokens": 128, "temperature": 0.2}), + ); + assert_eq!( + body, + json!({ + "inferenceConfig": {"maxTokens": 128, "temperature": 0.2}, + "messages": [{"role": "user", "content": [{"text": "hi"}]}], + "system": [{"text": "be terse"}] + }) + ); + } + + #[test] + fn always_emits_inference_config_even_when_empty() { + let body = transform(json!([{"role": "user", "content": "hi"}]), json!({})); + assert_eq!(body["inferenceConfig"], json!({})); + assert!(body.get("system").is_none()); + } + + #[test] + fn places_only_inference_params_in_inference_config() { + let body = transform( + json!([{"role": "user", "content": "hi"}]), + json!({ + "maxTokens": 64, + "temperature": 0.1, + "topP": 0.9, + "stopSequences": ["STOP"] + }), + ); + assert_eq!( + body["inferenceConfig"], + json!({"maxTokens": 64, "temperature": 0.1, "topP": 0.9, "stopSequences": ["STOP"]}) + ); + assert!(body.get("additionalModelRequestFields").is_none()); + } + + #[test] + fn merges_consecutive_user_turns_into_one_message() { + let body = transform( + json!([ + {"role": "user", "content": "one"}, + {"role": "user", "content": [{"type": "text", "text": "two"}]}, + {"role": "assistant", "content": "ack"}, + {"role": "user", "content": "three"} + ]), + json!({}), + ); + assert_eq!( + body["messages"], + json!([ + {"role": "user", "content": [{"text": "one"}, {"text": "two"}]}, + {"role": "assistant", "content": [{"text": "ack"}]}, + {"role": "user", "content": [{"text": "three"}]} + ]) + ); + } + + #[test] + fn declines_streaming() { + assert_eq!( + reason( + json!([{"role": "user", "content": "hi"}]), + json!({"stream": true}) + ), + Some(Unsupported("streaming")) + ); + } + + #[test] + fn declines_top_k_because_python_routes_it_by_base_model() { + assert_eq!( + reason( + json!([{"role": "user", "content": "hi"}]), + json!({"topK": 40}) + ), + Some(Unsupported("unrecognized request parameter")) + ); + } + + #[rstest] + #[case::tools(json!({"tools": []}))] + #[case::tool_choice(json!({"tool_choice": {"auto": {}}}))] + #[case::thinking(json!({"thinking": {"type": "enabled"}}))] + #[case::request_metadata(json!({"requestMetadata": {"k": "v"}}))] + #[case::output_config(json!({"outputConfig": {}}))] + #[case::parallel_tool_use_config(json!({"_parallel_tool_use_config": {}}))] + fn declines_tools_and_other_params_outside_the_allowlist(#[case] param: Value) { + assert_eq!( + reason(json!([{"role": "user", "content": "hi"}]), param.clone()), + Some(Unsupported("unrecognized request parameter")), + "expected {param} to decline" + ); + } + + #[rstest] + #[case::empty_string(json!(""))] + #[case::whitespace_string(json!(" "))] + #[case::whitespace_text_block(json!([{"type": "text", "text": " "}]))] + fn declines_blank_text_rather_than_substituting_the_anthropic_placeholder( + #[case] content: Value, + ) { + assert_eq!( + reason( + json!([{"role": "user", "content": content}, {"role": "user", "content": "hi"}]), + json!({}) + ), + Some(Unsupported("blank message text")), + "expected blank content {content} to decline" + ); + } + + #[test] + fn declines_a_message_whose_content_list_is_empty() { + // The blank-text check scans parts, so an empty list clears it; Converse + // rejects an empty `content` array, which is a decline the core owes the + // host before the call rather than an error after it. + assert_eq!( + reason(json!([{"role": "user", "content": []}]), json!({})), + Some(Unsupported("message without content")) + ); + assert_eq!( + reason( + json!([{"role": "user", "content": [{"type": "text", "text": "hi"}]}]), + json!({}) + ), + None + ); + } + + #[test] + fn declines_a_conversation_that_opens_or_closes_on_an_assistant_turn() { + assert_eq!( + reason( + json!([ + {"role": "assistant", "content": "prefill"}, + {"role": "user", "content": "hi"} + ]), + json!({}) + ), + Some(Unsupported( + "conversation does not run user turn to user turn" + )) + ); + assert_eq!( + reason( + json!([ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "prefill"} + ]), + json!({}) + ), + Some(Unsupported( + "conversation does not run user turn to user turn" + )) + ); + } + + #[test] + fn accepts_a_user_to_user_text_conversation() { + assert_eq!( + reason( + json!([ + {"role": "system", "content": "be terse"}, + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + {"role": "user", "content": "again"} + ]), + json!({"maxTokens": 16}) + ), + None + ); + } + + #[test] + fn builds_the_converse_url_from_the_region_in_the_model_id() { + let config = &BEDROCK_CHAT_COMPLETIONS_CONFIG; + assert_eq!( + config + .get_complete_url(None, "us-east-1/anthropic.claude-v2", &Map::new(), &|_| { + None + }) + .expect("url builds"), + "https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-v2/converse" + ); + } + + #[test] + fn falls_back_to_the_region_env_then_the_default_region() { + let config = &BEDROCK_CHAT_COMPLETIONS_CONFIG; + let with_env = |key: &str| (key == "AWS_REGION_NAME").then(|| "eu-west-1".to_string()); + assert_eq!( + config + .get_complete_url(None, "anthropic.claude-v2", &Map::new(), &with_env) + .expect("url builds"), + "https://bedrock-runtime.eu-west-1.amazonaws.com/model/anthropic.claude-v2/converse" + ); + assert_eq!( + config + .get_complete_url(None, "anthropic.claude-v2", &Map::new(), &|_| None) + .expect("url builds"), + "https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-v2/converse" + ); + } + + #[test] + fn prefers_an_explicit_runtime_endpoint_over_the_api_base() { + let config = &BEDROCK_CHAT_COMPLETIONS_CONFIG; + let overrides = params(json!({"aws_bedrock_runtime_endpoint": "https://vpce.internal/"})); + assert_eq!( + config + .get_complete_url( + Some("https://ignored.example"), + "anthropic.claude-v2", + &overrides, + &|_| None + ) + .expect("url builds"), + "https://vpce.internal/model/anthropic.claude-v2/converse" + ); + } + + /// The bearer token a config named, or `None` for a SigV4 scheme in the given region. + fn bearer_or_region(auth: AuthScheme) -> Result { + match auth { + AuthScheme::Credential { + placement: CredentialPlacement::Bearer, + secret, + } => Ok(secret.expose().to_string()), + AuthScheme::AwsSigV4 { + region, + service: "bedrock", + .. + } => Err(region), + other => panic!("unexpected auth {other:?}"), + } + } + + #[test] + fn signs_with_sigv4_in_the_resolved_region() { + let validated = BEDROCK_CHAT_COMPLETIONS_CONFIG + .validate_environment( + Vec::new(), + None, + "eu-central-1/anthropic.claude-v2", + &Map::new(), + &|_| None, + ) + .expect("auth resolves"); + assert_eq!( + bearer_or_region(validated.auth), + Err("eu-central-1".to_string()) + ); + } + + #[test] + fn a_bearer_token_outranks_sigv4_the_way_python_resolves_it() { + // Python's get_request_headers reads `api_key` as the Bedrock bearer token + // and only falls back to the env when the caller passed none, so each case + // pins one of its precedence rules. Signing as the host principal when a + // bearer identity is configured would cross an account and quota boundary. + let bedrock_env = + |key: &str| (key == "AWS_BEARER_TOKEN_BEDROCK").then(|| "from-env".to_string()); + let no_env = |_: &str| None; + let resolve = |api_key, env: &dyn Fn(&str) -> Option| { + bearer_or_region( + BEDROCK_CHAT_COMPLETIONS_CONFIG + .validate_environment( + Vec::new(), + api_key, + "eu-central-1/anthropic.claude-v2", + &Map::new(), + env, + ) + .expect("auth resolves") + .auth, + ) + }; + let bearer = |token: &str| Ok(token.to_string()); + let sigv4 = Err("eu-central-1".to_string()); + + // A caller-supplied key is the bearer token, and outranks the env. + assert_eq!( + resolve(Some("bedrock-api-key"), &bedrock_env), + bearer("bedrock-api-key") + ); + // No key, so the env supplies it. + assert_eq!(resolve(None, &bedrock_env), bearer("from-env")); + // An empty key is not a bearer token, and deliberately does NOT reach for + // the env, which is what Python's `is not None` check does. + assert_eq!(resolve(Some(""), &bedrock_env), sigv4); + // Whitespace is truthy in Python, so it stays a bearer token rather than + // silently becoming a host-credentialed SigV4 request. + assert_eq!(resolve(Some(" "), &no_env), bearer(" ")); + // Neither present, so SigV4 as before. + assert_eq!(resolve(None, &no_env), sigv4); + } + + #[test] + fn normalizes_a_converse_response_into_openai_shape() { + let response = transform_response(json!({ + "output": {"message": {"role": "assistant", "content": [ + {"text": "hello"}, {"text": " there"} + ]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 11, "outputTokens": 4, "totalTokens": 15} + })) + .expect("response transforms"); + + assert_eq!(response.model, "anthropic.claude-sonnet-4-5-v1:0"); + assert_eq!( + response.choices[0].message.content.as_deref(), + Some("hello there") + ); + assert_eq!(response.choices[0].finish_reason, "stop"); + assert_eq!(response.usage.prompt_tokens, 11); + assert_eq!(response.usage.completion_tokens, 4); + assert_eq!(response.usage.total_tokens, 15); + } + + #[test] + fn maps_converse_stop_reasons_python_maps() { + for (provider_reason, expected) in [ + ("end_turn", "stop"), + ("stop_sequence", "stop"), + ("max_tokens", "length"), + ("guardrail_intervened", "content_filter"), + // Converse emits this one, and Python's `_FINISH_REASON_MAP` carries + // it. Folding it into `stop` reports a filtered completion as a normal + // one to anything keying on the finish reason. + ("content_filtered", "content_filter"), + ("content_filter", "content_filter"), + ] { + let response = transform_response(json!({ + "output": {"message": {"content": [{"text": "x"}]}}, + "stopReason": provider_reason, + "usage": {"inputTokens": 1, "outputTokens": 1} + })) + .expect("response transforms"); + assert_eq!( + response.choices[0].finish_reason, expected, + "stopReason {provider_reason}" + ); + } + } + + #[test] + fn reports_an_empty_converse_answer_as_an_empty_string_not_null() { + // Converse assigns the joined text unconditionally + // (`chat_completion_message["content"] = content_str`), unlike Anthropic's + // `merged_text or None`, so an empty answer is `""` on both paths. A caller + // calling `.strip()` on it would break on the Rust path alone. Reachable + // through a filtered or guardrail-intervened response. + for content in [json!([]), json!([{"text": ""}])] { + let response = transform_response(json!({ + "output": {"message": {"content": content}}, + "stopReason": "content_filtered", + "usage": {"inputTokens": 1, "outputTokens": 0} + })) + .expect("response transforms"); + assert_eq!(response.choices[0].message.content, Some(String::new())); + } + } + + #[test] + fn reports_the_total_tokens_converse_sent_rather_than_recomputing_them() { + // Python reads `usage["totalTokens"]` straight through here, where Anthropic + // has no such field and adds the two counts instead. The two agree while the + // gate declines every cache_control request, so this is what keeps them + // agreeing if that ever widens. + let response = transform_response(json!({ + "output": {"message": {"content": [{"text": "x"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 10, "outputTokens": 4, "cacheReadInputTokens": 7, "totalTokens": 14} + })) + .expect("response transforms"); + assert_eq!( + response.usage.total_tokens, 14, + "provider total was recomputed" + ); + assert_eq!(response.usage.prompt_tokens, 17); + assert_eq!(response.usage.completion_tokens, 4); + } + + #[test] + fn falls_back_to_the_computed_total_when_converse_omits_it() { + // Python raises a KeyError on a body with no `totalTokens`. Reporting a zero + // instead would be a worse divergence than the one above, so the computed + // total stands in. + let response = transform_response(json!({ + "output": {"message": {"content": [{"text": "x"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 10, "outputTokens": 4} + })) + .expect("response transforms"); + assert_eq!(response.usage.total_tokens, 14); + } + + #[test] + fn declines_a_cache_control_message_so_widening_the_gate_is_a_red_test() { + // Converse only reports cache token counts when the request carries a + // cachePoint block, which is why the provider total and the computed one + // cannot disagree today. This is the tripwire: whoever widens the gate to + // admit prompt caching has to come back and re-check the usage mapping + // rather than discovering a silent number change in production. + assert_eq!( + reason( + json!([{"role": "user", "content": [ + {"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}} + ]}]), + json!({}) + ), + Some(Unsupported("non-text message content")) + ); + } + + #[test] + fn folds_converse_cache_tokens_into_prompt_tokens() { + let response = transform_response(json!({ + "output": {"message": {"content": [{"text": "x"}]}}, + "stopReason": "end_turn", + "usage": { + "inputTokens": 10, + "outputTokens": 2, + "cacheReadInputTokens": 5, + "cacheWriteInputTokens": 3 + } + })) + .expect("response transforms"); + assert_eq!(response.usage.prompt_tokens, 18); + assert_eq!(response.usage.prompt_tokens_details.cached_tokens, 5); + assert_eq!( + response.usage.prompt_tokens_details.cache_creation_tokens, + 3 + ); + assert_eq!(response.usage.prompt_tokens_details.text_tokens, 10); + } + + #[test] + fn declines_a_response_carrying_a_tool_use_block() { + let err = transform_response(json!({ + "output": {"message": {"content": [ + {"toolUse": {"toolUseId": "t1", "name": "f", "input": {}}} + ]}}, + "stopReason": "tool_use", + "usage": {"inputTokens": 1, "outputTokens": 1} + })) + .expect_err("tool use block"); + assert_eq!(err, Error::Unsupported("non-text response content block")); + } + + #[test] + fn errors_on_a_response_missing_required_fields() { + assert_eq!( + transform_response(json!("nope")).expect_err("not an object"), + Error::InvalidResponse("converse response is not an object".to_string().into()) + ); + assert_eq!( + transform_response(json!({"usage": {}})).expect_err("no output"), + Error::InvalidResponse("invalid Converse response: missing field `output`".into()) + ); + assert_eq!( + transform_response(json!({"output": {"message": {"content": []}}})) + .expect_err("no usage"), + Error::InvalidResponse("invalid Converse response: missing field `usage`".into()) + ); + } + + #[test] + fn rejects_malformed_text_and_token_counts() { + for response in [ + json!({"output": {"message": {"content": [{"text": 123}]}}, "usage": {"inputTokens": 1, "outputTokens": 1}}), + json!({"output": {"message": {"content": [{"text": "x"}]}}, "usage": {"inputTokens": "1", "outputTokens": 1}}), + ] { + assert!(matches!( + transform_response(response), + Err(Error::InvalidResponse(message)) if message.to_string().starts_with("invalid Converse response:") + )); + } + } + + #[test] + fn accepts_aws_call_configuration_without_serializing_it() { + let call_config = json!({ + "maxTokens": 16, + "aws_access_key_id": "AKIA", + "aws_secret_access_key": "secret", + "aws_session_token": "token", + "aws_region_name": "us-east-1", + "aws_profile_name": "litellm-stage", + "aws_role_name": "role", + "aws_session_name": "session", + "aws_web_identity_token": "wit", + "aws_sts_endpoint": "https://sts.example", + "aws_external_id": "ext", + "aws_bedrock_runtime_endpoint": "https://vpce.internal" + }); + assert_eq!( + reason( + json!([{"role": "user", "content": "hi"}]), + call_config.clone() + ), + None + ); + let body = transform(json!([{"role": "user", "content": "hi"}]), call_config); + assert_eq!( + body, + json!({ + "inferenceConfig": {"maxTokens": 16}, + "messages": [{"role": "user", "content": [{"text": "hi"}]}] + }), + "aws call configuration must not reach the Converse body" + ); + } + + #[test] + fn leaves_a_complete_converse_url_untouched() { + let config = &BEDROCK_CHAT_COMPLETIONS_CONFIG; + let already_built = "https://bedrock-runtime.us-east-1.amazonaws.com/model/us.anthropic.claude-v2%3A0/converse"; + assert_eq!( + config + .get_complete_url( + Some(already_built), + "anthropic.claude-v2", + &Map::new(), + &|_| None + ) + .expect("url builds"), + already_built, + "a host that encoded the model id itself must not have it re-derived" + ); + } + + #[rstest] + #[case::full_static_pair( + json!({"aws_access_key_id": "AKIAHOST", "aws_secret_access_key": "hostsecret", "aws_session_token": "hosttoken"}), + Some(("AKIAHOST", "hostsecret", Some("hosttoken"))) + )] + #[case::pair_without_session_token( + json!({"aws_access_key_id": "AKIAHOST", "aws_secret_access_key": "hostsecret"}), + Some(("AKIAHOST", "hostsecret", None)) + )] + #[case::key_id_alone(json!({"aws_access_key_id": "AKIA"}), None)] + #[case::blank_key_id(json!({"aws_access_key_id": " ", "aws_secret_access_key": "s"}), None)] + #[case::nothing(json!({}), None)] + fn host_supplied_credentials_need_a_full_static_pair( + #[case] optional_params: Value, + #[case] expected: Option<(&str, &str, Option<&str>)>, + ) { + use litellm_auth::AwsParams; + use litellm_auth_aws::host_supplied_credentials; + + let credentials = + host_supplied_credentials(&AwsParams::from_optional_params(¶ms(optional_params))); + + assert_eq!( + credentials.as_ref().map(|credentials| ( + credentials.access_key_id(), + credentials.secret_access_key(), + credentials.session_token(), + )), + expected + ); + } +} diff --git a/litellm-rust/crates/llms/src/bedrock/messages/AGENTS.md b/litellm-rust/crates/llms/src/bedrock/messages/AGENTS.md index 4ad96b60514..48c222a1cdc 100644 --- a/litellm-rust/crates/llms/src/bedrock/messages/AGENTS.md +++ b/litellm-rust/crates/llms/src/bedrock/messages/AGENTS.md @@ -1,3 +1,8 @@ -This directory owns Bedrock's Messages adapter: its endpoints, authentication policy, wire adaptation, and response decoding. Implement the shared adapter contract from `base_llm/messages`, consume API data contracts from `litellm-llms-types::formats::messages`, and leave call orchestration to `inference-messages` +# rules -The Claude adapter may explicitly reuse payload policy from `anthropic/messages` when it applies to Bedrock's Claude backend. Keep Bedrock-specific differences here. Sharing that helper does not make Anthropic policy a format-wide default or justify a dependency from `base_llm/messages` on provider implementations +- Claude on Bedrock InvokeModel: reuses `anthropic/messages` shaping, moves the model into the URL, and decodes the AWS event stream +- Invocation metrics in the response map onto Messages usage here + +# references + +- https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages.html diff --git a/litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.rs b/litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.rs index 59452e0b693..4f8d4f01697 100644 --- a/litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.rs +++ b/litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.rs @@ -19,6 +19,9 @@ use litellm_llms_types::formats::messages::{ MessagesRequest, streaming::{MessagesStreamEvent, MessagesStreamUsage}, }; +use litellm_llms_types::providers::bedrock::{ + INVOCATION_METRICS_KEY, INVOKE_PATH, INVOKE_STREAM_PATH, +}; use litellm_router_types::LitellmParams; use serde_json::{Map, Value}; @@ -35,8 +38,6 @@ use crate::{ bedrock::chat::invoke_handler::{decode_invoke_anthropic_chunk, invoke_chunk_stream}, }; -const INVOCATION_METRICS_KEY: &str = "amazon-bedrock-invocationMetrics"; - const METRICS_USAGE_KEYS: [(&str, &str); 4] = [ ("input_tokens", "inputTokenCount"), ("output_tokens", "outputTokenCount"), @@ -44,8 +45,6 @@ const METRICS_USAGE_KEYS: [(&str, &str); 4] = [ ("cache_creation_input_tokens", "cacheWriteInputTokenCount"), ]; -const INVOKE_PATH: &str = "invoke"; -const INVOKE_STREAM_PATH: &str = "invoke-with-response-stream"; const INVOKE_MODEL_PREFIX: &str = "invoke/"; const SECRET_NAMES: &[&str] = &[ diff --git a/litellm-rust/crates/llms/src/cohere/ocr/AGENTS.md b/litellm-rust/crates/llms/src/cohere/ocr/AGENTS.md new file mode 100644 index 00000000000..2aefeee4ab1 --- /dev/null +++ b/litellm-rust/crates/llms/src/cohere/ocr/AGENTS.md @@ -0,0 +1,7 @@ +# rules + +- Translates Cohere Parse output into the shared OCR format. Azure AI reuses this adapter for its hosted Cohere + +# references + +- https://docs.cohere.com/reference/parse diff --git a/litellm-rust/crates/llms/src/deepseek/messages/AGENTS.md b/litellm-rust/crates/llms/src/deepseek/messages/AGENTS.md new file mode 100644 index 00000000000..2d7be84ea3a --- /dev/null +++ b/litellm-rust/crates/llms/src/deepseek/messages/AGENTS.md @@ -0,0 +1,9 @@ +# rules + +- DeepSeek's Anthropic-compatible host: reuses `anthropic/messages` shaping with `x-api-key` auth +- Drops the explicit `type: custom` tool discriminator and Claude Code billing system blocks, which DeepSeek rejects +- Peels OpenAI-style suffixes off a configured base before appending `/anthropic/v1/messages` + +# references + +- https://api-docs.deepseek.com/guides/anthropic_api diff --git a/litellm-rust/crates/llms/src/deepseek/messages/transformation.rs b/litellm-rust/crates/llms/src/deepseek/messages/transformation.rs index 43db1529cc8..9359184f40e 100644 --- a/litellm-rust/crates/llms/src/deepseek/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/deepseek/messages/transformation.rs @@ -2,6 +2,7 @@ use litellm_auth::{CredentialPlacement, SecretValue}; use litellm_core_utils::settings::resolve_non_empty; use litellm_llms_types::{ formats::messages::{MessagesOptionalParams, MessagesRequest, MessagesTool}, + providers::anthropic::{DEFAULT_HEADERS, MESSAGES_PATH}, recognized::Recognized, }; use litellm_router_types::LitellmParams; @@ -13,17 +14,12 @@ use crate::{ common_utils::{filter_billing_headers_from_system, has_anthropic_credential}, messages::{ handler::shape_anthropic_messages_request, - transformation::{ - DEFAULT_HEADERS, transform_messages_request, update_headers_with_anthropic_beta, - }, + transformation::{transform_messages_request, update_headers_with_anthropic_beta}, }, }, base_llm::{ auth::{AuthScheme, Headers, ValidatedEnvironment}, - messages::{ - context::MessagesTransformContext, - transformation::{BaseMessagesConfig, MESSAGES_PATH_SUFFIX}, - }, + messages::{context::MessagesTransformContext, transformation::BaseMessagesConfig}, }, }; @@ -166,18 +162,18 @@ pub fn complete_deepseek_anthropic_url( ) -> String { let api_base = get_api_base(api_base, env_lookup); let api_base = api_base.trim_end_matches('/'); - if api_base.ends_with(MESSAGES_PATH_SUFFIX) && api_base.contains("/anthropic/") { + if api_base.ends_with(MESSAGES_PATH) && api_base.contains("/anthropic/") { return api_base.to_string(); } - let api_base = [MESSAGES_PATH_SUFFIX, "/v1", "/beta"] + let api_base = [MESSAGES_PATH, "/v1", "/beta"] .into_iter() .fold(api_base, |base, suffix| { base.strip_suffix(suffix).unwrap_or(base) }); if api_base.ends_with(ANTHROPIC_PATH_SEGMENT) || api_base.contains("/anthropic/") { - return format!("{api_base}{MESSAGES_PATH_SUFFIX}"); + return format!("{api_base}{MESSAGES_PATH}"); } - format!("{api_base}{ANTHROPIC_PATH_SEGMENT}{MESSAGES_PATH_SUFFIX}") + format!("{api_base}{ANTHROPIC_PATH_SEGMENT}{MESSAGES_PATH}") } /// DeepSeek rejects Anthropic's explicit `{"type": "custom"}` tool discriminator, so it is diff --git a/litellm-rust/crates/llms/src/mistral/ocr/AGENTS.md b/litellm-rust/crates/llms/src/mistral/ocr/AGENTS.md new file mode 100644 index 00000000000..8c75f1c168e --- /dev/null +++ b/litellm-rust/crates/llms/src/mistral/ocr/AGENTS.md @@ -0,0 +1,8 @@ +# rules + +- Mistral is the origin of the shared OCR shape, documented in `llms-types/src/formats/ocr/AGENTS.md`. This adapter adds only auth and URL handling +- Azure AI and Vertex AI reuse `MistralOcrRequest` for their hosted Mistral OCR + +# references + +- https://docs.mistral.ai/capabilities/document_ai/basic_ocr diff --git a/litellm-rust/crates/llms/src/openai/responses/AGENTS.md b/litellm-rust/crates/llms/src/openai/responses/AGENTS.md new file mode 100644 index 00000000000..566ebe8edd0 --- /dev/null +++ b/litellm-rust/crates/llms/src/openai/responses/AGENTS.md @@ -0,0 +1,8 @@ +# rules + +- Format spec lives in `llms-types/src/formats/responses/AGENTS.md` +- Serves both the direct HTTP path and WebSocket mode, deriving the `wss://` URL from the HTTP base + +# references + +- https://developers.openai.com/api/docs/guides/websocket-mode diff --git a/litellm-rust/crates/llms/src/openai_like/chat/AGENTS.md b/litellm-rust/crates/llms/src/openai_like/chat/AGENTS.md new file mode 100644 index 00000000000..7fb55994b85 --- /dev/null +++ b/litellm-rust/crates/llms/src/openai_like/chat/AGENTS.md @@ -0,0 +1,5 @@ +# rules + +- The shared config for every OpenAI-compatible host. Format spec lives in `llms-types/src/formats/chat_completions/AGENTS.md` +- Keeps Python's two deviations: `max_completion_tokens` becomes `max_tokens`, and null usage counts become zero +- Declines tool requests before the call, since tool-call responses are not normalized yet diff --git a/litellm-rust/crates/llms/src/openai_like/chat/transformation.rs b/litellm-rust/crates/llms/src/openai_like/chat/transformation.rs index f1ec8dc8b1d..0a89adf90f7 100644 --- a/litellm-rust/crates/llms/src/openai_like/chat/transformation.rs +++ b/litellm-rust/crates/llms/src/openai_like/chat/transformation.rs @@ -272,3 +272,350 @@ fn normalize_choice(position: usize, choice: &Value) -> Result Vec { + serde_json::from_value(value).expect("valid messages") + } + + fn params(value: Value) -> Map { + match value { + Value::Object(map) => map, + other => panic!("params must be an object, got {other}"), + } + } + + fn no_env(_: &str) -> Option { + None + } + + fn env_with<'a>(name: &'a str, value: &'a str) -> impl Fn(&str) -> Option + 'a { + move |key| (key == name).then(|| value.to_string()) + } + + fn transform(model: &str, msgs: Value, opts: Value) -> Value { + OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG + .transform_request(model, messages(msgs), params(opts)) + .expect("request transforms") + .body + } + + fn transform_response(body: Value) -> Result { + OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG + .transform_response("some-model", ProviderChatResponseData { body }) + } + + fn reason(msgs: Value, opts: Value) -> Option { + OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG.unsupported_reason(&messages(msgs), ¶ms(opts)) + } + + #[rstest] + fn builds_the_openai_shaped_body() { + let body = transform( + "my-model", + json!([ + {"role": "system", "content": "be terse"}, + {"role": "user", "content": "hi"}, + ]), + json!({"temperature": 0.5, "max_tokens": 8}), + ); + assert_eq!(body["model"], json!("my-model")); + assert_eq!( + body["messages"], + json!([ + {"role": "system", "content": "be terse"}, + {"role": "user", "content": "hi"}, + ]) + ); + assert_eq!(body["temperature"], json!(0.5)); + assert_eq!(body["max_tokens"], json!(8)); + } + + #[rstest] + fn renames_max_completion_tokens_to_max_tokens() { + // `OpenAILikeChatConfig.map_openai_params`: most OpenAI-compatible providers + // support `max_tokens`, not `max_completion_tokens`. + let body = transform( + "my-model", + json!([{"role": "user", "content": "hi"}]), + json!({"max_completion_tokens": 12}), + ); + assert_eq!(body["max_tokens"], json!(12)); + assert!(body.get("max_completion_tokens").is_none()); + } + + #[rstest] + fn max_completion_tokens_wins_when_both_limits_are_sent() { + // Python assigns `max_tokens = max_completion_tokens` after copying the + // params, so the renamed value outranks a caller-supplied `max_tokens`. + let body = transform( + "my-model", + json!([{"role": "user", "content": "hi"}]), + json!({"max_tokens": 8, "max_completion_tokens": 12}), + ); + assert_eq!(body["max_tokens"], json!(12)); + assert!(body.get("max_completion_tokens").is_none()); + } + + #[rstest] + fn call_configuration_never_enters_the_body() { + let body = transform( + "my-model", + json!([{"role": "user", "content": "hi"}]), + json!({"custom_endpoint": true, "extra_headers": {"x": "y"}, "max_retries": 2}), + ); + assert_eq!( + body.as_object().unwrap().keys().collect::>(), + vec!["model", "messages"] + ); + } + + #[rstest] + #[case::appends_the_chat_completions_suffix("https://vllm.example.com/v1", json!({}), "https://vllm.example.com/v1/chat/completions")] + #[case::trims_a_trailing_slash("https://vllm.example.com/v1/", json!({}), "https://vllm.example.com/v1/chat/completions")] + #[case::a_custom_endpoint_is_used_as_is("https://vllm.example.com/v1/chat/completions", json!({"custom_endpoint": true}), "https://vllm.example.com/v1/chat/completions")] + fn complete_url(#[case] api_base: &str, #[case] opts: Value, #[case] expected: &str) { + assert_eq!( + OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG + .get_complete_url(Some(api_base), "my-model", ¶ms(opts), &no_env) + .expect("url resolves"), + expected + ); + } + + #[rstest] + fn api_base_falls_back_to_the_environment() { + assert_eq!( + OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG + .get_complete_url( + None, + "my-model", + ¶ms(json!({})), + &env_with("OPENAI_LIKE_API_BASE", "https://env.example.com/v1"), + ) + .expect("url resolves"), + "https://env.example.com/v1/chat/completions" + ); + } + + #[rstest] + fn a_missing_api_base_is_an_error() { + assert!(matches!( + OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG + .get_complete_url(None, "my-model", ¶ms(json!({})), &no_env), + Err(Error::InvalidRequest(message)) if message.to_string().starts_with("Missing API Base") + )); + } + + #[rstest] + fn the_resolved_key_authenticates_as_a_bearer() { + let validated = OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG + .validate_environment( + vec![], + Some("sk-test"), + "my-model", + ¶ms(json!({})), + &no_env, + ) + .expect("validates"); + assert!(matches!( + validated.auth, + AuthScheme::Credential { + placement: litellm_auth::CredentialPlacement::Bearer, + .. + } + )); + } + + #[rstest] + fn the_key_falls_back_to_the_environment() { + let validated = OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG + .validate_environment( + vec![], + None, + "my-model", + ¶ms(json!({})), + &env_with("OPENAI_LIKE_API_KEY", "sk-env"), + ) + .expect("validates"); + let AuthScheme::Credential { secret, .. } = validated.auth else { + panic!("expected a bearer credential"); + }; + assert_eq!(secret.expose(), "sk-env"); + } + + #[rstest] + fn a_forwarded_authorization_is_the_whole_credential() { + // Python adds `Bearer ` only when the caller did not already send + // `Authorization`, so the forwarded header wins over the deployment key. + let headers = vec![("Authorization".to_string(), "Bearer caller".to_string())]; + let validated = OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG + .validate_environment( + headers, + Some("sk-test"), + "my-model", + ¶ms(json!({})), + &no_env, + ) + .expect("validates"); + assert!(matches!(validated.auth, AuthScheme::Forwarded)); + } + + #[rstest] + fn keyless_calls_still_validate_for_endpoints_that_take_no_key() { + // vllm-compatible endpoints require no api key; Python resolves `""` and + // sends `Bearer `, so validation must not fail on the missing key. + let validated = OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG + .validate_environment(vec![], None, "my-model", ¶ms(json!({})), &no_env) + .expect("validates"); + let AuthScheme::Credential { secret, .. } = validated.auth else { + panic!("expected a bearer credential"); + }; + assert_eq!(secret.expose(), ""); + } + + #[rstest] + fn normalizes_an_openai_response() { + let response = transform_response(json!({ + "created": 1_700_000_000, + "model": "served-model-name", + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "hello"}, + "finish_reason": "stop", + }], + "usage": {"prompt_tokens": 3, "completion_tokens": 5, "total_tokens": 8}, + })) + .expect("response normalizes"); + assert_eq!(response.created, 1_700_000_000); + assert_eq!(response.model, "served-model-name"); + assert_eq!( + response.choices[0].message.content.as_deref(), + Some("hello") + ); + assert_eq!(response.choices[0].finish_reason, "stop"); + assert_eq!(response.usage.prompt_tokens, 3); + assert_eq!(response.usage.completion_tokens, 5); + assert_eq!(response.usage.total_tokens, 8); + } + + #[rstest] + fn null_token_fields_in_usage_become_zero() { + // `_sanitize_usage_obj`: providers that return null token values break + // OpenAI clients, so the response is scrubbed at the source. + let response = transform_response(json!({ + "model": "m", + "choices": [{"message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": null, "total_tokens": null}, + })) + .expect("response normalizes"); + assert_eq!(response.usage.completion_tokens, 0); + assert_eq!(response.usage.total_tokens, 0); + assert_eq!(response.usage.prompt_tokens, 3); + } + + #[rstest] + fn a_tool_call_response_declines_instead_of_dropping_the_calls() { + // The `json_mode` rewrite needs a request flag the route does not carry, so + // a tool-call answer falls back to Python rather than losing the calls. + assert_eq!( + transform_response(json!({ + "model": "m", + "choices": [{ + "message": { + "role": "assistant", + "content": null, + "tool_calls": [{ + "id": "call_1", + "type": "function", + "function": {"name": "f", "arguments": "{}"}, + }], + }, + "finish_reason": "tool_calls", + }], + })), + Err(Error::Unsupported("tool call response")) + ); + } + + #[rstest] + fn a_refusal_declines_instead_of_returning_an_empty_reply() { + assert_eq!( + transform_response(json!({ + "model": "m", + "choices": [{ + "message": {"role": "assistant", "content": null, "refusal": "cannot help"}, + "finish_reason": "stop", + }], + })), + Err(Error::Unsupported("refusal response")) + ); + } + + #[rstest] + fn a_non_text_response_content_declines() { + assert_eq!( + transform_response(json!({ + "model": "m", + "choices": [{ + "message": {"role": "assistant", "content": [{"type": "text", "text": "hi"}]}, + "finish_reason": "stop", + }], + })), + Err(Error::Unsupported("non-text response content")) + ); + } + + #[rstest] + #[case::streaming(json!({"stream": true}), "streaming")] + #[case::unrecognized_param(json!({"some_provider_knob": 1}), "unrecognized request parameter")] + fn declines(#[case] opts: Value, #[case] expected: &'static str) { + assert_eq!( + reason(json!([{"role": "user", "content": "hi"}]), opts), + Some(Unsupported(expected)) + ); + } + + #[rstest] + fn accepts_standard_openai_params() { + assert_eq!( + reason( + json!([{"role": "user", "content": "hi"}]), + json!({ + "temperature": 0.2, + "top_p": 0.9, + "max_tokens": 16, + "response_format": {"type": "json_object"}, + "custom_endpoint": true, + }), + ), + None + ); + } + + #[rstest] + fn tool_parameters_decline_before_the_call() { + // A `tools` request would come back with tool calls this port cannot + // normalize, so it declines at the gate instead of after the call. + assert_eq!( + reason( + json!([{"role": "user", "content": "hi"}]), + json!({"tools": [{"type": "function", "function": {"name": "f"}}]}), + ), + Some(Unsupported("unrecognized request parameter")) + ); + } +} diff --git a/litellm-rust/crates/llms/src/reducto/ocr/AGENTS.md b/litellm-rust/crates/llms/src/reducto/ocr/AGENTS.md new file mode 100644 index 00000000000..06128d3ed05 --- /dev/null +++ b/litellm-rust/crates/llms/src/reducto/ocr/AGENTS.md @@ -0,0 +1,9 @@ +# rules + +- Uploads the document first and parses by the returned file id, so guardrails see the source document before upload instead of the final body +- Supports both the v3 parse request and the legacy options shape + +# references + +- https://docs.reducto.ai/api-reference/upload +- https://docs.reducto.ai/api-reference/parse diff --git a/litellm-rust/crates/llms/src/vertex_ai/common_utils.rs b/litellm-rust/crates/llms/src/vertex_ai/common_utils.rs index 45abf5cd9f2..43b0f3f0de3 100644 --- a/litellm-rust/crates/llms/src/vertex_ai/common_utils.rs +++ b/litellm-rust/crates/llms/src/vertex_ai/common_utils.rs @@ -1,7 +1,6 @@ -use crate::Error; +use litellm_llms_types::providers::vertex_ai::{GLOBAL_API_BASE, GLOBAL_LOCATION}; -const GLOBAL_LOCATION: &str = "global"; -pub const DEFAULT_VERTEX_LOCATION: &str = "us-central1"; +use crate::Error; fn is_location_token(location: &str) -> bool { !location.is_empty() @@ -37,7 +36,7 @@ pub fn validate_vertex_location(location: &str) -> Result<&str, Error> { pub fn get_vertex_base_url(location: &str) -> Result { let location = validate_vertex_location(location)?; if location == GLOBAL_LOCATION { - return Ok("https://aiplatform.googleapis.com".to_string()); + return Ok(GLOBAL_API_BASE.to_string()); } if !location.contains('-') { return Ok(format!("https://aiplatform.{location}.rep.googleapis.com")); diff --git a/litellm-rust/crates/llms/src/vertex_ai/messages/AGENTS.md b/litellm-rust/crates/llms/src/vertex_ai/messages/AGENTS.md new file mode 100644 index 00000000000..b5d637d6bc7 --- /dev/null +++ b/litellm-rust/crates/llms/src/vertex_ai/messages/AGENTS.md @@ -0,0 +1,8 @@ +# rules + +- Claude on Vertex AI: reuses `anthropic/messages` shaping, moves the model into the publisher URL, puts the Vertex `anthropic_version` in the body, and authenticates with a Google access token +- Applies Vertex's beta policy and strips `output_config.effort` for models that do not accept it + +# references + +- https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/use-claude diff --git a/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs b/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs index 8bedea4d100..e6a3661a81f 100644 --- a/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs @@ -29,12 +29,12 @@ use crate::{ transformation::BaseMessagesConfig, }, }, - vertex_ai::common_utils::{DEFAULT_VERTEX_LOCATION, get_vertex_base_url}, + vertex_ai::common_utils::get_vertex_base_url, +}; +use litellm_llms_types::providers::vertex_ai::{ + ANTHROPIC_VERSION, DEFAULT_LOCATION, RAW_PREDICT, STREAM_RAW_PREDICT, }; -pub const VERTEX_ANTHROPIC_VERSION: &str = "vertex-2023-10-16"; -const RAW_PREDICT: &str = "rawPredict"; -const STREAM_RAW_PREDICT: &str = "streamRawPredict"; const WEB_SEARCH_TOOL_PREFIX: &str = "web_search"; /// Claude on Vertex AI, Python's `VertexAIPartnerModelsAnthropicMessagesConfig`: the Anthropic @@ -66,7 +66,7 @@ impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig { .or_else(|| get_vertex_ai_project_from_credentials(&config, env_lookup)) .ok_or_else(|| Error::Auth(VertexParams::PROJECT.missing("Vertex AI")))?; let location = get_vertex_ai_location(&config, env_lookup) - .unwrap_or_else(|| DEFAULT_VERTEX_LOCATION.to_string()); + .unwrap_or_else(|| DEFAULT_LOCATION.to_string()); complete_vertex_anthropic_url(api_base, &project, &location, model, stream) } @@ -94,10 +94,7 @@ impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig { .params .extra .into_iter() - .chain([( - "anthropic_version".to_string(), - json!(VERTEX_ANTHROPIC_VERSION), - )]) + .chain([("anthropic_version".to_string(), json!(ANTHROPIC_VERSION))]) .collect(), ..request.params }, diff --git a/litellm-rust/crates/llms/src/vertex_ai/ocr/AGENTS.md b/litellm-rust/crates/llms/src/vertex_ai/ocr/AGENTS.md new file mode 100644 index 00000000000..30b6bf0757a --- /dev/null +++ b/litellm-rust/crates/llms/src/vertex_ai/ocr/AGENTS.md @@ -0,0 +1,9 @@ +# rules + +- `transformation.rs` sends the Mistral OCR request to Vertex's Mistral publisher through `rawPredict` +- `deepseek_transformation.rs` sends DeepSeek-OCR as a Chat Completions call to Vertex's OpenAPI endpoint and parses the transcript into OCR pages. Requests default to greedy decoding with a mild repetition penalty + +# references + +- https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/mistral +- https://cloud.google.com/vertex-ai/generative-ai/docs/maas/deepseek diff --git a/litellm-rust/crates/llms/src/vertex_ai/ocr/deepseek_transformation.rs b/litellm-rust/crates/llms/src/vertex_ai/ocr/deepseek_transformation.rs index 2c341eb684e..67f9122ed67 100644 --- a/litellm-rust/crates/llms/src/vertex_ai/ocr/deepseek_transformation.rs +++ b/litellm-rust/crates/llms/src/vertex_ai/ocr/deepseek_transformation.rs @@ -16,10 +16,9 @@ use litellm_llms_types::formats::ocr::{ LiteLLMOcrResponse, OcrDocument, OcrPage, OcrPageDimensions, OcrPageImage, OcrResponseFormat, OcrUsageInfo, }; +use litellm_llms_types::providers::vertex_ai::{DEFAULT_LOCATION, GLOBAL_API_BASE}; -const DEFAULT_API_BASE: &str = "https://aiplatform.googleapis.com"; const MODEL_PREFIX: &str = "deepseek-ai/"; -const DEFAULT_LOCATION: &str = "us-central1"; const DEEPSEEK_OCR_PARAMS: &[&str] = &["stream", "temperature", "max_tokens", "top_p", "n", "stop"]; /// DeepSeek-OCR is a transcription model: at the endpoint's default sampling temperature it @@ -396,7 +395,7 @@ impl VertexAIDeepSeekOCRConfig { let base = api_base .map(str::trim) .filter(|base| !base.is_empty()) - .unwrap_or(DEFAULT_API_BASE); + .unwrap_or(GLOBAL_API_BASE); ApiUrl::parse(base) .and_then(|url| { url.complete_path(&[ diff --git a/litellm-rust/crates/llms/src/vertex_ai/ocr/transformation.rs b/litellm-rust/crates/llms/src/vertex_ai/ocr/transformation.rs index bfd7b530ffa..ab28ee8bff4 100644 --- a/litellm-rust/crates/llms/src/vertex_ai/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/vertex_ai/ocr/transformation.rs @@ -16,7 +16,7 @@ use crate::{ }; use litellm_llms_types::formats::ocr::{LiteLLMOcrResponse, OcrDocument, OcrResponseFormat}; -const DEFAULT_LOCATION: &str = "us-central1"; +use litellm_llms_types::providers::vertex_ai::DEFAULT_LOCATION; #[derive(Clone, Debug, Default)] pub struct VertexAiOcrConfig; diff --git a/litellm-rust/crates/llms/tests/anthropic_beta_headers.rs b/litellm-rust/crates/llms/tests/anthropic_beta_headers.rs deleted file mode 100644 index 422c3d4b5f0..00000000000 --- a/litellm-rust/crates/llms/tests/anthropic_beta_headers.rs +++ /dev/null @@ -1,111 +0,0 @@ -use litellm_llms::anthropic::beta_headers::BetaPolicy; -use litellm_llms_types::providers::anthropic::BetaProvider; -use rstest::rstest; - -fn headers(pairs: &[(&str, &str)]) -> Vec<(String, String)> { - pairs - .iter() - .map(|(name, value)| (name.to_string(), value.to_string())) - .collect() -} - -#[rstest] -fn forward_keeps_headers_as_is() { - let input = headers(&[ - ( - "Anthropic-Beta", - "example-beta-2099-01-01,web-search-2025-03-05", - ), - ("x-api-key", "k"), - ]); - assert_eq!(BetaPolicy::Forward.apply(input.clone()), input); -} - -#[rstest] -fn drop_removes_every_casing() { - assert_eq!( - BetaPolicy::Drop.apply(headers(&[ - ("anthropic-beta", "a"), - ("x-api-key", "k"), - ("ANTHROPIC-BETA", "b"), - ])), - headers(&[("x-api-key", "k")]) - ); -} - -#[rstest] -#[case::bedrock(BetaProvider::Bedrock, Some("tool-search-tool-2025-10-19"))] -#[case::bedrock_mantle(BetaProvider::BedrockMantle, Some("tool-search-tool-2025-10-19"))] -#[case::vertex_ai(BetaProvider::VertexAi, Some("tool-search-tool-2025-10-19"))] -#[case::anthropic(BetaProvider::Anthropic, Some("advanced-tool-use-2025-11-20"))] -#[case::azure_ai(BetaProvider::AzureAi, Some("advanced-tool-use-2025-11-20"))] -#[case::databricks(BetaProvider::Databricks, Some("advanced-tool-use-2025-11-20"))] -#[case::bedrock_converse(BetaProvider::BedrockConverse, None)] -fn filter_renames_per_host(#[case] provider: BetaProvider, #[case] expected_beta: Option<&str>) { - let expected = match expected_beta { - Some(beta) => headers(&[("x-api-key", "k"), ("anthropic-beta", beta)]), - None => headers(&[("x-api-key", "k")]), - }; - assert_eq!( - BetaPolicy::Filter(provider).apply(headers(&[ - ("Anthropic-Beta", "advanced-tool-use-2025-11-20"), - ("x-api-key", "k"), - ])), - expected - ); -} - -#[rstest] -#[case::azure_ai(BetaProvider::AzureAi, "fast-mode-2026-02-01,example-beta-2099-01-01")] -#[case::anthropic(BetaProvider::Anthropic, "bash_20241022")] -#[case::vertex_ai(BetaProvider::VertexAi, " , ")] -fn filter_removes_header_when_nothing_survives( - #[case] provider: BetaProvider, - #[case] beta_value: &str, -) { - assert_eq!( - BetaPolicy::Filter(provider).apply(headers(&[ - ("x-api-key", "k"), - ("anthropic-beta", beta_value) - ])), - headers(&[("x-api-key", "k")]) - ); -} - -#[rstest] -#[case::anthropic( - BetaProvider::Anthropic, - "web-search-2025-03-05", - "oauth-2025-04-20,web-search-2025-03-05,example-beta-2099-01-01", - "oauth-2025-04-20,web-search-2025-03-05" -)] -#[case::vertex_ai_renames_and_deduplicates( - BetaProvider::VertexAi, - "advanced-tool-use-2025-11-20", - "tool-search-tool-2025-10-19", - "tool-search-tool-2025-10-19" -)] -fn filter_deduplicates_and_sorts( - #[case] provider: BetaProvider, - #[case] first_beta: &str, - #[case] second_beta: &str, - #[case] expected_beta: &str, -) { - assert_eq!( - BetaPolicy::Filter(provider).apply(headers(&[ - ("ANTHROPIC-BETA", first_beta), - ("x-api-key", "k"), - ("anthropic-beta", second_beta), - ])), - headers(&[("x-api-key", "k"), ("anthropic-beta", expected_beta)]) - ); -} - -#[rstest] -#[case::forward(BetaPolicy::Forward)] -#[case::drop(BetaPolicy::Drop)] -#[case::filter(BetaPolicy::Filter(BetaProvider::Bedrock))] -fn headers_without_a_beta_are_untouched(#[case] policy: BetaPolicy) { - let input = headers(&[("x-api-key", "k"), ("anthropic-version", "2023")]); - assert_eq!(policy.apply(input.clone()), input); -} diff --git a/litellm-rust/crates/llms/tests/anthropic_chat_transformation.rs b/litellm-rust/crates/llms/tests/anthropic_chat_transformation.rs deleted file mode 100644 index 37e0ed80cc0..00000000000 --- a/litellm-rust/crates/llms/tests/anthropic_chat_transformation.rs +++ /dev/null @@ -1,455 +0,0 @@ -use litellm_llms::{ - Error, - anthropic::chat::transformation::ANTHROPIC_CHAT_COMPLETIONS_CONFIG, - base_llm::{ - auth::AuthScheme, - chat::transformation::{BaseConfig, ProviderChatResponseData, Unsupported}, - }, -}; -use litellm_llms_types::formats::chat_completions::{ChatCompletionsResponse, ChatMessage}; -use rstest::rstest; -use serde_json::{Map, Value, json}; - -fn messages(value: Value) -> Vec { - serde_json::from_value(value).expect("valid messages") -} - -fn params(value: Value) -> Map { - match value { - Value::Object(map) => map, - other => panic!("params must be an object, got {other}"), - } -} - -fn transform(model: &str, msgs: Value, opts: Value) -> Value { - ANTHROPIC_CHAT_COMPLETIONS_CONFIG - .transform_request(model, messages(msgs), params(opts)) - .expect("request transforms") - .body -} - -fn transform_response(body: Value) -> Result { - ANTHROPIC_CHAT_COMPLETIONS_CONFIG - .transform_response("claude-sonnet-4-5", ProviderChatResponseData { body }) -} - -fn reason(msgs: Value, opts: Value) -> Option { - ANTHROPIC_CHAT_COMPLETIONS_CONFIG.unsupported_reason(&messages(msgs), ¶ms(opts)) -} - -#[test] -fn builds_the_messages_body_python_builds() { - let body = transform( - "claude-sonnet-4-5", - json!([ - {"role": "system", "content": "be terse"}, - {"role": "user", "content": "hi"} - ]), - json!({"max_tokens": 128, "temperature": 0.2}), - ); - assert_eq!( - body, - json!({ - "model": "claude-sonnet-4-5", - "messages": [ - {"role": "user", "content": [{"type": "text", "text": "hi"}]} - ], - "system": [{"type": "text", "text": "be terse"}], - "max_tokens": 128, - "temperature": 0.2 - }) - ); -} - -#[test] -fn omits_system_when_no_system_message_is_present() { - let body = transform( - "claude-sonnet-4-5", - json!([{"role": "user", "content": "hi"}]), - json!({"max_tokens": 16}), - ); - assert!(body.get("system").is_none()); -} - -#[test] -fn merges_consecutive_turns_and_wraps_every_text_in_a_block() { - let body = transform( - "claude-sonnet-4-5", - json!([ - {"role": "user", "content": "one"}, - {"role": "user", "content": [{"type": "text", "text": "two"}]}, - {"role": "assistant", "content": "ack"} - ]), - json!({"max_tokens": 16}), - ); - assert_eq!( - body["messages"], - json!([ - {"role": "user", "content": [ - {"type": "text", "text": "one"}, - {"type": "text", "text": "two"} - ]}, - {"role": "assistant", "content": [{"type": "text", "text": "ack"}]} - ]) - ); -} - -#[test] -fn right_strips_a_trailing_assistant_prefill_like_python() { - let body = transform( - "claude-sonnet-4-5", - json!([ - {"role": "user", "content": "hi"}, - {"role": "assistant", "content": "Argentina "} - ]), - json!({"max_tokens": 16}), - ); - assert_eq!( - body["messages"][1]["content"][0]["text"], - json!("Argentina") - ); -} - -#[test] -fn passes_every_supported_param_through_untouched() { - let body = transform( - "claude-sonnet-4-5", - json!([{"role": "user", "content": "hi"}]), - json!({ - "max_tokens": 64, - "temperature": 0.1, - "top_p": 0.9, - "stop_sequences": ["STOP"] - }), - ); - assert_eq!(body["max_tokens"], json!(64)); - assert_eq!(body["temperature"], json!(0.1)); - assert_eq!(body["top_p"], json!(0.9)); - assert_eq!(body["stop_sequences"], json!(["STOP"])); -} - -#[test] -fn declines_top_k_because_python_gates_it_by_model_below_this_point() { - // `temperature` and `top_p` arrive already resolved, because - // `map_openai_params` applies `_apply_sampling_param` to them before the - // gate runs. `top_k` bypasses that and is gated inside `transform_request`, - // the function this route replaces, so forwarding it would send `top_k` to - // a model that removed sampling params and take a 400 after the call, where - // Python drops it and succeeds. - assert_eq!( - reason( - json!([{"role": "user", "content": "hi"}]), - json!({"top_k": 40}) - ), - Some(Unsupported("unrecognized request parameter")) - ); -} - -#[test] -fn declines_streaming_before_anything_else() { - assert_eq!( - reason( - json!([{"role": "user", "content": "hi"}]), - json!({"stream": true, "max_tokens": 16}) - ), - Some(Unsupported("streaming")) - ); -} - -#[test] -fn accepts_an_explicit_stream_false() { - assert_eq!( - reason( - json!([{"role": "user", "content": "hi"}]), - json!({"stream": false, "max_tokens": 16}) - ), - None - ); -} - -#[rstest] -#[case::tools(json!({"tools": []}))] -#[case::tool_choice(json!({"tool_choice": {"type": "auto"}}))] -#[case::thinking(json!({"thinking": {"type": "enabled"}}))] -#[case::system(json!({"system": "injected"}))] -#[case::metadata(json!({"metadata": {"user_id": "u1"}}))] -#[case::output_config(json!({"output_config": {"effort": "high"}}))] -fn declines_any_param_outside_the_allowlist(#[case] param: Value) { - assert_eq!( - reason(json!([{"role": "user", "content": "hi"}]), param.clone()), - Some(Unsupported("unrecognized request parameter")), - "expected {param} to decline" - ); -} - -#[test] -fn declines_tool_calls_tool_results_and_multimodal_content() { - assert_eq!( - reason( - json!([ - {"role": "user", "content": "hi"}, - {"role": "assistant", "content": null, "tool_calls": [ - {"id": "c1", "type": "function", - "function": {"name": "f", "arguments": "{}"}} - ]} - ]), - json!({}) - ), - Some(Unsupported("unrecognized message field")) - ); - assert_eq!( - reason( - json!([ - {"role": "user", "content": "hi"}, - {"role": "tool", "tool_call_id": "c1", "content": "ok"} - ]), - json!({}) - ), - Some(Unsupported("unrecognized message field")) - ); - assert_eq!( - reason( - json!([ - {"role": "user", "content": [ - {"type": "image_url", "image_url": {"url": "https://x/y.png"}} - ]}]), - json!({}) - ), - Some(Unsupported("non-text message content")) - ); - assert_eq!( - reason( - json!([ - {"role": "user", "content": [ - {"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}} - ]}]), - json!({}) - ), - Some(Unsupported("non-text message content")) - ); -} - -#[test] -fn declines_a_message_whose_content_list_is_empty() { - // An empty list passes every per-part check, so without this it would reach - // the provider as an empty `content` array and fail after the call rather - // than declining to Python before it. - assert_eq!( - reason(json!([{"role": "user", "content": []}]), json!({})), - Some(Unsupported("message without content")) - ); - assert_eq!( - reason( - json!([{"role": "user", "content": [{"type": "text", "text": "hi"}]}]), - json!({}) - ), - None - ); -} - -#[test] -fn declines_a_conversation_that_does_not_open_on_a_user_turn() { - assert_eq!( - reason( - json!([ - {"role": "system", "content": "be terse"}, - {"role": "assistant", "content": "prefill"} - ]), - json!({}) - ), - Some(Unsupported("conversation does not open on a user turn")) - ); -} - -#[test] -fn accepts_a_plain_text_conversation() { - assert_eq!( - reason( - json!([ - {"role": "system", "content": "be terse"}, - {"role": "user", "content": "hi"}, - {"role": "assistant", "content": "hello"}, - {"role": "user", "content": [{"type": "text", "text": "again"}]} - ]), - json!({"max_tokens": 16, "temperature": 0.5}) - ), - None - ); -} - -#[test] -fn normalizes_a_text_response_into_openai_shape() { - let response = transform_response(json!({ - "id": "msg_123", - "type": "message", - "role": "assistant", - "model": "claude-sonnet-4-5-20260101", - "content": [{"type": "text", "text": "hello"}, {"type": "text", "text": " there"}], - "stop_reason": "end_turn", - "stop_sequence": null, - "usage": {"input_tokens": 11, "output_tokens": 4} - })) - .expect("response transforms"); - - assert_eq!(response.model, "claude-sonnet-4-5-20260101"); - assert_eq!(response.choices.len(), 1); - assert_eq!(response.choices[0].index, 0); - assert_eq!(response.choices[0].message.role, "assistant"); - assert_eq!( - response.choices[0].message.content.as_deref(), - Some("hello there") - ); - assert_eq!(response.choices[0].finish_reason, "stop"); - assert_eq!(response.usage.prompt_tokens, 11); - assert_eq!(response.usage.completion_tokens, 4); - assert_eq!(response.usage.total_tokens, 15); -} - -#[test] -fn folds_cache_tokens_into_prompt_tokens_like_python() { - let response = transform_response(json!({ - "model": "claude-sonnet-4-5", - "content": [{"type": "text", "text": "hi"}], - "stop_reason": "end_turn", - "usage": { - "input_tokens": 10, - "output_tokens": 2, - "cache_read_input_tokens": 5, - "cache_creation_input_tokens": 3 - } - })) - .expect("response transforms"); - assert_eq!(response.usage.prompt_tokens, 18); - assert_eq!(response.usage.total_tokens, 20); - assert_eq!(response.usage.prompt_tokens_details.cached_tokens, 5); - assert_eq!( - response.usage.prompt_tokens_details.cache_creation_tokens, - 3 - ); - assert_eq!(response.usage.prompt_tokens_details.text_tokens, 10); -} - -#[test] -fn maps_max_tokens_stop_reason_to_length() { - let response = transform_response(json!({ - "model": "claude-sonnet-4-5", - "content": [{"type": "text", "text": "hi"}], - "stop_reason": "max_tokens", - "usage": {"input_tokens": 1, "output_tokens": 1} - })) - .expect("response transforms"); - assert_eq!(response.choices[0].finish_reason, "length"); -} - -#[test] -fn a_refusal_returns_the_completion_python_returns() { - // `refusal` is a stop_reason, not a content block type, so the content is - // ordinary text and this normalizes rather than declining. Python maps it - // to content_filter in _FINISH_REASON_MAP and returns the completion. - let response = transform_response(json!({ - "model": "claude-sonnet-4-5", - "content": [{"type": "text", "text": "I can't help with that."}], - "stop_reason": "refusal", - "usage": {"input_tokens": 9, "output_tokens": 6} - })) - .expect("a refusal still transforms"); - assert_eq!(response.choices[0].finish_reason, "content_filter"); - assert_eq!( - response.choices[0].message.content.as_deref(), - Some("I can't help with that.") - ); -} - -#[test] -fn reports_no_content_rather_than_an_empty_string() { - let response = transform_response(json!({ - "model": "claude-sonnet-4-5", - "content": [], - "stop_reason": "end_turn", - "usage": {"input_tokens": 1, "output_tokens": 0} - })) - .expect("response transforms"); - assert_eq!(response.choices[0].message.content, None); -} - -#[test] -fn response_carries_no_id_so_python_keeps_its_chatcmpl_id() { - let response = transform_response(json!({ - "id": "msg_should_not_leak", - "model": "claude-sonnet-4-5", - "content": [{"type": "text", "text": "hi"}], - "stop_reason": "end_turn", - "usage": {"input_tokens": 1, "output_tokens": 1} - })) - .expect("response transforms"); - let value = serde_json::to_value(response).expect("serializable"); - assert!( - value.get("id").is_none(), - "the rust response must not carry an id, got {value}" - ); -} - -#[test] -fn declines_a_response_carrying_a_non_text_block() { - let err = transform_response(json!({ - "model": "claude-sonnet-4-5", - "content": [{"type": "tool_use", "id": "t1", "name": "f", "input": {}}], - "stop_reason": "tool_use", - "usage": {"input_tokens": 1, "output_tokens": 1} - })) - .expect_err("non-text block"); - assert_eq!(err, Error::Unsupported("non-text response content block")); -} - -#[rstest::rstest] -#[case::not_an_object(json!("nope"))] -#[case::missing_content(json!({"model": "test-model", "usage": {"input_tokens": 1, "output_tokens": 1}}))] -#[case::missing_usage(json!({"model": "test-model", "content": []}))] -#[case::missing_model(json!({"content": [], "usage": {"input_tokens": 1, "output_tokens": 1}}))] -fn errors_on_a_response_missing_required_fields(#[case] body: Value) { - let error = transform_response(body).expect_err("invalid response"); - assert!(matches!(error, Error::InvalidResponse(_))); - let source = std::iter::successors(Some(&error as &dyn std::error::Error), |error| { - error.source() - }) - .find_map(|error| error.downcast_ref::()) - .expect("the JSON decoding source is preserved"); - assert_eq!( - error.to_string(), - format!("invalid response: invalid messages response: {source}") - ); -} - -#[test] -fn resolves_the_messages_url_and_x_api_key_auth() { - let config = &ANTHROPIC_CHAT_COMPLETIONS_CONFIG; - assert_eq!( - config - .get_complete_url(None, "claude-sonnet-4-5", &Map::new(), &|_| None) - .expect("url builds"), - "https://api.anthropic.com/v1/messages" - ); - let validated = config - .validate_environment( - Vec::new(), - Some("sk-x"), - "claude-sonnet-4-5", - &Map::new(), - &|_| None, - ) - .expect("auth resolves"); - assert!(matches!( - validated.auth, - AuthScheme::Credential { - placement: litellm_auth::CredentialPlacement::Header("x-api-key"), - ref secret - } if secret.expose() == "sk-x" - )); - assert_eq!( - config.default_headers(), - &[ - ("anthropic-version", "2023-06-01"), - ("content-type", "application/json"), - ] - ); -} diff --git a/litellm-rust/crates/llms/tests/bedrock_converse_transformation.rs b/litellm-rust/crates/llms/tests/bedrock_converse_transformation.rs deleted file mode 100644 index dc3590c384f..00000000000 --- a/litellm-rust/crates/llms/tests/bedrock_converse_transformation.rs +++ /dev/null @@ -1,615 +0,0 @@ -use litellm_auth::CredentialPlacement; -use litellm_llms::{ - Error, - base_llm::{ - auth::AuthScheme, - chat::transformation::{BaseConfig, ProviderChatResponseData, Unsupported}, - }, - bedrock::chat::converse_transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG, -}; -use litellm_llms_types::formats::chat_completions::{ChatCompletionsResponse, ChatMessage}; -use rstest::rstest; -use serde_json::{Map, Value, json}; - -fn messages(value: Value) -> Vec { - serde_json::from_value(value).expect("valid messages") -} - -fn params(value: Value) -> Map { - match value { - Value::Object(map) => map, - other => panic!("params must be an object, got {other}"), - } -} - -fn transform(msgs: Value, opts: Value) -> Value { - BEDROCK_CHAT_COMPLETIONS_CONFIG - .transform_request( - "anthropic.claude-sonnet-4-5-v1:0", - messages(msgs), - params(opts), - ) - .expect("request transforms") - .body -} - -fn transform_response(body: Value) -> Result { - BEDROCK_CHAT_COMPLETIONS_CONFIG.transform_response( - "anthropic.claude-sonnet-4-5-v1:0", - ProviderChatResponseData { body }, - ) -} - -fn reason(msgs: Value, opts: Value) -> Option { - BEDROCK_CHAT_COMPLETIONS_CONFIG.unsupported_reason(&messages(msgs), ¶ms(opts)) -} - -#[test] -fn builds_the_converse_body_python_builds() { - let body = transform( - json!([ - {"role": "system", "content": "be terse"}, - {"role": "user", "content": "hi"} - ]), - json!({"maxTokens": 128, "temperature": 0.2}), - ); - assert_eq!( - body, - json!({ - "inferenceConfig": {"maxTokens": 128, "temperature": 0.2}, - "messages": [{"role": "user", "content": [{"text": "hi"}]}], - "system": [{"text": "be terse"}] - }) - ); -} - -#[test] -fn always_emits_inference_config_even_when_empty() { - let body = transform(json!([{"role": "user", "content": "hi"}]), json!({})); - assert_eq!(body["inferenceConfig"], json!({})); - assert!(body.get("system").is_none()); -} - -#[test] -fn places_only_inference_params_in_inference_config() { - let body = transform( - json!([{"role": "user", "content": "hi"}]), - json!({ - "maxTokens": 64, - "temperature": 0.1, - "topP": 0.9, - "stopSequences": ["STOP"] - }), - ); - assert_eq!( - body["inferenceConfig"], - json!({"maxTokens": 64, "temperature": 0.1, "topP": 0.9, "stopSequences": ["STOP"]}) - ); - assert!(body.get("additionalModelRequestFields").is_none()); -} - -#[test] -fn merges_consecutive_user_turns_into_one_message() { - let body = transform( - json!([ - {"role": "user", "content": "one"}, - {"role": "user", "content": [{"type": "text", "text": "two"}]}, - {"role": "assistant", "content": "ack"}, - {"role": "user", "content": "three"} - ]), - json!({}), - ); - assert_eq!( - body["messages"], - json!([ - {"role": "user", "content": [{"text": "one"}, {"text": "two"}]}, - {"role": "assistant", "content": [{"text": "ack"}]}, - {"role": "user", "content": [{"text": "three"}]} - ]) - ); -} - -#[test] -fn declines_streaming() { - assert_eq!( - reason( - json!([{"role": "user", "content": "hi"}]), - json!({"stream": true}) - ), - Some(Unsupported("streaming")) - ); -} - -#[test] -fn declines_top_k_because_python_routes_it_by_base_model() { - assert_eq!( - reason( - json!([{"role": "user", "content": "hi"}]), - json!({"topK": 40}) - ), - Some(Unsupported("unrecognized request parameter")) - ); -} - -#[rstest] -#[case::tools(json!({"tools": []}))] -#[case::tool_choice(json!({"tool_choice": {"auto": {}}}))] -#[case::thinking(json!({"thinking": {"type": "enabled"}}))] -#[case::request_metadata(json!({"requestMetadata": {"k": "v"}}))] -#[case::output_config(json!({"outputConfig": {}}))] -#[case::parallel_tool_use_config(json!({"_parallel_tool_use_config": {}}))] -fn declines_tools_and_other_params_outside_the_allowlist(#[case] param: Value) { - assert_eq!( - reason(json!([{"role": "user", "content": "hi"}]), param.clone()), - Some(Unsupported("unrecognized request parameter")), - "expected {param} to decline" - ); -} - -#[rstest] -#[case::empty_string(json!(""))] -#[case::whitespace_string(json!(" "))] -#[case::whitespace_text_block(json!([{"type": "text", "text": " "}]))] -fn declines_blank_text_rather_than_substituting_the_anthropic_placeholder(#[case] content: Value) { - assert_eq!( - reason( - json!([{"role": "user", "content": content}, {"role": "user", "content": "hi"}]), - json!({}) - ), - Some(Unsupported("blank message text")), - "expected blank content {content} to decline" - ); -} - -#[test] -fn declines_a_message_whose_content_list_is_empty() { - // The blank-text check scans parts, so an empty list clears it; Converse - // rejects an empty `content` array, which is a decline the core owes the - // host before the call rather than an error after it. - assert_eq!( - reason(json!([{"role": "user", "content": []}]), json!({})), - Some(Unsupported("message without content")) - ); - assert_eq!( - reason( - json!([{"role": "user", "content": [{"type": "text", "text": "hi"}]}]), - json!({}) - ), - None - ); -} - -#[test] -fn declines_a_conversation_that_opens_or_closes_on_an_assistant_turn() { - assert_eq!( - reason( - json!([ - {"role": "assistant", "content": "prefill"}, - {"role": "user", "content": "hi"} - ]), - json!({}) - ), - Some(Unsupported( - "conversation does not run user turn to user turn" - )) - ); - assert_eq!( - reason( - json!([ - {"role": "user", "content": "hi"}, - {"role": "assistant", "content": "prefill"} - ]), - json!({}) - ), - Some(Unsupported( - "conversation does not run user turn to user turn" - )) - ); -} - -#[test] -fn accepts_a_user_to_user_text_conversation() { - assert_eq!( - reason( - json!([ - {"role": "system", "content": "be terse"}, - {"role": "user", "content": "hi"}, - {"role": "assistant", "content": "hello"}, - {"role": "user", "content": "again"} - ]), - json!({"maxTokens": 16}) - ), - None - ); -} - -#[test] -fn builds_the_converse_url_from_the_region_in_the_model_id() { - let config = &BEDROCK_CHAT_COMPLETIONS_CONFIG; - assert_eq!( - config - .get_complete_url(None, "us-east-1/anthropic.claude-v2", &Map::new(), &|_| { - None - }) - .expect("url builds"), - "https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-v2/converse" - ); -} - -#[test] -fn falls_back_to_the_region_env_then_the_default_region() { - let config = &BEDROCK_CHAT_COMPLETIONS_CONFIG; - let with_env = |key: &str| (key == "AWS_REGION_NAME").then(|| "eu-west-1".to_string()); - assert_eq!( - config - .get_complete_url(None, "anthropic.claude-v2", &Map::new(), &with_env) - .expect("url builds"), - "https://bedrock-runtime.eu-west-1.amazonaws.com/model/anthropic.claude-v2/converse" - ); - assert_eq!( - config - .get_complete_url(None, "anthropic.claude-v2", &Map::new(), &|_| None) - .expect("url builds"), - "https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-v2/converse" - ); -} - -#[test] -fn prefers_an_explicit_runtime_endpoint_over_the_api_base() { - let config = &BEDROCK_CHAT_COMPLETIONS_CONFIG; - let overrides = params(json!({"aws_bedrock_runtime_endpoint": "https://vpce.internal/"})); - assert_eq!( - config - .get_complete_url( - Some("https://ignored.example"), - "anthropic.claude-v2", - &overrides, - &|_| None - ) - .expect("url builds"), - "https://vpce.internal/model/anthropic.claude-v2/converse" - ); -} - -/// The bearer token a config named, or `None` for a SigV4 scheme in the given region. -fn bearer_or_region(auth: AuthScheme) -> Result { - match auth { - AuthScheme::Credential { - placement: CredentialPlacement::Bearer, - secret, - } => Ok(secret.expose().to_string()), - AuthScheme::AwsSigV4 { - region, - service: "bedrock", - .. - } => Err(region), - other => panic!("unexpected auth {other:?}"), - } -} - -#[test] -fn signs_with_sigv4_in_the_resolved_region() { - let validated = BEDROCK_CHAT_COMPLETIONS_CONFIG - .validate_environment( - Vec::new(), - None, - "eu-central-1/anthropic.claude-v2", - &Map::new(), - &|_| None, - ) - .expect("auth resolves"); - assert_eq!( - bearer_or_region(validated.auth), - Err("eu-central-1".to_string()) - ); -} - -#[test] -fn a_bearer_token_outranks_sigv4_the_way_python_resolves_it() { - // Python's get_request_headers reads `api_key` as the Bedrock bearer token - // and only falls back to the env when the caller passed none, so each case - // pins one of its precedence rules. Signing as the host principal when a - // bearer identity is configured would cross an account and quota boundary. - let bedrock_env = - |key: &str| (key == "AWS_BEARER_TOKEN_BEDROCK").then(|| "from-env".to_string()); - let no_env = |_: &str| None; - let resolve = |api_key, env: &dyn Fn(&str) -> Option| { - bearer_or_region( - BEDROCK_CHAT_COMPLETIONS_CONFIG - .validate_environment( - Vec::new(), - api_key, - "eu-central-1/anthropic.claude-v2", - &Map::new(), - env, - ) - .expect("auth resolves") - .auth, - ) - }; - let bearer = |token: &str| Ok(token.to_string()); - let sigv4 = Err("eu-central-1".to_string()); - - // A caller-supplied key is the bearer token, and outranks the env. - assert_eq!( - resolve(Some("bedrock-api-key"), &bedrock_env), - bearer("bedrock-api-key") - ); - // No key, so the env supplies it. - assert_eq!(resolve(None, &bedrock_env), bearer("from-env")); - // An empty key is not a bearer token, and deliberately does NOT reach for - // the env, which is what Python's `is not None` check does. - assert_eq!(resolve(Some(""), &bedrock_env), sigv4); - // Whitespace is truthy in Python, so it stays a bearer token rather than - // silently becoming a host-credentialed SigV4 request. - assert_eq!(resolve(Some(" "), &no_env), bearer(" ")); - // Neither present, so SigV4 as before. - assert_eq!(resolve(None, &no_env), sigv4); -} - -#[test] -fn normalizes_a_converse_response_into_openai_shape() { - let response = transform_response(json!({ - "output": {"message": {"role": "assistant", "content": [ - {"text": "hello"}, {"text": " there"} - ]}}, - "stopReason": "end_turn", - "usage": {"inputTokens": 11, "outputTokens": 4, "totalTokens": 15} - })) - .expect("response transforms"); - - assert_eq!(response.model, "anthropic.claude-sonnet-4-5-v1:0"); - assert_eq!( - response.choices[0].message.content.as_deref(), - Some("hello there") - ); - assert_eq!(response.choices[0].finish_reason, "stop"); - assert_eq!(response.usage.prompt_tokens, 11); - assert_eq!(response.usage.completion_tokens, 4); - assert_eq!(response.usage.total_tokens, 15); -} - -#[test] -fn maps_converse_stop_reasons_python_maps() { - for (provider_reason, expected) in [ - ("end_turn", "stop"), - ("stop_sequence", "stop"), - ("max_tokens", "length"), - ("guardrail_intervened", "content_filter"), - // Converse emits this one, and Python's `_FINISH_REASON_MAP` carries - // it. Folding it into `stop` reports a filtered completion as a normal - // one to anything keying on the finish reason. - ("content_filtered", "content_filter"), - ("content_filter", "content_filter"), - ] { - let response = transform_response(json!({ - "output": {"message": {"content": [{"text": "x"}]}}, - "stopReason": provider_reason, - "usage": {"inputTokens": 1, "outputTokens": 1} - })) - .expect("response transforms"); - assert_eq!( - response.choices[0].finish_reason, expected, - "stopReason {provider_reason}" - ); - } -} - -#[test] -fn reports_an_empty_converse_answer_as_an_empty_string_not_null() { - // Converse assigns the joined text unconditionally - // (`chat_completion_message["content"] = content_str`), unlike Anthropic's - // `merged_text or None`, so an empty answer is `""` on both paths. A caller - // calling `.strip()` on it would break on the Rust path alone. Reachable - // through a filtered or guardrail-intervened response. - for content in [json!([]), json!([{"text": ""}])] { - let response = transform_response(json!({ - "output": {"message": {"content": content}}, - "stopReason": "content_filtered", - "usage": {"inputTokens": 1, "outputTokens": 0} - })) - .expect("response transforms"); - assert_eq!(response.choices[0].message.content, Some(String::new())); - } -} - -#[test] -fn reports_the_total_tokens_converse_sent_rather_than_recomputing_them() { - // Python reads `usage["totalTokens"]` straight through here, where Anthropic - // has no such field and adds the two counts instead. The two agree while the - // gate declines every cache_control request, so this is what keeps them - // agreeing if that ever widens. - let response = transform_response(json!({ - "output": {"message": {"content": [{"text": "x"}]}}, - "stopReason": "end_turn", - "usage": {"inputTokens": 10, "outputTokens": 4, "cacheReadInputTokens": 7, "totalTokens": 14} - })) - .expect("response transforms"); - assert_eq!( - response.usage.total_tokens, 14, - "provider total was recomputed" - ); - assert_eq!(response.usage.prompt_tokens, 17); - assert_eq!(response.usage.completion_tokens, 4); -} - -#[test] -fn falls_back_to_the_computed_total_when_converse_omits_it() { - // Python raises a KeyError on a body with no `totalTokens`. Reporting a zero - // instead would be a worse divergence than the one above, so the computed - // total stands in. - let response = transform_response(json!({ - "output": {"message": {"content": [{"text": "x"}]}}, - "stopReason": "end_turn", - "usage": {"inputTokens": 10, "outputTokens": 4} - })) - .expect("response transforms"); - assert_eq!(response.usage.total_tokens, 14); -} - -#[test] -fn declines_a_cache_control_message_so_widening_the_gate_is_a_red_test() { - // Converse only reports cache token counts when the request carries a - // cachePoint block, which is why the provider total and the computed one - // cannot disagree today. This is the tripwire: whoever widens the gate to - // admit prompt caching has to come back and re-check the usage mapping - // rather than discovering a silent number change in production. - assert_eq!( - reason( - json!([{"role": "user", "content": [ - {"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}} - ]}]), - json!({}) - ), - Some(Unsupported("non-text message content")) - ); -} - -#[test] -fn folds_converse_cache_tokens_into_prompt_tokens() { - let response = transform_response(json!({ - "output": {"message": {"content": [{"text": "x"}]}}, - "stopReason": "end_turn", - "usage": { - "inputTokens": 10, - "outputTokens": 2, - "cacheReadInputTokens": 5, - "cacheWriteInputTokens": 3 - } - })) - .expect("response transforms"); - assert_eq!(response.usage.prompt_tokens, 18); - assert_eq!(response.usage.prompt_tokens_details.cached_tokens, 5); - assert_eq!( - response.usage.prompt_tokens_details.cache_creation_tokens, - 3 - ); - assert_eq!(response.usage.prompt_tokens_details.text_tokens, 10); -} - -#[test] -fn declines_a_response_carrying_a_tool_use_block() { - let err = transform_response(json!({ - "output": {"message": {"content": [ - {"toolUse": {"toolUseId": "t1", "name": "f", "input": {}}} - ]}}, - "stopReason": "tool_use", - "usage": {"inputTokens": 1, "outputTokens": 1} - })) - .expect_err("tool use block"); - assert_eq!(err, Error::Unsupported("non-text response content block")); -} - -#[test] -fn errors_on_a_response_missing_required_fields() { - assert_eq!( - transform_response(json!("nope")).expect_err("not an object"), - Error::InvalidResponse("converse response is not an object".to_string().into()) - ); - assert_eq!( - transform_response(json!({"usage": {}})).expect_err("no output"), - Error::InvalidResponse("invalid Converse response: missing field `output`".into()) - ); - assert_eq!( - transform_response(json!({"output": {"message": {"content": []}}})).expect_err("no usage"), - Error::InvalidResponse("invalid Converse response: missing field `usage`".into()) - ); -} - -#[test] -fn rejects_malformed_text_and_token_counts() { - for response in [ - json!({"output": {"message": {"content": [{"text": 123}]}}, "usage": {"inputTokens": 1, "outputTokens": 1}}), - json!({"output": {"message": {"content": [{"text": "x"}]}}, "usage": {"inputTokens": "1", "outputTokens": 1}}), - ] { - assert!(matches!( - transform_response(response), - Err(Error::InvalidResponse(message)) if message.to_string().starts_with("invalid Converse response:") - )); - } -} - -#[test] -fn accepts_aws_call_configuration_without_serializing_it() { - let call_config = json!({ - "maxTokens": 16, - "aws_access_key_id": "AKIA", - "aws_secret_access_key": "secret", - "aws_session_token": "token", - "aws_region_name": "us-east-1", - "aws_profile_name": "litellm-stage", - "aws_role_name": "role", - "aws_session_name": "session", - "aws_web_identity_token": "wit", - "aws_sts_endpoint": "https://sts.example", - "aws_external_id": "ext", - "aws_bedrock_runtime_endpoint": "https://vpce.internal" - }); - assert_eq!( - reason( - json!([{"role": "user", "content": "hi"}]), - call_config.clone() - ), - None - ); - let body = transform(json!([{"role": "user", "content": "hi"}]), call_config); - assert_eq!( - body, - json!({ - "inferenceConfig": {"maxTokens": 16}, - "messages": [{"role": "user", "content": [{"text": "hi"}]}] - }), - "aws call configuration must not reach the Converse body" - ); -} - -#[test] -fn leaves_a_complete_converse_url_untouched() { - let config = &BEDROCK_CHAT_COMPLETIONS_CONFIG; - let already_built = - "https://bedrock-runtime.us-east-1.amazonaws.com/model/us.anthropic.claude-v2%3A0/converse"; - assert_eq!( - config - .get_complete_url( - Some(already_built), - "anthropic.claude-v2", - &Map::new(), - &|_| None - ) - .expect("url builds"), - already_built, - "a host that encoded the model id itself must not have it re-derived" - ); -} - -#[rstest] -#[case::full_static_pair( - json!({"aws_access_key_id": "AKIAHOST", "aws_secret_access_key": "hostsecret", "aws_session_token": "hosttoken"}), - Some(("AKIAHOST", "hostsecret", Some("hosttoken"))) -)] -#[case::pair_without_session_token( - json!({"aws_access_key_id": "AKIAHOST", "aws_secret_access_key": "hostsecret"}), - Some(("AKIAHOST", "hostsecret", None)) -)] -#[case::key_id_alone(json!({"aws_access_key_id": "AKIA"}), None)] -#[case::blank_key_id(json!({"aws_access_key_id": " ", "aws_secret_access_key": "s"}), None)] -#[case::nothing(json!({}), None)] -fn host_supplied_credentials_need_a_full_static_pair( - #[case] optional_params: Value, - #[case] expected: Option<(&str, &str, Option<&str>)>, -) { - use litellm_auth::AwsParams; - use litellm_auth_aws::host_supplied_credentials; - - let credentials = - host_supplied_credentials(&AwsParams::from_optional_params(¶ms(optional_params))); - - assert_eq!( - credentials.as_ref().map(|credentials| ( - credentials.access_key_id(), - credentials.secret_access_key(), - credentials.session_token(), - )), - expected - ); -} diff --git a/litellm-rust/crates/llms/tests/messages_normalization.rs b/litellm-rust/crates/llms/tests/messages_normalization.rs deleted file mode 100644 index bd54618aef0..00000000000 --- a/litellm-rust/crates/llms/tests/messages_normalization.rs +++ /dev/null @@ -1,53 +0,0 @@ -use litellm_llms::base_llm::messages::normalization::normalize_system_role_messages; -use litellm_llms_types::formats::messages::MessagesRequest; -use rstest::rstest; -use serde_json::{Value, json}; - -#[rstest] -#[case::text_system(json!("existing"), json!([{"type": "text", "text": "existing"}]))] -#[case::block_system(json!([{ "type": "future", "payload": 7 }]), json!([{ "type": "future", "payload": 7 }]))] -#[case::no_system(Value::Null, json!([]))] -fn hoisting_preserves_block_fields_order_and_unrelated_request_fields( - #[case] system: Value, - #[case] initial_blocks: Value, -) { - let cache_control = json!({"type": "ephemeral", "scope": "global", "future": true}); - let leading_block = json!({"type": "text", "text": "second", "cache_control": cache_control}); - let user = json!({"role": "user", "content": "hello", "future_message": 42}); - let later_turn = json!({"role": "system", "content": "mid-conversation reminder"}); - let request: MessagesRequest = serde_json::from_value(json!({ - "model": "test-model", - "max_tokens": 64, - "system": system, - "messages": [ - {"role": "system", "content": "first"}, - {"role": "system", "content": [leading_block]}, - user, - later_turn - ], - "future_request": {"nested": true} - })) - .unwrap(); - let normalized = normalize_system_role_messages(request, true); - let expected_blocks: Vec = initial_blocks - .as_array() - .unwrap() - .iter() - .cloned() - .chain([json!({"type": "text", "text": "first"}), leading_block]) - .collect(); - assert_eq!( - serde_json::to_value(&normalized).unwrap(), - json!({ - "model": "test-model", - "max_tokens": 64, - "system": expected_blocks, - "messages": [user, later_turn], - "future_request": {"nested": true} - }) - ); - assert_eq!( - normalize_system_role_messages(normalized.clone(), true), - normalized - ); -} diff --git a/litellm-rust/crates/llms/tests/ocr_handler.rs b/litellm-rust/crates/llms/tests/ocr_handler.rs deleted file mode 100644 index 5ed3244087d..00000000000 --- a/litellm-rust/crates/llms/tests/ocr_handler.rs +++ /dev/null @@ -1,79 +0,0 @@ -use std::time::Duration; - -use litellm_llms::base_llm::ocr::{error::Error, handler::read_response_bytes}; -use rstest::rstest; -use tokio::{ - io::{AsyncReadExt, AsyncWriteExt}, - net::TcpListener, -}; - -/// Answers one request with raw `response` bytes and then holds the connection open, so a -/// read that waits for the rest of an oversized body hangs instead of passing. -async fn read_bounded(response: String, limit: usize) -> Result { - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { - let (mut socket, _) = listener.accept().await.unwrap(); - let mut request = [0; 4096]; - assert!(socket.read(&mut request).await.unwrap() > 0); - socket.write_all(response.as_bytes()).await.unwrap(); - std::future::pending::<()>().await; - }); - let response = litellm_http::Client::plain_for_test() - .get(format!("http://{address}")) - .send() - .await - .unwrap(); - let result = - tokio::time::timeout(Duration::from_secs(2), read_response_bytes(response, limit)).await; - server.abort(); - result.expect("bounded reads must finish without waiting for the rest of an oversized body") -} - -#[rstest] -#[case::declared("HTTP/1.1 200 OK\r\nContent-Length: 8\r\n\r\nabcdefgh")] -#[case::chunked( - "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nabcd\r\n4\r\nefgh\r\n0\r\n\r\n" -)] -#[tokio::test] -async fn a_body_of_exactly_the_limit_is_read(#[case] response: &str) { - assert_eq!(read_bounded(response.into(), 8).await.unwrap(), "abcdefgh"); -} - -#[rstest] -#[case::declared("HTTP/1.1 200 OK\r\nContent-Length: 9\r\n\r\n")] -#[case::chunked("HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nabcd\r\n5\r\nefghi\r\n")] -#[tokio::test] -async fn a_body_over_the_limit_is_rejected(#[case] response: &str) { - assert!(matches!( - read_bounded(response.into(), 8).await, - Err(Error::TooLarge { limit: 8 }) - )); -} - -#[rstest] -#[case::declared("Content-Length: 1000000")] -#[case::chunked("Transfer-Encoding: chunked")] -#[tokio::test] -async fn an_oversized_error_keeps_its_status_and_a_bounded_body_without_draining( - #[case] headers: &str, -) { - let prefix = "x".repeat(4096); - let body = match headers.starts_with("Transfer") { - true => format!("{:x}\r\n{prefix}\r\n", prefix.len()), - false => prefix.clone(), - }; - - let error = read_bounded( - format!("HTTP/1.1 429 Too Many Requests\r\n{headers}\r\n\r\n{body}"), - prefix.len(), - ) - .await - .unwrap_err(); - - let Error::Transport(litellm_http::transport::Error::Http { status, body }) = error else { - panic!("unexpected error: {error}"); - }; - assert_eq!(status, 429); - assert_eq!(body, prefix); -} diff --git a/litellm-rust/crates/llms/tests/openai_like_chat_transformation.rs b/litellm-rust/crates/llms/tests/openai_like_chat_transformation.rs deleted file mode 100644 index 1c18873772b..00000000000 --- a/litellm-rust/crates/llms/tests/openai_like_chat_transformation.rs +++ /dev/null @@ -1,343 +0,0 @@ -use litellm_llms::{ - Error, - base_llm::{ - auth::AuthScheme, - chat::transformation::{BaseConfig, ProviderChatResponseData, Unsupported}, - }, - openai_like::chat::transformation::OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG, -}; -use litellm_llms_types::formats::chat_completions::{ChatCompletionsResponse, ChatMessage}; -use rstest::rstest; -use serde_json::{Map, Value, json}; - -fn messages(value: Value) -> Vec { - serde_json::from_value(value).expect("valid messages") -} - -fn params(value: Value) -> Map { - match value { - Value::Object(map) => map, - other => panic!("params must be an object, got {other}"), - } -} - -fn no_env(_: &str) -> Option { - None -} - -fn env_with<'a>(name: &'a str, value: &'a str) -> impl Fn(&str) -> Option + 'a { - move |key| (key == name).then(|| value.to_string()) -} - -fn transform(model: &str, msgs: Value, opts: Value) -> Value { - OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG - .transform_request(model, messages(msgs), params(opts)) - .expect("request transforms") - .body -} - -fn transform_response(body: Value) -> Result { - OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG - .transform_response("some-model", ProviderChatResponseData { body }) -} - -fn reason(msgs: Value, opts: Value) -> Option { - OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG.unsupported_reason(&messages(msgs), ¶ms(opts)) -} - -#[rstest] -fn builds_the_openai_shaped_body() { - let body = transform( - "my-model", - json!([ - {"role": "system", "content": "be terse"}, - {"role": "user", "content": "hi"}, - ]), - json!({"temperature": 0.5, "max_tokens": 8}), - ); - assert_eq!(body["model"], json!("my-model")); - assert_eq!( - body["messages"], - json!([ - {"role": "system", "content": "be terse"}, - {"role": "user", "content": "hi"}, - ]) - ); - assert_eq!(body["temperature"], json!(0.5)); - assert_eq!(body["max_tokens"], json!(8)); -} - -#[rstest] -fn renames_max_completion_tokens_to_max_tokens() { - // `OpenAILikeChatConfig.map_openai_params`: most OpenAI-compatible providers - // support `max_tokens`, not `max_completion_tokens`. - let body = transform( - "my-model", - json!([{"role": "user", "content": "hi"}]), - json!({"max_completion_tokens": 12}), - ); - assert_eq!(body["max_tokens"], json!(12)); - assert!(body.get("max_completion_tokens").is_none()); -} - -#[rstest] -fn max_completion_tokens_wins_when_both_limits_are_sent() { - // Python assigns `max_tokens = max_completion_tokens` after copying the - // params, so the renamed value outranks a caller-supplied `max_tokens`. - let body = transform( - "my-model", - json!([{"role": "user", "content": "hi"}]), - json!({"max_tokens": 8, "max_completion_tokens": 12}), - ); - assert_eq!(body["max_tokens"], json!(12)); - assert!(body.get("max_completion_tokens").is_none()); -} - -#[rstest] -fn call_configuration_never_enters_the_body() { - let body = transform( - "my-model", - json!([{"role": "user", "content": "hi"}]), - json!({"custom_endpoint": true, "extra_headers": {"x": "y"}, "max_retries": 2}), - ); - assert_eq!( - body.as_object().unwrap().keys().collect::>(), - vec!["model", "messages"] - ); -} - -#[rstest] -#[case::appends_the_chat_completions_suffix("https://vllm.example.com/v1", json!({}), "https://vllm.example.com/v1/chat/completions")] -#[case::trims_a_trailing_slash("https://vllm.example.com/v1/", json!({}), "https://vllm.example.com/v1/chat/completions")] -#[case::a_custom_endpoint_is_used_as_is("https://vllm.example.com/v1/chat/completions", json!({"custom_endpoint": true}), "https://vllm.example.com/v1/chat/completions")] -fn complete_url(#[case] api_base: &str, #[case] opts: Value, #[case] expected: &str) { - assert_eq!( - OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG - .get_complete_url(Some(api_base), "my-model", ¶ms(opts), &no_env) - .expect("url resolves"), - expected - ); -} - -#[rstest] -fn api_base_falls_back_to_the_environment() { - assert_eq!( - OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG - .get_complete_url( - None, - "my-model", - ¶ms(json!({})), - &env_with("OPENAI_LIKE_API_BASE", "https://env.example.com/v1"), - ) - .expect("url resolves"), - "https://env.example.com/v1/chat/completions" - ); -} - -#[rstest] -fn a_missing_api_base_is_an_error() { - assert!(matches!( - OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG - .get_complete_url(None, "my-model", ¶ms(json!({})), &no_env), - Err(Error::InvalidRequest(message)) if message.to_string().starts_with("Missing API Base") - )); -} - -#[rstest] -fn the_resolved_key_authenticates_as_a_bearer() { - let validated = OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG - .validate_environment( - vec![], - Some("sk-test"), - "my-model", - ¶ms(json!({})), - &no_env, - ) - .expect("validates"); - assert!(matches!( - validated.auth, - AuthScheme::Credential { - placement: litellm_auth::CredentialPlacement::Bearer, - .. - } - )); -} - -#[rstest] -fn the_key_falls_back_to_the_environment() { - let validated = OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG - .validate_environment( - vec![], - None, - "my-model", - ¶ms(json!({})), - &env_with("OPENAI_LIKE_API_KEY", "sk-env"), - ) - .expect("validates"); - let AuthScheme::Credential { secret, .. } = validated.auth else { - panic!("expected a bearer credential"); - }; - assert_eq!(secret.expose(), "sk-env"); -} - -#[rstest] -fn a_forwarded_authorization_is_the_whole_credential() { - // Python adds `Bearer ` only when the caller did not already send - // `Authorization`, so the forwarded header wins over the deployment key. - let headers = vec![("Authorization".to_string(), "Bearer caller".to_string())]; - let validated = OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG - .validate_environment( - headers, - Some("sk-test"), - "my-model", - ¶ms(json!({})), - &no_env, - ) - .expect("validates"); - assert!(matches!(validated.auth, AuthScheme::Forwarded)); -} - -#[rstest] -fn keyless_calls_still_validate_for_endpoints_that_take_no_key() { - // vllm-compatible endpoints require no api key; Python resolves `""` and - // sends `Bearer `, so validation must not fail on the missing key. - let validated = OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG - .validate_environment(vec![], None, "my-model", ¶ms(json!({})), &no_env) - .expect("validates"); - let AuthScheme::Credential { secret, .. } = validated.auth else { - panic!("expected a bearer credential"); - }; - assert_eq!(secret.expose(), ""); -} - -#[rstest] -fn normalizes_an_openai_response() { - let response = transform_response(json!({ - "created": 1_700_000_000, - "model": "served-model-name", - "choices": [{ - "index": 0, - "message": {"role": "assistant", "content": "hello"}, - "finish_reason": "stop", - }], - "usage": {"prompt_tokens": 3, "completion_tokens": 5, "total_tokens": 8}, - })) - .expect("response normalizes"); - assert_eq!(response.created, 1_700_000_000); - assert_eq!(response.model, "served-model-name"); - assert_eq!( - response.choices[0].message.content.as_deref(), - Some("hello") - ); - assert_eq!(response.choices[0].finish_reason, "stop"); - assert_eq!(response.usage.prompt_tokens, 3); - assert_eq!(response.usage.completion_tokens, 5); - assert_eq!(response.usage.total_tokens, 8); -} - -#[rstest] -fn null_token_fields_in_usage_become_zero() { - // `_sanitize_usage_obj`: providers that return null token values break - // OpenAI clients, so the response is scrubbed at the source. - let response = transform_response(json!({ - "model": "m", - "choices": [{"message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], - "usage": {"prompt_tokens": 3, "completion_tokens": null, "total_tokens": null}, - })) - .expect("response normalizes"); - assert_eq!(response.usage.completion_tokens, 0); - assert_eq!(response.usage.total_tokens, 0); - assert_eq!(response.usage.prompt_tokens, 3); -} - -#[rstest] -fn a_tool_call_response_declines_instead_of_dropping_the_calls() { - // The `json_mode` rewrite needs a request flag the route does not carry, so - // a tool-call answer falls back to Python rather than losing the calls. - assert_eq!( - transform_response(json!({ - "model": "m", - "choices": [{ - "message": { - "role": "assistant", - "content": null, - "tool_calls": [{ - "id": "call_1", - "type": "function", - "function": {"name": "f", "arguments": "{}"}, - }], - }, - "finish_reason": "tool_calls", - }], - })), - Err(Error::Unsupported("tool call response")) - ); -} - -#[rstest] -fn a_refusal_declines_instead_of_returning_an_empty_reply() { - assert_eq!( - transform_response(json!({ - "model": "m", - "choices": [{ - "message": {"role": "assistant", "content": null, "refusal": "cannot help"}, - "finish_reason": "stop", - }], - })), - Err(Error::Unsupported("refusal response")) - ); -} - -#[rstest] -fn a_non_text_response_content_declines() { - assert_eq!( - transform_response(json!({ - "model": "m", - "choices": [{ - "message": {"role": "assistant", "content": [{"type": "text", "text": "hi"}]}, - "finish_reason": "stop", - }], - })), - Err(Error::Unsupported("non-text response content")) - ); -} - -#[rstest] -#[case::streaming(json!({"stream": true}), "streaming")] -#[case::unrecognized_param(json!({"some_provider_knob": 1}), "unrecognized request parameter")] -fn declines(#[case] opts: Value, #[case] expected: &'static str) { - assert_eq!( - reason(json!([{"role": "user", "content": "hi"}]), opts), - Some(Unsupported(expected)) - ); -} - -#[rstest] -fn accepts_standard_openai_params() { - assert_eq!( - reason( - json!([{"role": "user", "content": "hi"}]), - json!({ - "temperature": 0.2, - "top_p": 0.9, - "max_tokens": 16, - "response_format": {"type": "json_object"}, - "custom_endpoint": true, - }), - ), - None - ); -} - -#[rstest] -fn tool_parameters_decline_before_the_call() { - // A `tools` request would come back with tool calls this port cannot - // normalize, so it declines at the gate instead of after the call. - assert_eq!( - reason( - json!([{"role": "user", "content": "hi"}]), - json!({"tools": [{"type": "function", "function": {"name": "f"}}]}), - ), - Some(Unsupported("unrecognized request parameter")) - ); -} diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index faba4f5eb20..42f605443c7 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -29,9 +29,10 @@ from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( ) from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info from litellm.llms.custom_httpx.http_handler import get_httpx_client +from litellm.responses.utils import ResponseAPILoggingUtils from litellm.secret_managers.main import str_to_bool from litellm.types.integrations.langfuse import * -from litellm.types.llms.openai import HttpxBinaryResponseContent, ResponsesAPIResponse +from litellm.types.llms.openai import HttpxBinaryResponseContent, ResponseAPIUsage, ResponsesAPIResponse from litellm.types.utils import ( EmbeddingResponse, ImageResponse, @@ -108,6 +109,27 @@ class _UsageObject(Protocol): def get(self, key: Literal["cache_creation_input_tokens", "cache_read_input_tokens"], /) -> int | None: ... +def _chat_usage(raw_usage: _UsageObject | None) -> _UsageObject | None: + """Chat-shaped view of a response's usage. + + /v1/responses carries ResponseAPIUsage (input_tokens/output_tokens), and an + assembled /v1/responses stream carries a plain dict of chat usage fields. + Any other usage object, or a dict whose token counts are not integers, is + returned unchanged. + """ + if isinstance(raw_usage, ResponseAPIUsage): + return ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(raw_usage) + usage_fields: Final = _object_mapping(raw_usage) + if usage_fields is None or not ( + ResponseAPILoggingUtils.is_response_api_usage(usage_fields) or "prompt_tokens" in usage_fields + ): + return raw_usage + try: + return ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(usage_fields) + except ValueError: + return raw_usage + + def _extract_cache_read_input_tokens(usage_obj) -> int: """ Extract cache_read_input_tokens from usage object. @@ -815,7 +837,8 @@ class LangFuseLogger: if response_obj is not None: if hasattr(response_obj, "id") and response_obj.get("id", None) is not None: generation_id = _logging_id(start_time, response_obj) - _usage_obj: Final[_UsageObject | None] = getattr(response_obj, "usage", None) + _raw_usage_obj: Final[_UsageObject | None] = getattr(response_obj, "usage", None) + _usage_obj: Final = _chat_usage(_raw_usage_obj) if _usage_obj: # Safely get usage values, defaulting None to 0 for Langfuse compatibility. diff --git a/litellm/litellm_core_utils/get_blog_posts.py b/litellm/litellm_core_utils/get_blog_posts.py index 276d2a968a1..0dd09064dcc 100644 --- a/litellm/litellm_core_utils/get_blog_posts.py +++ b/litellm/litellm_core_utils/get_blog_posts.py @@ -22,6 +22,7 @@ from litellm import verbose_logger from litellm.types.llms.base import LiteLLMBaseModel BLOG_POSTS_TTL_SECONDS: Final[int] = 3600 # 1 hour +BLOG_POSTS_MAX: Final[int] = 6 class BlogPost(LiteLLMBaseModel): @@ -40,7 +41,7 @@ class GetBlogPosts: Fetches, validates, and caches LiteLLM blog posts. - Fetches RSS feed from docs site with a 5-second timeout - - Parses the XML and extracts the latest blog post + - Parses the XML and extracts the latest BLOG_POSTS_MAX blog posts - Caches the result in-process for BLOG_POSTS_TTL_SECONDS (1 hour) - Falls back to the bundled local backup on any failure """ @@ -66,7 +67,7 @@ class GetBlogPosts: return response.text @staticmethod - def parse_rss_to_posts(xml_text: str, max_posts: int = 1) -> list[dict[str, str]]: + def parse_rss_to_posts(xml_text: str, max_posts: int = BLOG_POSTS_MAX) -> list[dict[str, str]]: """ Parse RSS XML and return a list of blog post dicts. diff --git a/litellm/llms/bedrock/guardrail_attachments.py b/litellm/llms/bedrock/guardrail_attachments.py new file mode 100644 index 00000000000..cad26206fa3 --- /dev/null +++ b/litellm/llms/bedrock/guardrail_attachments.py @@ -0,0 +1,406 @@ +import base64 +import binascii +from collections.abc import Callable, Iterable, Mapping, Sequence +from functools import reduce +from itertools import chain +from types import MappingProxyType +from typing import Final, Literal, NamedTuple, TypeAlias, TypeGuard # noqa: TID251 # narrows request JSON to str keys + +from litellm.types.proxy.guardrails.guardrail_hooks.bedrock_guardrails import ( + BedrockContentItem, + BedrockImageContent, + BedrockImageSource, +) +from litellm.types.utils import CallTypes + +BedrockImageFormat: TypeAlias = Literal["png", "jpeg"] + + +class RequestAttachments(NamedTuple): + images: tuple[BedrockContentItem, ...] + unscannable: tuple[str, ...] + document_texts: tuple[str, ...] = () + + +class _Image(NamedTuple): + item: BedrockContentItem + + +class _Unscannable(NamedTuple): + label: str + + +class _DocumentText(NamedTuple): + text: str + + +class _Block(NamedTuple): + block: Mapping[str, object] + from_tool: bool + document_depth: int = 0 + + +_Classified: TypeAlias = _Image | _Unscannable | _DocumentText | None +_BlockClassifier: TypeAlias = Callable[[Mapping[str, object]], _Classified] +_NestedToolBlocks: TypeAlias = Callable[ + [Mapping[str, object]], + tuple[Mapping[str, object], ...], +] + +NO_ATTACHMENTS: Final = RequestAttachments(images=(), unscannable=()) + +_IMAGE_FORMAT_BY_MIME: Final[Mapping[str, BedrockImageFormat]] = MappingProxyType( + {"image/png": "png", "image/jpeg": "jpeg", "image/jpg": "jpeg"} +) +_CHAT_CALL_TYPES: Final = frozenset({CallTypes.completion.value, CallTypes.acompletion.value}) +_RESPONSES_CALL_TYPES: Final = frozenset({CallTypes.responses.value, CallTypes.aresponses.value}) +_OPENAI_IMAGE_TYPES: Final = frozenset({"image_url", "input_image", "computer_screenshot"}) +_OPENAI_UNSCANNABLE_TYPES: Final = frozenset( + {"file", "input_file", "input_audio", "video_url", "audio_url", "document", "container_upload"} +) +_ANTHROPIC_UNSCANNABLE_TYPES: Final = frozenset({"document", "container_upload"}) +_TEXT_DOCUMENT_SOURCE_TYPES: Final = frozenset({"text", "content"}) +_MAX_DOCUMENT_DEPTH: Final = 3 +_CONVERSE_UNSCANNABLE_KEYS: Final = ("document", "video", "audio") +_CONVERSE_BINARY_SOURCE_KEYS: Final = ("bytes", "s3Location") +_MAX_IMAGE_BYTES: Final = 4 * 1024 * 1024 +_MAX_IMAGE_BASE64_CHARS: Final = -(-_MAX_IMAGE_BYTES // 3) * 4 +_URL_SAFE_TO_STANDARD_BASE64: Final = str.maketrans("-_", "+/") +_CONVERSE_ACTIONS: Final = frozenset({"converse", "converse-stream"}) +_TOOL_ROLES: Final = frozenset({"tool", "function"}) +_TOOL_OUTPUT_ITEM_TYPES: Final = frozenset({"function_call_output", "custom_tool_call_output", "computer_call_output"}) +_MAX_LABEL_MIME_CHARS: Final = 40 +_MAX_IMAGE_SIDE_PIXELS: Final = 8000 +_PNG_SIGNATURE: Final = b"\x89PNG\r\n\x1a\n" +_JPEG_SIGNATURE: Final = b"\xff\xd8\xff" +_JPEG_STANDALONE_MARKERS: Final = frozenset({0x01, *range(0xD0, 0xD9)}) +_JPEG_FRAME_MARKERS: Final = frozenset(range(0xC0, 0xD0)) - {0xC4, 0xC8, 0xCC} + + +def find_request_attachments( + data: Mapping[str, object], + call_type: str, + skip_tool_messages: bool, + latest_user_message_only: bool, + scan_only_tool_results: bool = False, +) -> RequestAttachments: + messages, classify, nested_tool_blocks = _messages_and_classifier(data, call_type) + selected: Final = messages if not latest_user_message_only else _latest_user_message(messages) + entries: Final = chain.from_iterable(_message_blocks(message, nested_tool_blocks) for message in selected) + classified: Final = tuple( + chain.from_iterable( + _classify_entry(entry, classify) + for entry in entries + if _in_scope(entry, skip_tool_messages, scan_only_tool_results) + ) + ) + return RequestAttachments( + images=tuple(result.item for result in classified if isinstance(result, _Image)), + unscannable=tuple(result.label for result in classified if isinstance(result, _Unscannable)), + document_texts=tuple(result.text for result in classified if isinstance(result, _DocumentText)), + ) + + +def _classify_entry(entry: _Block, classify: _BlockClassifier) -> tuple[_Image | _Unscannable | _DocumentText, ...]: + if entry.document_depth >= _MAX_DOCUMENT_DEPTH and _content_document_source(entry.block) is not None: + return (_Unscannable("document (nested too deep)"),) + text: Final = _document_text(entry) + result: Final = classify(entry.block) + return tuple(item for item in (_DocumentText(text) if text else None, result) if item is not None) + + +def _document_text(entry: _Block) -> str: + block: Final = entry.block + if entry.document_depth > 0 and block.get("type") == "text": + text: Final = block.get("text") + return text if isinstance(text, str) else "" + converse_document: Final = block.get("document") if block.get("type") is None else None + if _is_mapping(converse_document): + return _converse_document_text(converse_document) + if block.get("type") != "document": + return "" + source: Final = block.get("source") + source_type: Final = source.get("type") if _is_mapping(source) else None + source_text: Final = ( + (source.get("data") if source_type == "text" else source.get("content")) + if _is_mapping(source) and isinstance(source_type, str) and source_type in _TEXT_DOCUMENT_SOURCE_TYPES + else None + ) + return "\n".join( + part for part in (block.get("title"), block.get("context"), source_text) if isinstance(part, str) and part + ) + + +def _messages_and_classifier( + data: Mapping[str, object], call_type: str +) -> tuple[Sequence[Mapping[str, object]], _BlockClassifier, _NestedToolBlocks]: + if call_type in _CHAT_CALL_TYPES: + return _mappings(data.get("messages")), _classify_openai_block, _no_nested_blocks + if call_type == CallTypes.anthropic_messages.value: + return _mappings(data.get("messages")), _classify_anthropic_block, _anthropic_tool_result_blocks + if call_type in _RESPONSES_CALL_TYPES: + return _mappings(data.get("input")), _classify_openai_block, _no_nested_blocks + if call_type == CallTypes.allm_passthrough_route.value and _is_bedrock_converse(data): + body: Final = data.get("data") + messages: Final = _mappings(body.get("messages") if _is_mapping(body) else None) + return messages, _classify_converse_block, _converse_tool_result_blocks + return (), _classify_nothing, _no_nested_blocks + + +def _is_bedrock_converse(data: Mapping[str, object]) -> bool: + endpoint: Final = data.get("endpoint") + return ( + data.get("custom_llm_provider") == "bedrock" + and isinstance(endpoint, str) + and endpoint.rstrip("/").rsplit("/", 1)[-1] in _CONVERSE_ACTIONS + ) + + +def _is_mapping(value: object) -> TypeGuard[Mapping[str, object]]: # guard-ok: isinstance narrows correctly; predicate is trivially correct # fmt: skip + return isinstance(value, Mapping) + + +def _is_list(value: object) -> TypeGuard[Sequence[object]]: # guard-ok: isinstance narrows correctly; predicate is trivially correct # fmt: skip + return isinstance(value, list) + + +def _mappings(value: object) -> tuple[Mapping[str, object], ...]: + if not _is_list(value): + return () + return tuple(item for item in value if _is_mapping(item)) + + +def _latest_user_message(messages: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, object], ...]: + return tuple(message for message in messages if message.get("role") == "user")[-1:] + + +def _message_blocks(message: Mapping[str, object], nested_tool_blocks: _NestedToolBlocks) -> tuple[_Block, ...]: + message_type: Final = message.get("type") + if isinstance(message_type, str) and message_type in _TOOL_OUTPUT_ITEM_TYPES: + output: Final = message.get("output") + blocks: Final = (output,) if _is_mapping(output) else _mappings(output) + return _with_document_contents(_Block(block, from_tool=True) for block in blocks) + role: Final = message.get("role") + from_tool_message: Final = isinstance(role, str) and role in _TOOL_ROLES + return _with_document_contents( + chain.from_iterable( + ( + _Block(block, from_tool=from_tool_message), + *(_Block(inner, from_tool=True) for inner in nested_tool_blocks(block)), + ) + for block in _mappings(message.get("content")) + ) + ) + + +def _with_document_contents(entries: Iterable[_Block]) -> tuple[_Block, ...]: + return reduce(_with_document_level_expanded, range(_MAX_DOCUMENT_DEPTH), tuple(entries)) + + +def _with_document_level_expanded(entries: tuple[_Block, ...], depth: int) -> tuple[_Block, ...]: + return tuple( + chain.from_iterable( + _with_document_children(entry) if entry.document_depth == depth else (entry,) for entry in entries + ) + ) + + +def _with_document_children(entry: _Block) -> tuple[_Block, ...]: + source: Final = _content_document_source(entry.block) + if source is None: + return (entry,) + return ( + entry, + *( + _Block(inner, from_tool=entry.from_tool, document_depth=entry.document_depth + 1) + for inner in _mappings(source.get("content")) + ), + ) + + +def _content_document_source(block: Mapping[str, object]) -> Mapping[str, object] | None: + source: Final = block.get("source") + if block.get("type") != "document" or not _is_mapping(source) or source.get("type") != "content": + return None + return source + + +def _in_scope(entry: _Block, skip_tool_messages: bool, scan_only_tool_results: bool) -> bool: + return not skip_tool_messages if entry.from_tool else not scan_only_tool_results + + +def _no_nested_blocks(block: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: + return () + + +def _anthropic_tool_result_blocks(block: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: + return _mappings(block.get("content")) if block.get("type") == "tool_result" else () + + +def _converse_tool_result_blocks(block: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: + tool_result: Final = block.get("toolResult") + return _mappings(tool_result.get("content")) if _is_mapping(tool_result) else () + + +def _classify_nothing(block: Mapping[str, object]) -> _Classified: + return None + + +def _classify_openai_block(block: Mapping[str, object]) -> _Classified: + block_type: Final = block.get("type") + if block_type in ("image", "document"): + return _classify_anthropic_block(block) + if isinstance(block_type, str) and block_type in _OPENAI_IMAGE_TYPES: + image_url: Final = block.get("image_url") + url: Final = image_url.get("url") if _is_mapping(image_url) else image_url + return _classify_data_uri(url, block_type) + if isinstance(block_type, str) and block_type in _OPENAI_UNSCANNABLE_TYPES: + return _Unscannable(block_type) + return None + + +def _classify_anthropic_block(block: Mapping[str, object]) -> _Classified: + block_type: Final = block.get("type") + if block_type == "image": + source: Final = block.get("source") + if _is_mapping(source) and source.get("type") == "base64": + return _classify_base64(source.get("media_type"), source.get("data"), "image") + return _Unscannable("image (url or file source)") + if block_type == "document" and _is_text_document(block): + return None + if isinstance(block_type, str) and block_type in _ANTHROPIC_UNSCANNABLE_TYPES: + return _Unscannable(block_type) + return None + + +def _is_text_document(block: Mapping[str, object]) -> bool: + source: Final = block.get("source") + source_type: Final = source.get("type") if _is_mapping(source) else None + return isinstance(source_type, str) and source_type in _TEXT_DOCUMENT_SOURCE_TYPES + + +def _classify_converse_block(block: Mapping[str, object]) -> _Classified: + guard_content: Final = block.get("guardContent") + image: Final = guard_content.get("image") if _is_mapping(guard_content) else block.get("image") + if _is_mapping(image): + image_format: Final = image.get("format") + source: Final = image.get("source") + encoded: Final = source.get("bytes") if _is_mapping(source) else None + if encoded is None: + return _Unscannable("image (no inline bytes)") + mime: Final = f"image/{image_format}" if isinstance(image_format, str) else None + return _classify_base64(mime, encoded, "image") + document: Final = block.get("document") + if _is_mapping(document) and _converse_text_parts(document) is not None: + return None + for key in _CONVERSE_UNSCANNABLE_KEYS: + if block.get(key) is not None: + return _Unscannable(key) + return None + + +def _converse_document_text(document: Mapping[str, object]) -> str: + parts: Final = _converse_text_parts(document) or () + return "\n".join( + part for part in (document.get("name"), document.get("context"), *parts) if isinstance(part, str) and part + ) + + +def _converse_text_parts(document: Mapping[str, object]) -> tuple[str, ...] | None: + source: Final = document.get("source") + if not _is_mapping(source) or any(source.get(key) is not None for key in _CONVERSE_BINARY_SOURCE_KEYS): + return None + return _converse_document_parts(source) + + +def _converse_document_parts(source: Mapping[str, object]) -> tuple[str, ...] | None: + text: Final = source.get("text") + if isinstance(text, str): + return (text,) + content: Final = source.get("content") + if not _is_list(content): + return None + parts: Final = tuple(item.get("text") if _is_mapping(item) else None for item in content) + return ( + tuple(part for part in parts if isinstance(part, str)) if all(isinstance(part, str) for part in parts) else None + ) + + +def _classify_data_uri(url: object, label: str) -> _Classified: + if not isinstance(url, str) or not url.startswith("data:"): + return _Unscannable(f"{label} (remote URL or file id)") + header, _, payload = url.partition(",") + params: Final = header.removeprefix("data:").lower().split(";") + if "base64" not in params[1:]: + return _Unscannable(f"{label} (not base64)") + return _classify_base64(params[0], payload, label) + + +def _classify_base64(mime: object, encoded: object, label: str) -> _Classified: + image_format: Final = _IMAGE_FORMAT_BY_MIME.get(mime.lower()) if isinstance(mime, str) else None + if image_format is None: + return _Unscannable(f"{label} ({mime[:_MAX_LABEL_MIME_CHARS]})" if isinstance(mime, str) and mime else label) + standard: Final = _standard_base64(encoded) + if len(standard) > _MAX_IMAGE_BASE64_CHARS: + return _Unscannable(f"{label} (over 4 MB)") + decoded: Final = _decoded(standard) + if decoded is None: + return _Unscannable(f"{label} (invalid base64)") + if len(decoded) > _MAX_IMAGE_BYTES: + return _Unscannable(f"{label} (over 4 MB)") + actual_format: Final = _sniffed_format(decoded) + if actual_format is None: + return _Unscannable(f"{label} (not PNG or JPEG data)") + if max(_pixel_size(decoded, actual_format), default=0) > _MAX_IMAGE_SIDE_PIXELS: + return _Unscannable(f"{label} (over {_MAX_IMAGE_SIDE_PIXELS} pixels)") + return _Image( + BedrockContentItem(image=BedrockImageContent(format=actual_format, source=BedrockImageSource(bytes=standard))) + ) + + +def _sniffed_format(data: bytes) -> BedrockImageFormat | None: + if data.startswith(_PNG_SIGNATURE): + return "png" + if data.startswith(_JPEG_SIGNATURE): + return "jpeg" + return None + + +def _pixel_size(data: bytes, image_format: BedrockImageFormat) -> tuple[int, ...]: + if image_format == "png": + return ( + (int.from_bytes(data[16:20], "big"), int.from_bytes(data[20:24], "big")) if data[12:16] == b"IHDR" else () + ) + offset = 2 # rebind-ok: cursor that walks the JPEG segments + while offset + 4 <= len(data) and data[offset] == 0xFF: + marker = data[offset + 1] + if marker == 0xFF or marker in _JPEG_STANDALONE_MARKERS: + offset += 1 if marker == 0xFF else 2 + continue + if marker in _JPEG_FRAME_MARKERS: + return ( + ( + int.from_bytes(data[offset + 7 : offset + 9], "big"), + int.from_bytes(data[offset + 5 : offset + 7], "big"), + ) + if offset + 9 <= len(data) + else () + ) + offset += 2 + int.from_bytes(data[offset + 2 : offset + 4], "big") + return () + + +def _standard_base64(encoded: object) -> str: + compact: Final = ( + "".join(encoded.split()).translate(_URL_SAFE_TO_STANDARD_BASE64) if isinstance(encoded, str) else "" + ) + return compact + "=" * (-len(compact) % 4) + + +def _decoded(standard: str) -> bytes | None: + if not standard: + return None + try: + return base64.b64decode(standard, validate=True) + except (binascii.Error, ValueError): + return None diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index 2c4c942ebeb..18b7c279606 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -1546,6 +1546,31 @@ class MCPRequestHandler: authorization: Final = next((value for key, value in headers.items() if key.lower() == "authorization"), "") return SecretStr(authorization[len("bearer ") :]) if authorization.lower().startswith("bearer ") else None + @staticmethod + def get_guardrail_bearer_token(headers: Headers, auth: UserAPIKeyAuth) -> SecretStr | None: + from litellm.proxy._experimental.mcp_server.utils import ( + _custom_litellm_key_header_name, # pyright: ignore[reportPrivateUsage] # reuse the shared credential-header setting + ) + + bearer: Final = MCPRequestHandler.get_incoming_bearer_token(headers) + if bearer is None: + return None + value: Final = bearer.get_secret_value() + if MCPRequestHandler._is_gateway_admission_credential(value): + return None + if MCPRequestHandler._is_caller_admission_key( + value, auth.api_key + ) or MCPRequestHandler._is_caller_admission_key(hash_token(value), auth.api_key): + return None + admitted_credential: Final = MCPRequestHandler.caller_admission_credential( + headers, + auth, + custom_key_header_name=_custom_litellm_key_header_name(), + ) + if not auth.jwt_claims and MCPRequestHandler._is_caller_admission_key(value, admitted_credential): + return None + return bearer + @staticmethod def get_oauth2_headers_from_headers(headers: Headers) -> dict[str, str]: """ diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index 38f526bcd13..38d9ca52abb 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -412,7 +412,7 @@ if MCP_AVAILABLE: raw_headers=virtual_raw_headers, litellm_logging_obj=virtual_logging_obj, guardrail_context=MCPRequestContext.resolve_guardrail_context(virtual_data), - incoming_bearer_token=MCPRequestHandler.get_incoming_bearer_token(request.headers), + incoming_bearer_token=MCPRequestHandler.get_guardrail_bearer_token(request.headers, user_api_key_dict), ) except Exception as e: virtual_request_data: Final = virtual_processor.data @@ -1346,7 +1346,9 @@ if MCP_AVAILABLE: litellm_logging_obj=data.get("litellm_logging_obj"), guardrail_context=MCPRequestContext.resolve_guardrail_context(data), requested_server_id=canonical_server_id, - incoming_bearer_token=MCPRequestHandler.get_incoming_bearer_token(request.headers), + incoming_bearer_token=MCPRequestHandler.get_guardrail_bearer_token( + request.headers, user_api_key_dict + ), ) result: Final = complete_call_tool_result(executed, WireCompat.LEGACY) except Exception as e: diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index d299f470f12..1916a19a2c4 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -855,7 +855,7 @@ if MCP_AVAILABLE: wire_compat_for(ctx.protocol_version), ctx.protocol_version, incoming_bearer_token=( - MCPRequestHandler.get_incoming_bearer_token(request.headers) + MCPRequestHandler.get_guardrail_bearer_token(request.headers, auth) if isinstance(request, StarletteRequest) else None ), diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 8bf0881c129..b04a958cd03 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -13714,7 +13714,7 @@ } ], "default": false, - "description": "Implemented by guardrail='model_armor'. When True, attachment references that carry no inline bytes (file_id, gs://, or http(s) URLs) pass through unscanned instead of blocking, while fail_on_error still governs real Model Armor API errors. Default False blocks them.", + "description": "Implemented by guardrail='model_armor' and guardrail='bedrock'. When True, attachments the guardrail cannot scan pass through unscanned instead of blocking. For Model Armor these are references with no inline bytes (file_id, gs://, or http(s) URLs), and fail_on_error still governs real Model Armor API errors. For Bedrock these are documents, files, audio, video, and images that are not inline PNG or JPEG up to 4 MB. Default False blocks them.", "title": "Skip Unscannable Attachments" }, "sticky_session_routing": { @@ -16767,7 +16767,7 @@ } ], "default": false, - "description": "Implemented by guardrail='model_armor'. When True, attachment references that carry no inline bytes (file_id, gs://, or http(s) URLs) pass through unscanned instead of blocking, while fail_on_error still governs real Model Armor API errors. Default False blocks them.", + "description": "Implemented by guardrail='model_armor' and guardrail='bedrock'. When True, attachments the guardrail cannot scan pass through unscanned instead of blocking. For Model Armor these are references with no inline bytes (file_id, gs://, or http(s) URLs), and fail_on_error still governs real Model Armor API errors. For Bedrock these are documents, files, audio, video, and images that are not inline PNG or JPEG up to 4 MB. Default False blocks them.", "title": "Skip Unscannable Attachments" }, "sticky_session_routing": { diff --git a/litellm/proxy/dd_span_tagger.py b/litellm/proxy/dd_span_tagger.py index 55f3ecf4da3..08a99e363cd 100644 --- a/litellm/proxy/dd_span_tagger.py +++ b/litellm/proxy/dd_span_tagger.py @@ -36,11 +36,15 @@ class DDSpanTagger: - ``litellm.key_alias`` — human-readable alias for the API key - ``litellm.key_hash`` — hashed API key (safe to log; never the raw secret) - ``litellm.requested_model``— model name as sent by the client + - ``litellm.user_email`` — email of the authenticated user Use cases: - Trace all requests from a specific user/key: filter by ``litellm.key_alias`` or ``litellm.key_hash``. - Trace all requests for a specific model: filter by ``litellm.requested_model``. + - Trace all requests from a specific person under JWT auth, where there is no key_alias: + filter by ``@litellm.user_email:"user@example.com"``. The value comes from the JWT claim + mapped by ``litellm_jwtauth.user_email_jwt_field``, or from the user row for virtual keys. Note: key_alias / key_hash are not available for unauthenticated (e.g. 401) requests. """ @@ -51,8 +55,10 @@ class DDSpanTagger: set_active_span_tag("litellm.key_hash", str(user_api_key_dict.token)) if requested_model: set_active_span_tag("litellm.requested_model", str(requested_model)) + if user_api_key_dict.user_email: + set_active_span_tag("litellm.user_email", str(user_api_key_dict.user_email)) except Exception: verbose_proxy_logger.debug( - "Failed to tag active ddtrace span with key/model tags", + "Failed to tag active ddtrace span with key/model/user tags", exc_info=True, ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index 37318feba45..34f5fdc7c5f 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -44,8 +44,10 @@ from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import ( from litellm.llms.anthropic.chat.guardrail_translation.handler import AnthropicMessagesHandler from litellm.llms.base_llm.guardrail_translation.utils import ( effective_scan_only_tool_results_for_guardrail, + effective_skip_tool_message_for_guardrail, ) from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, bedrock_bearer_token, run_aws_signing +from litellm.llms.bedrock.guardrail_attachments import find_request_attachments from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -76,6 +78,8 @@ from litellm.types.proxy.guardrails.guardrail_hooks.bedrock_guardrails import ( BedrockGuardrailQualifier, BedrockGuardrailResponse, BedrockGuardrailUsage, + BedrockImageContent, + BedrockImageSource, BedrockRequest, BedrockTextContent, ) @@ -120,6 +124,7 @@ _BEDROCK_INVOKE_GUARDRAIL_CHECKS_PATH: Final = "/guardrail-checks/invoke" # more text blocks is split across multiple messages so ALL content is scanned -- # never truncated (truncation would let a user hide content past the limit). _BEDROCK_CHECKS_MAX_CONTENT_BLOCKS: Final = 10 +_BEDROCK_APPLY_GUARDRAIL_MAX_IMAGES: Final = 20 _BEDROCK_CHECKS_KNOWN_KEYS: Final = frozenset({"contentFilter", "promptAttack", "sensitiveInformation"}) # Keys in a sensitiveInformation result that pinpoint the PII location. They are # stripped before the response is handed to standard logging / telemetry so the @@ -223,6 +228,28 @@ def _redact_assessment_match_fields(assessments: list[dict]) -> list[dict]: _RESPONSES_API_CALL_TYPES: Final = frozenset({CallTypes.responses, CallTypes.aresponses}) +def _without_image_bytes(content: Sequence[BedrockContentItem]) -> tuple[BedrockContentItem, ...]: + return tuple( + BedrockContentItem( + image=BedrockImageContent( + format=item["image"]["format"], + source=BedrockImageSource(bytes=f"<{len(item['image']['source']['bytes'])} base64 chars>"), + ) + ) + if "image" in item + else item + for item in content + ) + + +def _allowed_dynamic_body_params(params: Mapping[str, object]) -> Mapping[str, object]: + return {key: value for key, value in params.items() if key not in _BEDROCK_DYNAMIC_BODY_DENYLIST} + + +def _overridden(guardrail: object, name: str) -> bool: + return getattr(getattr(guardrail, name, None), "__func__", None) is not getattr(BedrockGuardrail, name) + + def _is_responses_api_route(request_route: str | None) -> bool: if request_route is None: return False @@ -271,6 +298,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): self.chunk_budget_chars = chunk_budget_chars self.contextual_grounding_from_messages = contextual_grounding_from_messages self.experimental_use_latest_role_message_only = bool(kwargs.get("experimental_use_latest_role_message_only")) + self.skip_unscannable_attachments: Final = kwargs.get("skip_unscannable_attachments") is True # Resource-less, detect-only InvokeGuardrailChecks mode. Present `checks` # routes the guardrail to InvokeGuardrailChecks; absent => ApplyGuardrail. @@ -957,10 +985,29 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): source, ) return BedrockGuardrailResponse() + return await self._apply_guardrail_to_content( + content=content, + bedrock_request_data=bedrock_request_data, + api_key=api_key, + request_data=request_data, + event_type=event_type, + start_time=start_time, + allow_chunking=not self._content_uses_contextual_grounding(content), + ) + + async def _apply_guardrail_to_content( + self, + content: Sequence[BedrockContentItem], + bedrock_request_data: Mapping[str, object], + api_key: str | None, + request_data: dict[str, object] | None, # mutable-ok: proxy request body dict, mutated by the logging helper + event_type: GuardrailEventHooks, + start_time: "datetime", + allow_chunking: bool, + ) -> BedrockGuardrailResponse: credentials, aws_region_name = await run_aws_signing( self._load_credentials, bearer_token=bedrock_bearer_token(api_key) ) - allow_chunking: Final = not self._content_uses_contextual_grounding(content) completed_chunk_usages: Final[list[BedrockGuardrailUsage]] = [] # mutable-ok: billed-chunk usage accumulator try: @@ -1004,7 +1051,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): credentials: "Credentials | None", aws_region_name: str, api_key: str | None, - request_data: dict | None, # mutable-ok: proxy request body dict, mutated by the logging helper + request_data: dict[str, object] | None, # mutable-ok: proxy request body dict, mutated by the logging helper event_type: GuardrailEventHooks, start_time: "datetime", allow_chunking: bool, @@ -1231,7 +1278,9 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): headers_dict: Final = dict(prepared_request.headers) verbose_proxy_logger.debug( "Bedrock AI request body: %s, url %s, headers: %s", - bedrock_request_data, + {**bedrock_request_data, "content": _without_image_bytes(content)} + if any("image" in item for item in content) + else bedrock_request_data, prepared_request.url, get_masked_values(headers_dict), ) @@ -1322,7 +1371,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): def _log_apply_guardrail_success( self, merged_response: BedrockGuardrailResponse, - request_data: dict | None, # mutable-ok: proxy request body dict, mutated by the logging helper + request_data: dict[str, object] | None, # mutable-ok: proxy request body dict, mutated by the logging helper event_type: GuardrailEventHooks, start_time: "datetime", aws_region_name: str | None, @@ -1356,7 +1405,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): def _log_apply_guardrail_failure( self, detail: object, - request_data: dict | None, # mutable-ok: proxy request body dict, mutated by the logging helper + request_data: dict[str, object] | None, # mutable-ok: proxy request body dict, mutated by the logging helper event_type: GuardrailEventHooks, start_time: "datetime", aws_region_name: str | None, @@ -2486,11 +2535,125 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): # This means all actions were ANONYMIZED or NONE, so don't raise exception return False + async def async_scan_request_attachments( + self, + data: dict[str, object], # mutable-ok: proxy request body dict, mutated by the logging helper + call_type: CallTypesLiteral, + event_type: GuardrailEventHooks = GuardrailEventHooks.pre_call, + ) -> None: + if _overridden(self, "apply_guardrail") or _overridden(self, "make_bedrock_api_request"): + return + attachments: Final = find_request_attachments( + data, + call_type, + skip_tool_messages=effective_skip_tool_message_for_guardrail(self), + latest_user_message_only=self.experimental_use_latest_role_message_only, + scan_only_tool_results=effective_scan_only_tool_results_for_guardrail(self), + ) + images: Final = attachments.images if self.checks is None else () + unscannable: Final = attachments.unscannable + tuple("image" for _ in attachments.images if self.checks) + if unscannable: + if not self.skip_unscannable_attachments: + raise self._unscannable_attachments_exception(unscannable, request_data=data, event_type=event_type) + verbose_proxy_logger.warning( + "Bedrock Guardrail %s: letting %d unscannable attachment(s) through unscanned because " + "skip_unscannable_attachments is enabled", + self.guardrail_name, + len(unscannable), + ) + if attachments.document_texts: + await self._scan_document_texts(attachments.document_texts, request_data=data, event_type=event_type) + if not images: + return + bedrock_request_data: Final = { + "source": "INPUT", + **_allowed_dynamic_body_params( + self.get_guardrail_dynamic_request_body_params(request_data=data) # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType] # the base helper returns an untyped dict + ), + } + api_key: Final = data.get("api_key") + try: + for start in range(0, len(images), _BEDROCK_APPLY_GUARDRAIL_MAX_IMAGES): + await self._apply_guardrail_to_content( + content=images[start : start + _BEDROCK_APPLY_GUARDRAIL_MAX_IMAGES], + bedrock_request_data=bedrock_request_data, + api_key=api_key if isinstance(api_key, str) else None, + request_data=data, + event_type=event_type, + start_time=datetime.now(timezone.utc), + allow_chunking=False, + ) + except (HTTPException, ModifyResponseException): + raise + except Exception as e: + if event_type != GuardrailEventHooks.pre_call: + raise + verbose_proxy_logger.error("Bedrock Guardrail: Failed to scan attachments: %s", str(e)) + raise Exception(f"Bedrock guardrail failed: {e}") from e # noqa: TRY002 # same error the text scan raises + + async def _scan_document_texts( + self, + document_texts: Sequence[str], + request_data: dict[str, object], # mutable-ok: proxy request body dict, mutated by the logging helper + event_type: GuardrailEventHooks, + ) -> None: + response: Final = await self.make_bedrock_api_request( + source="INPUT", + messages=[{"role": "user", "content": text} for text in document_texts], + request_data=request_data, + logging_event_type=event_type, + ) + if response.get("action") == "GUARDRAIL_INTERVENED": + raise self._unscannable_attachments_exception( + tuple("document (text the guardrail would mask)" for _ in document_texts), + request_data=request_data, + event_type=event_type, + ) + + def _unscannable_attachments_exception( + self, + unscannable: Sequence[str], + request_data: dict[str, object], # mutable-ok: proxy request body dict, mutated by the logging helper + event_type: GuardrailEventHooks, + ) -> HTTPException | ModifyResponseException: + reason: Final = ( + f"Bedrock guardrail cannot scan {len(unscannable)} attachment(s) " + f"({', '.join(sorted(frozenset(unscannable)))}), so the request was blocked" + ) + now: Final = datetime.now(timezone.utc).timestamp() + self.add_standard_logging_guardrail_information_to_request_data( + guardrail_provider=self.guardrail_provider, + guardrail_json_response={"unscannable_attachments": list(unscannable)}, + request_data=request_data, + guardrail_status="guardrail_intervened", + start_time=now, + end_time=now, + duration=0.0, + event_type=event_type, + ) + if self.disable_exception_on_block is True: + model: Final = request_data.get("model", "bedrock-guardrail") + return ModifyResponseException( + message=reason, + model=model if isinstance(model, str) else "bedrock-guardrail", + request_data=request_data, + guardrail_name=self.guardrail_name, + ) + detail: Final[dict[str, object]] = { # mutable-ok: HTTPException detail + "error": "Violated guardrail policy", + "reason": reason, + } + if self.guardrailIdentifier: + detail["guardrailIdentifier"] = self.guardrailIdentifier + if self.guardrailVersion: + detail["guardrailVersion"] = self.guardrailVersion + return HTTPException(status_code=400, detail=detail) + async def async_pre_call_hook( self, user_api_key_dict: UserAPIKeyAuth, cache: DualCache, - data: dict, + data: dict[str, object], # mutable-ok: the hook writes the guarded messages back into the request call_type: CallTypesLiteral, ) -> Exception | str | dict | None: verbose_proxy_logger.debug("Inside Bedrock Pre-Call Hook for call_type: %s", call_type) @@ -2502,6 +2665,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): event_type: Final[GuardrailEventHooks] = GuardrailEventHooks.pre_call if self.should_run_guardrail(data=data, event_type=event_type) is not True: return data + await self.async_scan_request_attachments(data=data, call_type=call_type, event_type=event_type) new_messages: Final = self.get_guardrails_messages_for_call_type( call_type=cast(CallTypes, call_type), @@ -2555,7 +2719,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): async def async_moderation_hook( self, - data: dict, + data: dict[str, object], # mutable-ok: the hook writes the guarded messages back into the request user_api_key_dict: UserAPIKeyAuth, call_type: CallTypesLiteral, ): @@ -2569,6 +2733,8 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): if self.should_run_guardrail(data=data, event_type=event_type) is not True: return + await self.async_scan_request_attachments(data=data, call_type=call_type, event_type=event_type) + new_messages: Final = self.get_guardrails_messages_for_call_type( call_type=cast(CallTypes, call_type), data=data, diff --git a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py index 03796569043..a7f813d953e 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py @@ -11,7 +11,7 @@ import contextlib import copy import json from collections.abc import AsyncGenerator, AsyncIterable, Awaitable, Callable, Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final, Protocol, TypeAlias, cast +from typing import TYPE_CHECKING, Any, Final, Protocol, TypeAlias, cast, runtime_checkable import anyio from fastapi import HTTPException @@ -53,6 +53,22 @@ GUARDRAIL_NAME: Final = "unified_llm_guardrails" _RequestData: TypeAlias = dict[str, object] +@runtime_checkable +class RequestAttachmentScanner(Protocol): + async def async_scan_request_attachments( + self, + data: _RequestData, + call_type: CallTypesLiteral, + ) -> None: ... + + +async def _scan_request_attachments( + guardrail: CustomGuardrail, data: _RequestData, call_type: CallTypesLiteral +) -> None: + if isinstance(guardrail, RequestAttachmentScanner) and hasattr(type(guardrail), "async_scan_request_attachments"): + await guardrail.async_scan_request_attachments(data=data, call_type=call_type) + + class _EndpointTranslation(Protocol): @property def process_input_messages(self) -> "Callable[..., Awaitable[dict[str, object]]]": ... @@ -251,6 +267,8 @@ class UnifiedLLMGuardrails(CustomLogger): _ensure_litellm_metadata(data, user_api_key_dict) + await _scan_request_attachments(guardrail_to_apply, data, call_type) # pyright: ignore[reportUnknownArgumentType] # hook data is an untyped dict + data = await endpoint_translation.process_input_messages( data=data, guardrail_to_apply=guardrail_to_apply, diff --git a/litellm/proxy/guardrails/guardrail_initializers.py b/litellm/proxy/guardrails/guardrail_initializers.py index 3151e316fdf..0b38f2d9488 100644 --- a/litellm/proxy/guardrails/guardrail_initializers.py +++ b/litellm/proxy/guardrails/guardrail_initializers.py @@ -41,6 +41,7 @@ def initialize_bedrock(litellm_params: LitellmParams, guardrail: Guardrail): aws_bedrock_runtime_endpoint=litellm_params.aws_bedrock_runtime_endpoint, experimental_use_latest_role_message_only=litellm_params.experimental_use_latest_role_message_only, only_scan_new_messages=litellm_params.only_scan_new_messages or False, + skip_unscannable_attachments=litellm_params.skip_unscannable_attachments, streaming_buffer_until_moderated=streaming_params.streaming_buffer_until_moderated, streaming_sampling_rate=streaming_params.streaming_sampling_rate, streaming_end_of_stream_only=streaming_params.streaming_end_of_stream_only, diff --git a/litellm/proxy/public_endpoints/public_endpoints.py b/litellm/proxy/public_endpoints/public_endpoints.py index 1cccc6b2cfb..47caacdcfff 100644 --- a/litellm/proxy/public_endpoints/public_endpoints.py +++ b/litellm/proxy/public_endpoints/public_endpoints.py @@ -490,7 +490,7 @@ async def get_litellm_blog_posts(): verbose_logger.warning("LiteLLM: get_litellm_blog_posts endpoint fallback triggered: %s", str(e)) posts_data = GetBlogPosts.load_local_blog_posts() - posts: Final = [BlogPost(**p) for p in posts_data[:5]] + posts: Final = [BlogPost(**p) for p in posts_data] return BlogPostsResponse(posts=posts) diff --git a/litellm/router.py b/litellm/router.py index fd16c289b0d..cdede46ef5c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -11396,6 +11396,10 @@ class Router: reasoning_efforts_initialized = False reasoning_efforts_unknown = False model_list: Final = self.get_model_list_of_routed_group(model_group) + model_group_description: Final[str | None] = next( + (d for d in (_configured_model_info(m).get("description") for m in model_list) if isinstance(d, str)), + None, + ) for model in model_list: is_match = False if ( @@ -11501,6 +11505,7 @@ class Router: **{ "model_group": user_facing_model_group_name, "providers": [llm_provider], + "description": model_group_description, **model_info, "supports_fast_mode": True, "supported_reasoning_efforts": None, diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index e19f73e556a..b3b57d53bdf 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -1137,9 +1137,11 @@ class BaseLitellmParams(ContentFilterConfigModel): # works for new and patch up skip_unscannable_attachments: bool | None = Field( default=False, description=( - "Implemented by guardrail='model_armor'. When True, attachment references that carry no " - "inline bytes (file_id, gs://, or http(s) URLs) pass through unscanned instead of blocking, " - "while fail_on_error still governs real Model Armor API errors. Default False blocks them." + "Implemented by guardrail='model_armor' and guardrail='bedrock'. When True, attachments the " + "guardrail cannot scan pass through unscanned instead of blocking. For Model Armor these are " + "references with no inline bytes (file_id, gs://, or http(s) URLs), and fail_on_error still " + "governs real Model Armor API errors. For Bedrock these are documents, files, audio, video, " + "and images that are not inline PNG or JPEG up to 4 MB. Default False blocks them." ), ) sanitize_error_detail: bool | None = Field( diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/types/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index 8d66b624341..c2c737fcc03 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -1,6 +1,6 @@ from typing import Literal -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict # Bedrock contextual grounding tags each content block so the guardrail knows # which text is the reference source, the user question, and the content to grade. @@ -12,8 +12,18 @@ class BedrockTextContent(TypedDict, total=False): qualifiers: list[BedrockGuardrailQualifier] +class BedrockImageSource(TypedDict): + bytes: ReadOnly[str] + + +class BedrockImageContent(TypedDict): + format: ReadOnly[Literal["png", "jpeg"]] + source: ReadOnly[BedrockImageSource] + + class BedrockContentItem(TypedDict, total=False): text: BedrockTextContent + image: ReadOnly[BedrockImageContent] class BedrockRequest(TypedDict, total=False): diff --git a/litellm/types/router.py b/litellm/types/router.py index cbe332dab12..273edf0f913 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -926,6 +926,7 @@ class ModelGroupInfo(LiteLLMBaseModel): supported_reasoning_efforts: tuple[str, ...] | None = Field(default=None) supported_openai_params: list[str] | None = Field(default=[]) configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None + description: str | None = None def __init__(self, **data) -> None: for field_name, field_type in _resolved_annotations(self.__class__).items(): diff --git a/tests/e2e/coverage_registry/logging.yaml b/tests/e2e/coverage_registry/logging.yaml index 9a851587fce..69cad8eb22e 100644 --- a/tests/e2e/coverage_registry/logging.yaml +++ b/tests/e2e/coverage_registry/logging.yaml @@ -28,3 +28,4 @@ - {id: logging.niche_integrations.success.logs_spend, module: logging, tier: P2, event: success, assertions: [logs_spend], exercised_on: [chat_completions], source: grammar, rationale: "SMOKE cohort: athina/galileo/deepeval/langtrace/weave/lunary/humanloop/traceloop/helicone/argilla/newrelic/sqs/supabase/dynamodb/agentops/lago/etc"} - {id: logging.niche_integrations.failure.logs_spend, module: logging, tier: P2, event: failure, assertions: [logs_spend], exercised_on: [chat_completions], source: grammar, rationale: "SMOKE niche failure path"} - {id: logging.langfuse.success.logs_spend, module: logging, tier: P1, event: success, assertions: [logs_spend], exercised_on: [chat_completions], source: "integrations/langfuse/langfuse_otel.py", rationale: "Team-scoped Langfuse delivery via /team/callback; LangChain-ecosystem evals spend"} +- {id: logging.langfuse.success.logs_usage, module: logging, tier: P1, event: success, assertions: [logs_usage], exercised_on: [responses], source: "integrations/langfuse/langfuse.py", rationale: "Classic Langfuse SDK generations must carry token usage on the /v1/responses path; ResponseAPIUsage read as chat Usage logs all zeros (LIT-8238)"} diff --git a/tests/e2e/logging/test_responses_langfuse_usage_e2e.py b/tests/e2e/logging/test_responses_langfuse_usage_e2e.py new file mode 100644 index 00000000000..4dfd5d0ec33 --- /dev/null +++ b/tests/e2e/logging/test_responses_langfuse_usage_e2e.py @@ -0,0 +1,158 @@ +"""Live e2e: Langfuse token usage on the /v1/responses path. + +Covers logging.langfuse.success.logs_usage: a key-scoped ``langfuse`` +(classic SDK) callback must log non-zero input/output tokens to the Langfuse +generation for a /v1/responses call, streamed or not. A ResponsesAPIResponse +carries ResponseAPIUsage (input_tokens/output_tokens), and an assembled stream +carries a plain dict of chat usage fields; the logger must normalize both +before building usage_details, or Langfuse shows 0 input / 0 output while +cost is still right. +""" + +from __future__ import annotations + +import pytest +from e2e_config import CHEAP_OPENAI_MODEL, unique_marker +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta +from lifecycle import ResourceManager +from logging_client import ( + LangfuseCreds, + LoggingClient, + first_ok, + load_langfuse_creds, +) +from models import ( + KeyLoggingCallback, + KeyLoggingCallbackVars, + KeyMetadata, + ResponsesApiResponse, + ResponsesStreamEvent, + ResponsesUsage, +) +from pydantic import BaseModel, ConfigDict, TypeAdapter + +pytestmark = pytest.mark.e2e + + +class _LangfuseUsageDetails(BaseModel): + model_config = ConfigDict(extra="ignore") + + input: int | None = None + output: int | None = None + total: int | None = None + cache_read_input_tokens: int | None = None + + +_USAGE_DETAILS_ADAPTER: TypeAdapter[_LangfuseUsageDetails] = TypeAdapter(_LangfuseUsageDetails) + + +@pytest.fixture(scope="session") +def langfuse_creds() -> LangfuseCreds: + return load_langfuse_creds() + + +def _completed_usage(stream_events: list[str]) -> ResponsesUsage | None: + """Usage the proxy relayed on the stream's response.completed event.""" + for raw in stream_events: + event = ResponsesStreamEvent.model_validate_json(raw) + if event.type == "response.completed" and event.response is not None: + return event.response.usage + return None + + +class TestResponsesLangfuseUsage: + @pytest.mark.covers("logging.langfuse.success.logs_usage", exercised_on=["responses"]) + @pytest.mark.parametrize( + "stream", + [ + pytest.param( + False, + marks=meta( + Subject( + domain=Domain.OBSERVABILITY, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=(CHEAP_OPENAI_MODEL,), + mode=Mode.NONSTREAM, + ) + ), + id="non_streaming", + ), + pytest.param( + True, + marks=meta( + Subject( + domain=Domain.OBSERVABILITY, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=(CHEAP_OPENAI_MODEL,), + mode=Mode.STREAM, + ) + ), + id="streaming", + ), + ], + ) + def test_responses_call_logs_nonzero_token_usage( + self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager, stream: bool + ) -> None: + key_alias = f"lf-resp-key-{unique_marker()}" + key = client.key_with_alias( + key_alias, + models=[CHEAP_OPENAI_MODEL], + metadata=KeyMetadata( + logging=[ + KeyLoggingCallback( + callback_name="langfuse", + callback_type="success", + callback_vars=KeyLoggingCallbackVars( + langfuse_public_key=langfuse_creds.public_key, + langfuse_secret_key=langfuse_creds.secret_key, + langfuse_host=langfuse_creds.host, + ), + ) + ] + ), + ) + resources.defer(lambda: client.delete_key(key)) + + marker = unique_marker() + outcome = first_ok( + client, + lambda: client.responses_raw( + key, CHEAP_OPENAI_MODEL, f"reply with one word {marker}", max_output_tokens=64, stream=stream + ), + ) + usage = ( + _completed_usage(outcome.stream_events) + if stream + else ResponsesApiResponse.model_validate_json(outcome.body).usage + ) + assert usage is not None and usage.input_tokens > 0, ( + f"the /v1/responses reply must report input_tokens, got {outcome.body[:300]} " + f"with {len(outcome.stream_events)} stream events" + ) + + observation = client.poll_langfuse_observation( + langfuse_creds, + key_alias=key_alias, + prompt_marker=marker, + ) + assert observation is not None, ( + f"the key's /v1/responses call (marker {marker}) never reached Langfuse within the deadline" + ) + assert observation.usage_details is not None, ( + f"the Langfuse generation must carry usageDetails, got {observation.model_dump_json()}" + ) + details = _USAGE_DETAILS_ADAPTER.validate_python(observation.usage_details) + cache_read = details.cache_read_input_tokens or 0 + assert details.input is not None and details.input > 0, ( + f"usageDetails.input must be non-zero, got {details.model_dump()}" + ) + assert details.input + cache_read == usage.input_tokens, ( + f"usageDetails.input {details.input} + cache_read {cache_read} must equal the proxy's " + f"input_tokens {usage.input_tokens}" + ) + assert details.output == usage.output_tokens, ( + f"usageDetails.output {details.output} must equal the proxy's output_tokens {usage.output_tokens}" + ) diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 21fa015525a..b2b35d953d0 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -477,6 +477,35 @@ class ChatResponse(BaseModel): guardrail_information: list[GuardrailInformationEntry] | None = None +class ResponsesInputTokensDetails(BaseModel): + cached_tokens: int | None = None + + +class ResponsesUsage(BaseModel): + """`/v1/responses` usage shape: input/output tokens, not prompt/completion.""" + + input_tokens: int + output_tokens: int + total_tokens: int | None = None + input_tokens_details: ResponsesInputTokensDetails | None = None + + +class ResponsesApiResponse(BaseModel): + model_config = ConfigDict(extra="ignore") + + id: str | None = None + usage: ResponsesUsage | None = None + + +class ResponsesStreamEvent(BaseModel): + """One `/v1/responses` SSE event; `response.completed` carries the final response.""" + + model_config = ConfigDict(extra="ignore") + + type: str + response: ResponsesApiResponse | None = None + + # ---------- anthropic /v1/messages + count_tokens ---------- diff --git a/tests/integration/observability/test_bedrock_guardrail_attachments.py b/tests/integration/observability/test_bedrock_guardrail_attachments.py new file mode 100644 index 00000000000..0d53e335cd1 --- /dev/null +++ b/tests/integration/observability/test_bedrock_guardrail_attachments.py @@ -0,0 +1,525 @@ +import base64 +import json +import threading +import uuid +from collections.abc import Callable, Iterator, Mapping +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +import httpx +import pytest +import yaml +from integration._support.client import Gateway, gateway_from_environment +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue + +_GUARDRAIL_ID: Final = "synthetic-attachment-guard" +_APPLY_PATH: Final = f"/guardrail/{_GUARDRAIL_ID}/version/DRAFT/apply" +_CHECKS_PATH: Final = "/guardrail-checks/invoke" +_BEDROCK_MODEL: Final = "anthropic.claude-3-haiku-20240307-v1:0" +_SSN: Final = "123-45-6789" +_ENDPOINTS: Final = ("chat", "messages", "responses", "converse") + + +def _png(marker: bytes) -> bytes: + return b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR" + (64).to_bytes(4, "big") * 2 + b"\x08\x02" + marker + + +_THREAT_PNG: Final = _png(b"synthetic-threat") +_PLAIN_PNG: Final = _png(b"synthetic-plain") +_PDF: Final = b"%PDF-1.4 synthetic employee record" + + +def _b64(raw: bytes) -> str: + return base64.b64encode(raw).decode() + + +@dataclass(frozen=True, slots=True) +class _Rig: + strict: Gateway + lenient: Gateway + checks: Gateway + policy: Wire + upstream: Wire + outage: threading.Event + + +def _blocked(assessment: Mapping[str, JsonValue]) -> Reply: + return Reply( + body=json.dumps( + { + "action": "GUARDRAIL_INTERVENED", + "outputs": [{"text": "blocked by synthetic policy"}], + "assessments": [assessment], + "usage": {"contentPolicyUnits": 1, "contentPolicyImageUnits": 1}, + } + ).encode() + ) + + +def _guardrail(outage: threading.Event) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + if request.target == _CHECKS_PATH: + return Reply(body=json.dumps({"results": {}}).encode()) + assert request.target == _APPLY_PATH, request.target + if outage.is_set(): + return Reply(status=500, body=b'{"message":"synthetic guardrail outage"}') + content: Final = json.loads(request.body)["content"] + images: Final = [base64.b64decode(item["image"]["source"]["bytes"]) for item in content if "image" in item] + texts: Final = [item["text"]["text"] for item in content if "text" in item] + if _THREAT_PNG in images: + return _blocked( + {"contentPolicy": {"filters": [{"type": "VIOLENCE", "confidence": "HIGH", "action": "BLOCKED"}]}} + ) + if any(_SSN in text for text in texts): + return _blocked( + { + "sensitiveInformationPolicy": { + "piiEntities": [{"type": "US_SOCIAL_SECURITY_NUMBER", "match": _SSN, "action": "BLOCKED"}] + } + } + ) + return Reply(body=json.dumps({"action": "NONE", "outputs": [], "assessments": []}).encode()) + + return respond + + +def _upstream(request: Request) -> Reply: + if request.target == "/v1/chat/completions": + return Reply( + body=json.dumps( + { + "id": f"chatcmpl-{uuid.uuid4().hex}", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "served"}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}, + } + ).encode() + ) + if request.target == "/v1/messages": + return Reply( + body=json.dumps( + { + "id": f"msg_{uuid.uuid4().hex}", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5-20250929", + "content": [{"type": "text", "text": "served"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 3, "output_tokens": 1}, + } + ).encode() + ) + if request.target == "/v1/responses": + return Reply( + body=json.dumps( + { + "id": f"resp_{uuid.uuid4().hex}", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": "gpt-4.1-mini", + "output": [ + { + "type": "message", + "id": "msg_synthetic", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "served", "annotations": []}], + } + ], + "usage": {"input_tokens": 3, "output_tokens": 1, "total_tokens": 4}, + } + ).encode() + ) + assert request.target == f"/model/{_BEDROCK_MODEL}/converse", request.target + return Reply( + body=json.dumps( + { + "output": {"message": {"role": "assistant", "content": [{"text": "served"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 3, "outputTokens": 1, "totalTokens": 4}, + "metrics": {"latencyMs": 1}, + } + ).encode() + ) + + +def _config( + directory: Path, name: str, policy: Wire, upstream: Wire, skip_unscannable: bool, checks: bool = False +) -> Path: + configuration: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + configuration["litellm_settings"] = {} + configuration["model_list"] = [ + { + "model_name": "attach-chat", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_base": f"{upstream.url}/v1", "api_key": "sk-chat"}, + }, + { + "model_name": "attach-messages", + "litellm_params": { + "model": "anthropic/claude-sonnet-4-5-20250929", + "api_base": upstream.url, + "api_key": "sk-ant-synthetic", + }, + }, + { + "model_name": "attach-responses", + "litellm_params": {"model": "openai/gpt-4.1-mini", "api_base": f"{upstream.url}/v1", "api_key": "sk-resp"}, + }, + { + "model_name": "attach-converse", + "litellm_params": { + "model": f"bedrock/{_BEDROCK_MODEL}", + "api_base": upstream.url, + "aws_access_key_id": "AKIASYNTHETICRUNTIME", + "aws_secret_access_key": "synthetic-runtime-secret", + "aws_region_name": "us-east-1", + }, + }, + ] + target: Final[dict[str, JsonValue]] = ( + {"checks": {"contentFilter": {"categories": [{"category": "VIOLENCE"}]}}} + if checks + else {"guardrailIdentifier": _GUARDRAIL_ID, "guardrailVersion": "DRAFT"} + ) + configuration["guardrails"] = [ + { + "guardrail_name": f"bedrock-attachments-{name}", + "litellm_params": { + "guardrail": "bedrock", + "mode": "pre_call", + "default_on": True, + "skip_unscannable_attachments": skip_unscannable, + **target, + "aws_region_name": "us-east-1", + "aws_access_key_id": "AKIASYNTHETICGUARDRAIL", + "aws_secret_access_key": "synthetic-guardrail-secret", + "aws_bedrock_runtime_endpoint": policy.url, + }, + } + ] + path: Final = directory / f"bedrock-attachments-{name}.yaml" + path.write_text(yaml.safe_dump(configuration)) + return path + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]: + directory: Final = tmp_path_factory.mktemp("bedrock-guardrail-attachments") + outage: Final = threading.Event() + with ( + wire_server(_guardrail(outage)) as policy, + wire_server(_upstream) as upstream, + gateway_from_environment() as shared, + owned_proxy( + shared, directory, {}, config=_config(directory, "strict", policy, upstream, False), workers=2 + ) as strict, + owned_proxy( + shared, directory, {}, config=_config(directory, "lenient", policy, upstream, True), workers=2 + ) as lenient, + owned_proxy( + shared, directory, {}, config=_config(directory, "checks", policy, upstream, False, checks=True), workers=2 + ) as checks, + ): + yield _Rig(strict, lenient, checks, policy, upstream, outage) + + +@pytest.fixture(autouse=True) +def _drained(rig: _Rig) -> Iterator[None]: + rig.policy.drain() + rig.upstream.drain() + rig.outage.clear() + yield + rig.outage.clear() + + +def _image_block(endpoint: str, encoded: str, mime: str = "image/png") -> dict[str, JsonValue]: + if endpoint == "chat": + return {"type": "image_url", "image_url": {"url": f"data:{mime};base64,{encoded}"}} + if endpoint == "messages": + return {"type": "image", "source": {"type": "base64", "media_type": mime, "data": encoded}} + if endpoint == "responses": + return {"type": "input_image", "image_url": f"data:{mime};base64,{encoded}"} + return {"image": {"format": mime.removeprefix("image/"), "source": {"bytes": encoded}}} + + +def _pdf_block(endpoint: str) -> dict[str, JsonValue]: + if endpoint == "chat": + return {"type": "file", "file": {"file_data": f"data:application/pdf;base64,{_b64(_PDF)}", "filename": "r.pdf"}} + if endpoint == "messages": + return {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": _b64(_PDF)}} + if endpoint == "responses": + return {"type": "input_file", "file_data": f"data:application/pdf;base64,{_b64(_PDF)}", "filename": "r.pdf"} + return {"document": {"format": "pdf", "name": "record", "source": {"bytes": _b64(_PDF)}}} + + +def _text_block(endpoint: str, text: str) -> dict[str, JsonValue]: + if endpoint == "converse": + return {"text": text} + return {"type": "input_text" if endpoint == "responses" else "text", "text": text} + + +def _send(gateway: Gateway, endpoint: str, *blocks: JsonValue, stream: bool = False) -> httpx.Response: + content: Final = list(blocks) + if endpoint == "chat": + body: dict[str, JsonValue] = {"model": "attach-chat", "messages": [{"role": "user", "content": content}]} + return gateway.request("POST", "/v1/chat/completions", {**body, "stream": True} if stream else body) + if endpoint == "messages": + return gateway.request( + "POST", + "/v1/messages", + {"model": "attach-messages", "max_tokens": 32, "messages": [{"role": "user", "content": content}]}, + ) + if endpoint == "responses": + return gateway.request( + "POST", "/v1/responses", {"model": "attach-responses", "input": [{"role": "user", "content": content}]} + ) + return gateway.request( + "POST", "/bedrock/model/attach-converse/converse", {"messages": [{"role": "user", "content": content}]} + ) + + +def _scans(policy: Wire) -> list[list[dict[str, JsonValue]]]: + return [json.loads(request.body)["content"] for request in policy.drain()] + + +def _image_scans(scans: list[list[dict[str, JsonValue]]]) -> list[list[JsonValue]]: + return [[item["image"] for item in content if "image" in item] for content in scans if content[0].get("image")] + + +def _text_scans(scans: list[list[dict[str, JsonValue]]]) -> list[str]: + return [item["text"]["text"] for content in scans for item in content if "text" in item] + + +def _forwarded(upstream: Wire) -> list[dict[str, JsonValue]]: + return [json.loads(request.body) for request in upstream.drain() if request.method == "POST"] + + +@pytest.mark.parametrize("endpoint", _ENDPOINTS) +def test_an_image_only_turn_is_scanned_and_a_threat_image_is_blocked(rig: _Rig, endpoint: str) -> None: + response: Final = _send(rig.strict, endpoint, _image_block(endpoint, _b64(_THREAT_PNG))) + + assert response.status_code == 400, response.text + assert "VIOLENCE" in response.text, response.text + assert _image_scans(_scans(rig.policy)) == [[{"format": "png", "source": {"bytes": _b64(_THREAT_PNG)}}]] + assert _forwarded(rig.upstream) == [] + + +@pytest.mark.parametrize("endpoint", _ENDPOINTS) +def test_a_harmless_image_is_scanned_and_reaches_the_model(rig: _Rig, endpoint: str) -> None: + response: Final = _send(rig.strict, endpoint, _image_block(endpoint, _b64(_PLAIN_PNG))) + + assert response.status_code == 200, response.text + assert _image_scans(_scans(rig.policy)) == [[{"format": "png", "source": {"bytes": _b64(_PLAIN_PNG)}}]] + assert len(_forwarded(rig.upstream)) == 1 + + +@pytest.mark.parametrize("endpoint", _ENDPOINTS) +def test_a_pdf_is_refused_without_calling_bedrock_or_the_model(rig: _Rig, endpoint: str) -> None: + question: Final = f"What is in the attachment {uuid.uuid4().hex}?" + response: Final = _send(rig.strict, endpoint, _text_block(endpoint, question), _pdf_block(endpoint)) + + assert response.status_code == 400, response.text + assert "Bedrock guardrail cannot scan 1 attachment(s)" in response.text, response.text + assert _scans(rig.policy) == [] + assert _forwarded(rig.upstream) == [] + + +@pytest.mark.parametrize("endpoint", _ENDPOINTS) +def test_skip_unscannable_attachments_lets_a_pdf_through_and_still_scans_the_text(rig: _Rig, endpoint: str) -> None: + question: Final = f"What is in the attachment {uuid.uuid4().hex}?" + response: Final = _send(rig.lenient, endpoint, _text_block(endpoint, question), _pdf_block(endpoint)) + + assert response.status_code == 200, response.text + assert question in _text_scans(_scans(rig.policy)) + (forwarded,) = _forwarded(rig.upstream) + assert _b64(_PDF) in json.dumps(forwarded) + + +@pytest.mark.parametrize("endpoint", _ENDPOINTS) +def test_skip_unscannable_attachments_still_blocks_a_threat_image(rig: _Rig, endpoint: str) -> None: + response: Final = _send(rig.lenient, endpoint, _image_block(endpoint, _b64(_THREAT_PNG))) + + assert response.status_code == 400, response.text + assert "VIOLENCE" in response.text, response.text + assert _forwarded(rig.upstream) == [] + + +def test_skip_unscannable_attachments_still_scans_a_converse_pdf_name_and_context(rig: _Rig) -> None: + context: Final = f"Employee SSN {_SSN} {uuid.uuid4().hex}" + document: Final = { + "document": {"format": "pdf", "name": "record", "context": context, "source": {"bytes": _b64(_PDF)}} + } + response: Final = _send(rig.lenient, "converse", _text_block("converse", "summarize"), document) + + assert response.status_code == 400, response.text + assert f"record\n{context}" in _text_scans(_scans(rig.policy)) + assert _forwarded(rig.upstream) == [] + + +def test_a_streaming_chat_request_with_a_pdf_is_refused_before_streaming(rig: _Rig) -> None: + response: Final = _send(rig.strict, "chat", _text_block("chat", "summarize"), _pdf_block("chat"), stream=True) + + assert response.status_code == 400, response.text + assert "Bedrock guardrail cannot scan 1 attachment(s)" in response.text, response.text + assert _forwarded(rig.upstream) == [] + + +def _anthropic_text_document(text: str) -> dict[str, JsonValue]: + return {"type": "document", "title": "record", "source": {"type": "text", "media_type": "text/plain", "data": text}} + + +def _converse_text_document(source: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + return {"document": {"format": "txt", "name": "record", "source": dict(source)}} + + +_TEXT_DOCUMENTS: Final = ( + pytest.param("messages", _anthropic_text_document, id="messages-text-source"), + pytest.param("converse", lambda text: _converse_text_document({"text": text}), id="converse-text-source"), + pytest.param( + "converse", + lambda text: _converse_text_document({"content": [{"text": "cover page"}, {"text": text}]}), + id="converse-content-source", + ), +) + + +@pytest.mark.parametrize("endpoint, document", _TEXT_DOCUMENTS) +def test_a_text_document_with_an_ssn_is_scanned_as_text_and_blocked( + rig: _Rig, endpoint: str, document: Callable[[str], dict[str, JsonValue]] +) -> None: + secret: Final = f"Employee SSN {_SSN} {uuid.uuid4().hex}" + response: Final = _send(rig.strict, endpoint, _text_block(endpoint, "summarize"), document(secret)) + + assert response.status_code == 400, response.text + assert any(secret in text for text in _text_scans(_scans(rig.policy))) + assert _forwarded(rig.upstream) == [] + + +@pytest.mark.parametrize("endpoint, document", _TEXT_DOCUMENTS) +def test_a_harmless_text_document_is_scanned_as_text_and_reaches_the_model( + rig: _Rig, endpoint: str, document: Callable[[str], dict[str, JsonValue]] +) -> None: + note: Final = f"The meeting is at noon {uuid.uuid4().hex}" + response: Final = _send(rig.strict, endpoint, _text_block(endpoint, "summarize"), document(note)) + + assert response.status_code == 200, response.text + assert any(note in text for text in _text_scans(_scans(rig.policy))) + (forwarded,) = _forwarded(rig.upstream) + assert note in json.dumps(forwarded) + + +def test_a_converse_txt_document_sent_as_bytes_stays_unscannable(rig: _Rig) -> None: + document: Final = {"document": {"format": "txt", "name": "record", "source": {"bytes": _b64(b"notes")}}} + response: Final = _send(rig.strict, "converse", _text_block("converse", "summarize"), document) + + assert response.status_code == 400, response.text + assert "cannot scan 1 attachment(s) (document)" in response.text, response.text + assert _forwarded(rig.upstream) == [] + + +def test_a_converse_guard_content_threat_image_is_scanned_and_blocked(rig: _Rig) -> None: + guarded: Final = {"guardContent": {"image": {"format": "png", "source": {"bytes": _b64(_THREAT_PNG)}}}} + response: Final = _send(rig.strict, "converse", _text_block("converse", "describe"), guarded) + + assert response.status_code == 400, response.text + assert "VIOLENCE" in response.text, response.text + assert _forwarded(rig.upstream) == [] + + +@pytest.mark.parametrize("endpoint", _ENDPOINTS) +def test_checks_mode_refuses_an_image_without_calling_bedrock(rig: _Rig, endpoint: str) -> None: + response: Final = _send(rig.checks, endpoint, _image_block(endpoint, _b64(_PLAIN_PNG))) + + assert response.status_code == 400, response.text + assert "cannot scan 1 attachment(s) (image)" in response.text, response.text + assert list(rig.policy.drain()) == [] + assert _forwarded(rig.upstream) == [] + + +def test_checks_mode_sends_a_converse_text_document_to_invoke_guardrail_checks(rig: _Rig) -> None: + note: Final = f"The meeting is at noon {uuid.uuid4().hex}" + response: Final = _send(rig.checks, "converse", _converse_text_document({"text": note})) + + assert response.status_code == 200, response.text + calls: Final = list(rig.policy.drain()) + assert [call.target for call in calls] == [_CHECKS_PATH] + assert note in calls[0].body.decode() + (forwarded,) = _forwarded(rig.upstream) + assert note in json.dumps(forwarded) + + +@pytest.mark.parametrize( + "encoded", + [ + pytest.param(_b64(_THREAT_PNG).rstrip("="), id="unpadded"), + pytest.param(base64.urlsafe_b64encode(_THREAT_PNG).decode(), id="url-safe"), + ], +) +def test_non_standard_base64_is_normalized_before_the_scan(rig: _Rig, encoded: str) -> None: + response: Final = _send(rig.strict, "chat", _image_block("chat", encoded)) + + assert response.status_code == 400, response.text + assert _image_scans(_scans(rig.policy)) == [[{"format": "png", "source": {"bytes": _b64(_THREAT_PNG)}}]] + + +def test_the_image_format_sent_to_bedrock_comes_from_the_bytes(rig: _Rig) -> None: + response: Final = _send(rig.strict, "chat", _image_block("chat", _b64(_PLAIN_PNG), mime="image/jpeg")) + + assert response.status_code == 200, response.text + assert _image_scans(_scans(rig.policy)) == [[{"format": "png", "source": {"bytes": _b64(_PLAIN_PNG)}}]] + + +@pytest.mark.parametrize("endpoint", ("chat", "converse")) +def test_a_guardrail_outage_fails_closed_on_an_image_turn(rig: _Rig, endpoint: str) -> None: + rig.outage.set() + response: Final = _send(rig.strict, endpoint, _image_block(endpoint, _b64(_PLAIN_PNG))) + + assert response.status_code == 500, response.text + assert "synthetic guardrail outage" in response.text, response.text + assert len(_image_scans(_scans(rig.policy))) == 1 + assert _forwarded(rig.upstream) == [] + + +def test_a_concurrent_mixed_burst_across_two_workers_keeps_each_verdict(rig: _Rig) -> None: + cases: Final = tuple( + (kind, uuid.uuid4().hex) + for kind in ("threat", "plain", "pdf", "converse-note", "converse-ssn") + for _ in range(5) + ) + + def send(case: tuple[str, str]) -> tuple[str, str, int]: + kind, tag = case + if kind == "threat": + response = _send(rig.strict, "chat", _text_block("chat", tag), _image_block("chat", _b64(_THREAT_PNG))) + elif kind == "plain": + response = _send( + rig.strict, "messages", _text_block("messages", tag), _image_block("messages", _b64(_PLAIN_PNG)) + ) + elif kind == "pdf": + response = _send(rig.strict, "responses", _text_block("responses", tag), _pdf_block("responses")) + elif kind == "converse-note": + response = _send(rig.strict, "converse", _converse_text_document({"text": f"note {tag}"})) + else: + response = _send(rig.strict, "converse", _converse_text_document({"text": f"SSN {_SSN} {tag}"})) + return kind, tag, response.status_code + + with ThreadPoolExecutor(max_workers=12) as pool: + results: Final = tuple(pool.map(send, cases)) + + expected: Final = {"threat": 400, "plain": 200, "pdf": 400, "converse-note": 200, "converse-ssn": 400} + assert [(kind, status) for kind, _, status in results] == [(kind, expected[kind]) for kind, _, _ in results] + served: Final = json.dumps(_forwarded(rig.upstream)) + assert sorted(tag for kind, tag, _ in results if kind in ("plain", "converse-note") and tag in served) == sorted( + tag for kind, tag, _ in results if kind in ("plain", "converse-note") + ) + assert not any(tag in served for kind, tag, _ in results if expected[kind] == 400) diff --git a/tests/unit/integrations/test_langfuse.py b/tests/unit/integrations/test_langfuse.py index 6d09c42df76..aff69b6403b 100644 --- a/tests/unit/integrations/test_langfuse.py +++ b/tests/unit/integrations/test_langfuse.py @@ -5,7 +5,7 @@ import threading import time import types import unittest -from typing import Final, Optional +from typing import Final from unittest.mock import MagicMock, Mock, patch import httpx @@ -29,9 +29,10 @@ from litellm.types.utils import ( TextCompletionResponse, ) - # Import LangfuseUsageDetails directly from the module where it's defined from litellm.types.integrations.langfuse import * +from litellm.responses.utils import ResponseAPILoggingUtils +from litellm.types.llms.openai import InputTokensDetails, ResponseAPIUsage, ResponsesAPIResponse class TestLangfuseUsageDetails(unittest.TestCase): @@ -328,7 +329,103 @@ class TestLangfuseUsageDetails(unittest.TestCase): mock_add_prompt_params.assert_called_once() - def _build_standard_logging_payload(self, trace_id: Optional[str] = None): + def _responses_api_usage(self) -> ResponseAPIUsage: + return ResponseAPIUsage( + input_tokens=16, + output_tokens=21, + total_tokens=37, + input_tokens_details=InputTokensDetails(cached_tokens=4), + ) + + def _log_responses_api_generation(self, response_obj: ResponsesAPIResponse) -> dict[str, int]: + self.use_real_langfuse_client() + + kwargs = { + "model": "gpt-5.5", + "messages": [{"role": "user", "content": "Test"}], + "litellm_params": {"metadata": {}}, + "optional_params": {}, + "litellm_call_id": "test-call-id-responses-api-usage", + "standard_logging_object": self._build_standard_logging_payload(), + "response_cost": 0.0, + } + + fixed_time = datetime.datetime(2024, 1, 1, 12, 0, 0) + + self.logger._log_langfuse_v2( + user_id="test-user", + metadata={}, + litellm_params=kwargs["litellm_params"], + output={"role": "assistant", "content": "Response"}, + start_time=fixed_time, + end_time=fixed_time + datetime.timedelta(seconds=1), + kwargs=kwargs, + optional_params=kwargs["optional_params"], + input={"messages": kwargs["messages"]}, + response_obj=response_obj, + level="DEFAULT", + litellm_call_id=kwargs["litellm_call_id"], + ) + + return json.loads(self.exported_generation().attributes["langfuse.observation.usage_details"]) + + def test_log_langfuse_v2_responses_api_usage(self): + """ + Regression test: a /v1/responses response carries ResponseAPIUsage + (input_tokens/output_tokens), which must be normalized to a chat Usage + before Langfuse usage_details are read, or generations log 0 tokens. + """ + response_obj = ResponsesAPIResponse(id="resp_123", created_at=0, output=[], usage=self._responses_api_usage()) + + usage_details = self._log_responses_api_generation(response_obj) + + # input is reduced by cache_read_input_tokens per Langfuse docs + assert usage_details == { + "input": 12, + "output": 21, + "total": 37, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4, + } + + def test_log_langfuse_v2_streamed_responses_api_usage(self): + """ + Regression test: an assembled /v1/responses stream reaches the logger + with its usage already converted to a plain dict of chat usage fields, + the shape litellm_logging stores on the completed response. + """ + response_obj = ResponsesAPIResponse(id="resp_123", created_at=0, output=[]) + streamed_usage = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage( + self._responses_api_usage() + ).model_dump() + setattr(response_obj, "usage", streamed_usage) + + usage_details = self._log_responses_api_generation(response_obj) + + assert usage_details == { + "input": 12, + "output": 21, + "total": 37, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4, + } + + def test_log_langfuse_v2_malformed_responses_api_usage_still_logs(self): + """A usage dict whose token counts fail validation is logged as zero usage, not dropped.""" + response_obj = ResponsesAPIResponse(id="resp_123", created_at=0, output=[]) + setattr(response_obj, "usage", {"input_tokens": 16, "output_tokens": None}) + + usage_details = self._log_responses_api_generation(response_obj) + + assert usage_details == { + "input": 0, + "output": 0, + "total": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + } + + def _build_standard_logging_payload(self, trace_id: str | None = None): payload = { "id": "payload-id", "call_type": "completion", diff --git a/tests/unit/llms/bedrock/test_guardrail_attachments.py b/tests/unit/llms/bedrock/test_guardrail_attachments.py new file mode 100644 index 00000000000..211747069e2 --- /dev/null +++ b/tests/unit/llms/bedrock/test_guardrail_attachments.py @@ -0,0 +1,651 @@ +import base64 + +import pytest + +from litellm.llms.bedrock.guardrail_attachments import find_request_attachments +from litellm.types.utils import CallTypes + +PNG_B64 = base64.b64encode(b"\x89PNG\r\n\x1a\nfake-png").decode() +JPEG_B64 = base64.b64encode(b"\xff\xd8\xfffake-jpeg").decode() +OVERSIZE_PNG_B64 = base64.b64encode(b"\x89PNG" + b"\0" * (4 * 1024 * 1024)).decode() +PDF_B64 = base64.b64encode(b"%PDF-1.4 fake").decode() + + +def _png_item(encoded: str = PNG_B64) -> dict: + return {"image": {"format": "png", "source": {"bytes": encoded}}} + + +def _jpeg_item() -> dict: + return {"image": {"format": "jpeg", "source": {"bytes": JPEG_B64}}} + + +def _chat(*blocks: object, role: str = "user") -> dict: + return {"messages": [{"role": role, "content": list(blocks)}]} + + +def _converse(*blocks: object, endpoint: str = "model/haiku/converse", provider: str = "bedrock") -> dict: + return { + "endpoint": endpoint, + "custom_llm_provider": provider, + "data": {"messages": [{"role": "user", "content": list(blocks)}]}, + } + + +TEXT = {"type": "text", "text": "hello"} + + +@pytest.mark.parametrize( + "data, call_type, images, unscannable", + [ + pytest.param(_chat(TEXT), CallTypes.acompletion.value, [], [], id="chat-text-only"), + pytest.param( + {"messages": [{"role": "user", "content": "hi"}]}, + CallTypes.acompletion.value, + [], + [], + id="chat-string-content", + ), + pytest.param( + _chat(TEXT, {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{PNG_B64}"}}), + CallTypes.acompletion.value, + [_png_item()], + [], + id="chat-png-data-uri-after-text", + ), + pytest.param( + _chat({"type": "image_url", "image_url": f"data:image/jpeg;base64,{JPEG_B64}"}), + CallTypes.completion.value, + [_jpeg_item()], + [], + id="chat-jpeg-string-url", + ), + pytest.param( + _chat( + {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{PNG_B64}"}}, + TEXT, + {"type": "image_url", "image_url": {"url": f"data:image/jpg;base64,{JPEG_B64}"}}, + ), + CallTypes.acompletion.value, + [_png_item(), _jpeg_item()], + [], + id="chat-two-images", + ), + pytest.param( + _chat({"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}), + CallTypes.acompletion.value, + [], + ["image_url (remote URL or file id)"], + id="chat-remote-image", + ), + pytest.param( + _chat({"type": "image_url", "image_url": {"url": f"data:image/gif;base64,{PNG_B64}"}}), + CallTypes.acompletion.value, + [], + ["image_url (image/gif)"], + id="chat-gif", + ), + pytest.param( + _chat({"type": "image_url", "image_url": {"url": "data:image/png;base64,***not-base64***"}}), + CallTypes.acompletion.value, + [], + ["image_url (invalid base64)"], + id="chat-malformed-base64", + ), + pytest.param( + _chat({"type": "image_url", "image_url": {"url": f"data:image/png;base64,{OVERSIZE_PNG_B64}"}}), + CallTypes.acompletion.value, + [], + ["image_url (over 4 MB)"], + id="chat-image-over-4mb", + ), + pytest.param( + _chat({"type": "image_url", "image_url": {"url": "data:image/png;base64,"}}), + CallTypes.acompletion.value, + [], + ["image_url (invalid base64)"], + id="chat-empty-base64", + ), + pytest.param( + _chat( + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": PNG_B64}}, + {"type": "input_image", "image_url": f"data:image/jpeg;base64,{JPEG_B64}"}, + {"type": "input_file", "file_id": "file-1"}, + ), + CallTypes.acompletion.value, + [_png_item(), _jpeg_item()], + ["input_file"], + id="chat-anthropic-and-responses-shaped-parts", + ), + pytest.param( + {"input": [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "https://x/y.png"}}]}]}, + CallTypes.aresponses.value, + [], + ["image_url (remote URL or file id)"], + id="responses-chat-shaped-image-url", + ), + pytest.param( + _converse({"audio": {"format": "mp3", "source": {"bytes": PDF_B64}}}), + CallTypes.allm_passthrough_route.value, + [], + ["audio"], + id="converse-audio", + ), + pytest.param( + _chat(TEXT, {"type": "file", "file": {"file_data": f"data:application/pdf;base64,{PDF_B64}"}}), + CallTypes.acompletion.value, + [], + ["file"], + id="chat-pdf-file", + ), + pytest.param( + _chat( + {"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}}, + {"type": "video_url", "video_url": {"url": "https://example.com/v.mp4"}}, + ), + CallTypes.acompletion.value, + [], + ["input_audio", "video_url"], + id="chat-audio-and-video", + ), + pytest.param( + _chat( + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": PNG_B64}}, + {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": PDF_B64}}, + ), + CallTypes.anthropic_messages.value, + [_png_item()], + ["document"], + id="anthropic-image-and-document", + ), + pytest.param( + _chat({"type": "image", "source": {"type": "url", "url": "https://example.com/cat.png"}}), + CallTypes.anthropic_messages.value, + [], + ["image (url or file source)"], + id="anthropic-url-image", + ), + pytest.param( + { + "input": [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "hi"}, + {"type": "input_image", "image_url": f"data:image/png;base64,{PNG_B64}"}, + {"type": "input_file", "file_data": f"data:application/pdf;base64,{PDF_B64}"}, + ], + } + ] + }, + CallTypes.aresponses.value, + [_png_item()], + ["input_file"], + id="responses-image-and-file", + ), + pytest.param({"input": "just text"}, CallTypes.responses.value, [], [], id="responses-string-input"), + pytest.param( + _converse({"text": "hi"}, _png_item(), {"document": {"format": "pdf", "source": {"bytes": PDF_B64}}}), + CallTypes.allm_passthrough_route.value, + [_png_item()], + ["document"], + id="converse-image-and-document", + ), + pytest.param( + _converse({"image": {"format": "png", "source": {"s3Location": {"uri": "s3://b/k.png"}}}}), + CallTypes.allm_passthrough_route.value, + [], + ["image (no inline bytes)"], + id="converse-s3-image", + ), + pytest.param( + _converse( + {"video": {"format": "mp4", "source": {"bytes": PDF_B64}}}, endpoint="model/haiku/converse-stream" + ), + CallTypes.allm_passthrough_route.value, + [], + ["video"], + id="converse-stream-video", + ), + pytest.param( + _converse(_png_item(), endpoint="model/haiku/invoke"), + CallTypes.allm_passthrough_route.value, + [], + [], + id="bedrock-invoke-not-converse", + ), + pytest.param( + _converse(_png_item(), provider="vertex_ai"), + CallTypes.allm_passthrough_route.value, + [], + [], + id="non-bedrock-passthrough", + ), + pytest.param( + _chat({"type": "image_url", "image_url": {"url": f"data:image/PNG;BASE64,{PNG_B64}"}}), + CallTypes.acompletion.value, + [_png_item()], + [], + id="chat-uppercase-data-uri", + ), + pytest.param( + _chat( + { + "type": "tool_result", + "tool_use_id": "toolu_1", + "content": [ + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": PNG_B64}}, + {"type": "document", "source": {"type": "url", "url": "https://example.com/r.pdf"}}, + ], + } + ), + CallTypes.anthropic_messages.value, + [_png_item()], + ["document"], + id="anthropic-tool-result-image-and-document", + ), + pytest.param( + { + "input": [ + { + "type": "function_call_output", + "call_id": "call_1", + "output": [ + {"type": "input_image", "image_url": f"data:image/png;base64,{PNG_B64}"}, + {"type": "input_file", "file_id": "file-1"}, + ], + }, + {"type": "function_call_output", "call_id": "call_2", "output": "plain text"}, + ] + }, + CallTypes.aresponses.value, + [_png_item()], + ["input_file"], + id="responses-function-call-output-image-and-file", + ), + pytest.param( + { + "input": [ + { + "type": "computer_call_output", + "call_id": "call_1", + "output": {"type": "computer_screenshot", "image_url": f"data:image/png;base64,{PNG_B64}"}, + }, + { + "type": "computer_call_output", + "call_id": "call_2", + "output": {"type": "computer_screenshot", "file_id": "file-1"}, + }, + ] + }, + CallTypes.responses.value, + [_png_item()], + ["computer_screenshot (remote URL or file id)"], + id="responses-computer-screenshot", + ), + pytest.param( + _converse( + { + "toolResult": { + "toolUseId": "t1", + "content": [{"document": {"format": "pdf", "source": {"bytes": PDF_B64}}}, _png_item()], + } + } + ), + CallTypes.allm_passthrough_route.value, + [_png_item()], + ["document"], + id="converse-tool-result-document-and-image", + ), + pytest.param( + _chat({"type": "image_url", "image_url": {"url": f"data:image/png;base64,{PNG_B64}"}}), + CallTypes.call_mcp_tool.value, + [], + [], + id="mcp-call-type-ignored", + ), + ], +) +def test_find_request_attachments(data, call_type, images, unscannable): + found = find_request_attachments(data, call_type, skip_tool_messages=False, latest_user_message_only=False) + + assert list(found.images) == images + assert list(found.unscannable) == unscannable + + +def test_whitespace_in_base64_is_stripped_before_sending(): + wrapped = PNG_B64[:8] + "\n" + PNG_B64[8:] + data = _chat({"type": "image_url", "image_url": {"url": f"data:image/png;base64,{wrapped}"}}) + + found = find_request_attachments(data, CallTypes.acompletion.value, False, False) + + assert list(found.images) == [_png_item()] + + +def test_tool_messages_skipped_only_when_asked(): + data = { + "messages": [ + {"role": "user", "content": [TEXT]}, + {"role": "tool", "content": [{"type": "file", "file": {"file_id": "file-1"}}]}, + ] + } + + scanned = find_request_attachments(data, CallTypes.acompletion.value, False, False) + skipped = find_request_attachments(data, CallTypes.acompletion.value, True, False) + + assert scanned.unscannable == ("file",) + assert skipped.unscannable == () + + +def test_scan_only_tool_results_keeps_only_tool_output(): + data = _chat( + {"type": "document", "source": {"type": "url", "url": "https://example.com/user.pdf"}}, + {"type": "tool_result", "tool_use_id": "toolu_1", "content": [{"type": "document", "source": {}}]}, + ) + + everything = find_request_attachments(data, CallTypes.anthropic_messages.value, False, False) + tool_only = find_request_attachments(data, CallTypes.anthropic_messages.value, False, False, True) + skip_tool = find_request_attachments(data, CallTypes.anthropic_messages.value, True, False) + + assert everything.unscannable == ("document", "document") + assert tool_only.unscannable == ("document",) + assert skip_tool.unscannable == ("document",) + assert find_request_attachments(data, CallTypes.anthropic_messages.value, True, False, True).unscannable == () + + +def test_long_mime_is_truncated_in_label(): + data = _chat({"type": "image_url", "image_url": {"url": f"data:image/{'x' * 500};base64,{PNG_B64}"}}) + + found = find_request_attachments(data, CallTypes.acompletion.value, False, False) + + assert found.unscannable == (f"image_url (image/{'x' * 34})",) + + +def test_latest_user_message_only(): + data = { + "messages": [ + {"role": "user", "content": [{"type": "file", "file": {"file_id": "old"}}]}, + {"role": "assistant", "content": "ok"}, + {"role": "user", "content": [{"type": "image_url", "image_url": f"data:image/png;base64,{PNG_B64}"}]}, + ] + } + + found = find_request_attachments(data, CallTypes.acompletion.value, False, True) + + assert list(found.images) == [_png_item()] + assert found.unscannable == () + + +def test_non_string_role_and_type_do_not_raise(): + data = { + "messages": [ + {"role": "user", "content": [{"type": "image_url", "image_url": f"data:image/png;base64,{PNG_B64}"}]}, + {"role": ["tool"], "type": {"x": 1}, "content": [{"type": "file", "file": {"file_id": "f"}}]}, + {"role": {"r": "tool"}, "type": ["function_call_output"], "content": [{"type": "file", "file": {}}]}, + ] + } + + found = find_request_attachments(data, CallTypes.acompletion.value, True, False) + + assert list(found.images) == [_png_item()] + assert found.unscannable == ("file", "file") + + +def test_converse_null_document_is_not_an_attachment(): + data = _converse(_png_item(), {"document": None}, {"video": None, "audio": None}, {"audio": {"format": "mp3"}}) + + found = find_request_attachments(data, CallTypes.allm_passthrough_route.value, False, False) + + assert list(found.images) == [_png_item()] + assert found.unscannable == ("audio",) + + +@pytest.mark.parametrize( + "block", + [ + pytest.param( + {"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "hi"}}, id="text" + ), + pytest.param( + {"type": "document", "source": {"type": "content", "content": [{"type": "text", "text": "hi"}]}}, + id="content", + ), + pytest.param({"type": "document", "source": {"type": "content", "content": "hi"}}, id="content-string"), + ], +) +@pytest.mark.parametrize( + "call_type", [CallTypes.anthropic_messages.value, CallTypes.acompletion.value], ids=["messages", "chat"] +) +def test_text_source_document_is_not_an_attachment(block, call_type): + pdf = {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": PDF_B64}} + + found = find_request_attachments(_chat(TEXT, block, pdf), call_type, False, False) + + assert found.images == () + assert found.unscannable == ("document",) + assert found.document_texts == ("hi",) + + +def test_unpadded_and_url_safe_base64_are_sent_as_standard_base64(): + raw = b"\x89PNG\r\n\x1a\n\xfb\xff\xfe-fake" + standard = base64.b64encode(raw).decode() + unpadded = standard.rstrip("=") + url_safe = base64.urlsafe_b64encode(raw).decode().rstrip("=") + data = _chat( + *( + {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{encoded}"}} + for encoded in (unpadded, url_safe) + ) + ) + + found = find_request_attachments(data, CallTypes.acompletion.value, False, False) + + assert unpadded != standard + assert set("-_") & set(url_safe) + assert list(found.images) == [_png_item(standard), _png_item(standard)] + assert found.unscannable == () + + +def test_oversize_base64_is_refused_by_length_before_decoding(): + encoded = "!" * (len(OVERSIZE_PNG_B64) + 4) + data = _chat({"type": "image_url", "image_url": {"url": f"data:image/png;base64,{encoded}"}}) + + found = find_request_attachments(data, CallTypes.acompletion.value, False, False) + + assert found.unscannable == ("image_url (over 4 MB)",) + + +@pytest.mark.parametrize( + "call_type", [CallTypes.anthropic_messages.value, CallTypes.acompletion.value], ids=["messages", "chat"] +) +def test_images_inside_a_content_source_document_are_scanned(call_type): + image = {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": PNG_B64}} + pdf = {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": PDF_B64}} + block = {"type": "document", "source": {"type": "content", "content": [TEXT, image, pdf]}} + + found = find_request_attachments(_chat(block), call_type, False, False) + + assert list(found.images) == [_png_item()] + assert found.unscannable == ("document",) + assert found.document_texts == ("hello",) + + +def test_document_images_inside_a_tool_result_follow_the_tool_scope(): + image = {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": PNG_B64}} + document = {"type": "document", "source": {"type": "content", "content": [image]}} + data = _chat({"type": "tool_result", "tool_use_id": "t1", "content": [document]}) + + scanned = find_request_attachments(data, CallTypes.anthropic_messages.value, False, False) + skipped = find_request_attachments(data, CallTypes.anthropic_messages.value, True, False) + + assert list(scanned.images) == [_png_item()] + assert skipped.images == () + + +def test_document_title_and_context_are_scanned_as_text(): + text_doc = { + "type": "document", + "title": "record", + "context": "SSN 123-45-6789", + "source": {"type": "text", "media_type": "text/plain", "data": "body"}, + } + pdf = { + "type": "document", + "context": "note", + "source": {"type": "base64", "media_type": "application/pdf", "data": PDF_B64}, + } + + found = find_request_attachments(_chat(text_doc, pdf), CallTypes.anthropic_messages.value, False, False) + + assert found.document_texts == ("record\nSSN 123-45-6789\nbody", "note") + assert found.unscannable == ("document",) + + +def _content_document(*inner: object) -> dict: + return {"type": "document", "source": {"type": "content", "content": list(inner)}} + + +def test_nested_documents_are_scanned_and_deep_nesting_is_refused(): + inner_text = {"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "inner"}} + nested = _content_document(inner_text, _content_document({"type": "text", "text": "deeper"})) + too_deep = _content_document(_content_document(_content_document(_content_document(TEXT)))) + + found = find_request_attachments(_chat(nested, too_deep), CallTypes.anthropic_messages.value, False, False) + + assert found.document_texts == ("inner", "deeper") + assert found.unscannable == ("document (nested too deep)",) + + +def _png_bytes(width: int, height: int) -> bytes: + return b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR" + width.to_bytes(4, "big") + height.to_bytes(4, "big") + b"\x08\x02" + + +def _jpeg_bytes(width: int, height: int) -> bytes: + app0 = b"\xff\xe0\x00\x10JFIF\x00\x01\x01\x00\x00\x01\x00\x01\x00\x00" + sof0 = b"\xff\xc0\x00\x11\x08" + height.to_bytes(2, "big") + width.to_bytes(2, "big") + b"\x03" + return b"\xff\xd8" + app0 + b"\xff" + sof0 + + +def _image_url(mime: str, raw: bytes) -> dict: + return {"type": "image_url", "image_url": {"url": f"data:{mime};base64,{base64.b64encode(raw).decode()}"}} + + +@pytest.mark.parametrize( + "mime, raw, sent_format", + [ + pytest.param("image/png", _jpeg_bytes(256, 256), "jpeg", id="jpeg-labeled-png"), + pytest.param("image/jpeg", _png_bytes(256, 256), "png", id="png-labeled-jpeg"), + pytest.param("image/png", _png_bytes(8000, 8000), "png", id="png-at-pixel-limit"), + pytest.param("image/jpeg", _jpeg_bytes(8000, 10), "jpeg", id="jpeg-at-pixel-limit"), + ], +) +def test_image_format_is_taken_from_the_bytes(mime, raw, sent_format): + found = find_request_attachments(_chat(_image_url(mime, raw)), CallTypes.acompletion.value, False, False) + + assert [item["image"]["format"] for item in found.images] == [sent_format] + assert found.unscannable == () + + +@pytest.mark.parametrize( + "mime, raw, label", + [ + pytest.param("image/png", _png_bytes(8001, 10), "image_url (over 8000 pixels)", id="png-too-wide"), + pytest.param("image/png", _png_bytes(10, 9000), "image_url (over 8000 pixels)", id="png-too-tall"), + pytest.param("image/jpeg", _jpeg_bytes(9000, 10), "image_url (over 8000 pixels)", id="jpeg-too-wide"), + pytest.param("image/jpeg", _jpeg_bytes(10, 8001), "image_url (over 8000 pixels)", id="jpeg-too-tall"), + pytest.param("image/png", b"GIF89a\x01\x00\x01\x00", "image_url (not PNG or JPEG data)", id="gif-labeled-png"), + ], +) +def test_images_bedrock_cannot_scan_are_unscannable(mime, raw, label): + data = _chat(_image_url("image/png", _png_bytes(16, 16)), _image_url(mime, raw)) + + found = find_request_attachments(data, CallTypes.acompletion.value, False, False) + + assert len(found.images) == 1 + assert found.unscannable == (label,) + + +def _converse_document(source: dict, **fields: str) -> dict: + return {"document": {"format": "txt", "name": "notes", **fields, "source": source}} + + +@pytest.mark.parametrize( + "block, document_texts", + [ + pytest.param(_converse_document({"text": "SSN 123-45-6789"}), ("notes\nSSN 123-45-6789",), id="text"), + pytest.param( + _converse_document({"text": "body"}, context="SSN 123-45-6789"), + ("notes\nSSN 123-45-6789\nbody",), + id="text-with-context", + ), + pytest.param( + _converse_document({"content": [{"text": "first"}, {"text": "SSN 123-45-6789"}, {"text": "last"}]}), + ("notes\nfirst\nSSN 123-45-6789\nlast",), + id="content", + ), + pytest.param(_converse_document({"text": ""}, name=""), (), id="empty-text"), + ], +) +def test_converse_text_source_document_is_scanned_as_text(block, document_texts): + found = find_request_attachments( + _converse({"text": "hi"}, block), CallTypes.allm_passthrough_route.value, False, False + ) + + assert found.document_texts == document_texts + assert found.images == () + assert found.unscannable == () + + +@pytest.mark.parametrize( + "source", + [ + pytest.param({"bytes": PDF_B64}, id="bytes"), + pytest.param({"s3Location": {"uri": "s3://b/k.txt"}}, id="s3"), + pytest.param({"text": "hi", "bytes": PDF_B64}, id="text-and-bytes"), + pytest.param({"content": [{"text": "hi"}, {"image": {"format": "png"}}]}, id="content-with-non-text"), + pytest.param({"content": [{"text": "hi"}, "raw"]}, id="content-with-string"), + pytest.param({"content": "hi"}, id="content-not-a-list"), + pytest.param({}, id="no-source-fields"), + ], +) +def test_converse_document_without_text_source_stays_unscannable(source): + found = find_request_attachments( + _converse(_converse_document(source)), CallTypes.allm_passthrough_route.value, False, False + ) + + assert found.document_texts == ("notes",) + assert found.unscannable == ("document",) + + +def test_converse_binary_document_name_and_context_are_still_scanned_as_text(): + data = _converse( + _converse_document({"bytes": PDF_B64}, context="SSN 123-45-6789"), + {"toolResult": {"toolUseId": "t1", "content": [_converse_document({"s3Location": {"uri": "s3://b/k.pdf"}})]}}, + ) + + found = find_request_attachments(data, CallTypes.allm_passthrough_route.value, False, False) + + assert found.document_texts == ("notes\nSSN 123-45-6789", "notes") + assert found.unscannable == ("document", "document") + + +def test_converse_text_document_in_tool_result_follows_the_tool_scope(): + data = _converse({"toolResult": {"toolUseId": "t1", "content": [_converse_document({"text": "SSN 123-45-6789"})]}}) + + scanned = find_request_attachments(data, CallTypes.allm_passthrough_route.value, False, False) + skipped = find_request_attachments(data, CallTypes.allm_passthrough_route.value, True, False) + + assert scanned.document_texts == ("notes\nSSN 123-45-6789",) + assert skipped.document_texts == () + + +def test_converse_guard_content_image_is_scanned(): + data = _converse( + {"guardContent": {"image": _png_item()["image"]}}, + {"guardContent": {"image": {"format": "png", "source": {"s3Location": {"uri": "s3://b/k.png"}}}}}, + {"guardContent": {"text": {"text": "hi"}}}, + ) + + found = find_request_attachments(data, CallTypes.allm_passthrough_route.value, False, False) + + assert list(found.images) == [_png_item()] + assert found.unscannable == ("image (no inline bytes)",) diff --git a/tests/unit/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py b/tests/unit/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py index 9f717f522c0..6254b1bc44c 100644 --- a/tests/unit/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py +++ b/tests/unit/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py @@ -10814,3 +10814,39 @@ async def test_admission_request_body_serves_stashed_peek_callable(): {"type": "http", "method": "POST", "path": "/mcp", "headers": []} ) assert await without_peek.body() == b"{}" + + +@pytest.mark.parametrize("bearer,api_key,claims,explicit_key,expected", [ + (None, "sk-key", None, None, None), + ("sk-key", "sk-key", None, None, None), + ("sk-key", "sk-key", {"sub": "previous-jwt"}, "sk-key", None), + ("caller.jwt.assertion", "sk-key", {"sub": "caller"}, None, "caller.jwt.assertion"), + ("caller.jwt.assertion", None, {"sub": "caller"}, None, "caller.jwt.assertion"), + ("upstream", "sk-key", None, "sk-key", "upstream"), + ("llm_session_test", None, {"sub": "caller"}, None, None), + ("llm_srefresh_test", None, {"sub": "caller"}, None, None), + ("opaque-master", "litellm_proxy_master_key", None, None, None), +]) +def test_guardrail_bearer_preserves_identity_without_exposing_gateway_credentials( + bearer: str | None, api_key: str | None, claims: dict[str, str] | None, + explicit_key: str | None, expected: str | None, +) -> None: + headers: Final = Headers({ + **({"Authorization": f"Bearer {bearer}"} if bearer is not None else {}), + **({"x-litellm-api-key": explicit_key} if explicit_key else {}), + }) + token: Final = MCPRequestHandler.get_guardrail_bearer_token( + headers, UserAPIKeyAuth(api_key=api_key, jwt_claims=claims), + ) + assert (token.get_secret_value() if token is not None else None) == expected + + +def test_guardrail_bearer_respects_custom_admission_header(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy.proxy_server import general_settings + + monkeypatch.setitem(general_settings, "litellm_key_header_name", "x-gateway-key") + token: Final = MCPRequestHandler.get_guardrail_bearer_token( + Headers({"x-gateway-key": "opaque-master", "authorization": "Bearer opaque-master"}), + UserAPIKeyAuth(api_key="litellm_proxy_master_key"), + ) + assert token is None diff --git a/tests/unit/proxy/_experimental/mcp_server/test_operations.py b/tests/unit/proxy/_experimental/mcp_server/test_operations.py index 32a7c3fbe58..dcc4eb1a570 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_operations.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_operations.py @@ -71,7 +71,7 @@ async def test_mapped_caller_bearer_reaches_agent_evaluation( "headers": [] if verdict == "missing" else [(b"authorization", f"Bearer {bearer}".encode())], }) auth: Final = UserAPIKeyAuth( - api_key="stored-key-hash", user_id="mapped-caller", + api_key="stored-key-hash", user_id="mapped-caller", jwt_claims={"sub": "mapped-caller"}, object_permission={"object_permission_id": "mapped", "mcp_servers": ["guarded"], "mcp_tool_search_enabled": mode == "virtual"}, ) oauth, raw, _, _ = MCPRequestHandler.scrub_gateway_admission_credentials( @@ -1552,3 +1552,41 @@ def test_discovery_extra_headers_exclude_gateway_admission_key(header_name: str) assert auth_header is None assert extra_headers == {"X-Tenant": "tenant-control"} assert raw_headers == {"x-litellm-api-key": caller_key, "x-tenant": "tenant-control"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("request_kind", ["absent", "non_starlette", "without_bearer", "admission_key"]) +async def test_guardrail_context_never_recovers_a_previous_request_bearer( + _mcp_request_ctx: Callable[..., ServerRequestContext], request_kind: str, monkeypatch: pytest.MonkeyPatch, +) -> None: + from mcp.server.auth.middleware.auth_context import auth_context_var + from starlette.requests import Request + from litellm.proxy._experimental.mcp_server import server as gateway + + request: Final = ( + None if request_kind == "absent" else object() if request_kind == "non_starlette" else Request({ + "type": "http", "method": "POST", "path": "/mcp", + "headers": [(b"authorization", b"Bearer sk-current-key")] if request_kind == "admission_key" else [], + }) + ) + previous: Final = auth_context_var.set(None) + try: + gateway.set_auth_context( + UserAPIKeyAuth(api_key="sk-current-key"), + raw_headers={"authorization": "Bearer previous.caller.assertion"}, + ) + capture: Final = _CatalogHookCapture() + monkeypatch.setattr(litellm, "callbacks", [capture]) + async with gateway._legacy_operation_context(_mcp_request_ctx(request=request), trace=False) as context: + await operations.global_mcp_server_manager.pre_call_tool_check( + name="echo", arguments={}, server_name="guarded", + user_api_key_auth=context.user_api_key_auth, + proxy_logging_obj=ProxyLogging(user_api_key_cache=UserApiKeyCache()), + server=MCPServer(server_id="guarded", name="guarded", transport=MCPTransport.http, allow_all_keys=True), + raw_headers=dict(context.raw_headers or {}), + incoming_bearer_token=context.incoming_bearer_token, + ) + assert capture.data is not None + assert capture.data["incoming_bearer_token"] is None + finally: + auth_context_var.reset(previous) diff --git a/tests/unit/proxy/_experimental/mcp_server/test_rest_endpoints.py b/tests/unit/proxy/_experimental/mcp_server/test_rest_endpoints.py index fc61ec66b30..2a04c8cda2a 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_rest_endpoints.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_rest_endpoints.py @@ -4862,3 +4862,70 @@ async def test_saved_preview_protocol_omission_and_explicit_edits( assert result == {"protocol_version": expected} assert saved.protocol_version == "2025-11-25" assert payload.mcp_info == metadata + + +@pytest.mark.asyncio +@pytest.mark.parametrize("virtual", [False, True]) +@pytest.mark.parametrize("bearer,claims,expected", [ + ("sk-guardrail-admission", None, None), + ("caller.jwt.assertion", {"sub": "caller"}, "caller.jwt.assertion"), + ("upstream-token", None, "upstream-token"), +]) +async def test_rest_guardrail_bearer_excludes_admission_key( + monkeypatch: pytest.MonkeyPatch, virtual: bool, + bearer: str, claims: dict[str, str] | None, expected: str | None, +) -> None: + import litellm + from litellm.caching.dual_cache import DualCache + from litellm.integrations.custom_logger import CustomLogger + from litellm.proxy._experimental.mcp_server.tool_registry import global_mcp_tool_registry + + observed: Final[asyncio.Queue[object]] = asyncio.Queue() + + class CaptureBearer(CustomLogger): + async def async_pre_call_hook( + self, user_api_key_dict: UserAPIKeyAuth, cache: DualCache, + data: dict[str, object], call_type: str, + ) -> None: + if call_type == "call_mcp_tool" and "incoming_bearer_token" in data: + observed.put_nowait(data["incoming_bearer_token"]) + + async def echo() -> str: + return "guarded-result" + + auth: Final = UserAPIKeyAuth( + api_key="sk-guardrail-admission", jwt_claims=claims, + object_permission={"object_permission_id": "guarded", "mcp_servers": ["guarded"], "mcp_tool_search_enabled": virtual}, + ) + managed: Final = MCPServer( + server_id="guarded", name="guarded", transport=MCPTransport.http, + spec_path="/catalog.yaml", allow_all_keys=True, + ) + monkeypatch.setitem(rest_endpoints.global_mcp_server_manager.registry, managed.server_id, managed) + monkeypatch.setattr(litellm, "callbacks", [CaptureBearer()]) + global_mcp_tool_registry.register_tool( + name="guarded-echo", description="Echo", input_schema={"type": "object"}, handler=echo, + ) + headers: Final = {"authorization": f"Bearer {bearer}", **( + {} if claims else {"x-litellm-api-key": "sk-guardrail-admission"} + )} + payload: Final = { + "server_id": "guarded", "name": "mcp_tool_call" if virtual else "guarded-echo", + "arguments": {"tool_name": "guarded-echo", "arguments": {}} if virtual else {}, + } + try: + result: Final = await rest_endpoints.call_tool_rest_api( + _build_request(headers, path="/mcp-rest/tools/call", json_body=payload), + user_api_key_dict=auth, + ) + assert result.is_error is False + assert observed.qsize() == 1 + assert observed.get_nowait() == expected + if claims is None: + list_headers: Final = rest_endpoints._extract_mcp_headers_from_request( + _build_request(headers), auth, + )[2] + authorization: Final = list_headers.get("authorization", "") + assert (authorization.removeprefix("Bearer ") or None) == expected + finally: + global_mcp_tool_registry.unregister_tools_with_prefix("guarded-") diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py index ccf3852c2c5..3b33ba75eb3 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py @@ -4,6 +4,7 @@ Unit tests for Bedrock Guardrails import json import asyncio +import logging from typing import Final from datetime import datetime, timezone import sys @@ -16,6 +17,7 @@ from fastapi import HTTPException import litellm +from litellm._logging import verbose_proxy_logger from litellm.caching.caching import DualCache from litellm.exceptions import ModifyResponseException from litellm.proxy._types import UserAPIKeyAuth @@ -6104,7 +6106,9 @@ async def test_bearer_token_never_runs_the_sigv4_credential_chain(monkeypatch): mock_response.json.return_value = {"action": "NONE", "assessments": []} with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock, return_value=mock_response) as mock_post: - response = await guardrail.make_bedrock_api_request(source="INPUT", messages=[{"role": "user", "content": "hello"}]) + response = await guardrail.make_bedrock_api_request( + source="INPUT", messages=[{"role": "user", "content": "hello"}] + ) assert response["action"] == "NONE" assert mock_post.call_args.kwargs["headers"]["Authorization"] == "Bearer env-bearer-token-12345" @@ -6143,6 +6147,443 @@ async def test_apply_guardrail_signs_off_the_event_loop(monkeypatch): assert probe.served_during_refresh is True +_ATTACHMENT_PNG_B64 = "iVBORw0KGgpmYWtlLXBuZw==" +_ATTACHMENT_PDF_URI = "data:application/pdf;base64,JVBERi0xLjQgZmFrZQ==" + + +def _image_only_chat_request() -> dict: + return { + "model": "claude", + "messages": [ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{_ATTACHMENT_PNG_B64}"}} + ], + } + ], + } + + +def _pdf_chat_request() -> dict: + return { + "model": "claude", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "summarize"}, + {"type": "file", "file": {"file_data": _ATTACHMENT_PDF_URI}}, + ], + } + ], + } + + +def _attachment_guardrail(**kwargs) -> BedrockGuardrail: + return BedrockGuardrail( + guardrail_name="bedrock-attachments", + guardrailIdentifier="gid", + guardrailVersion="DRAFT", + **kwargs, + ) + + +def _patched_bedrock_post(guardrail: BedrockGuardrail, response: MagicMock): + credentials = MagicMock(access_key="k", secret_key="s", token=None) + return ( + patch.object(guardrail.async_handler, "post", new_callable=AsyncMock, return_value=response), + patch.object(guardrail, "_load_credentials", return_value=(credentials, "us-east-1")), + patch.object(guardrail, "_prepare_request", return_value=MagicMock()), + ) + + +@pytest.mark.asyncio +async def test_attachment_scan_sends_image_only_turn_to_apply_guardrail_and_blocks(): + guardrail = _attachment_guardrail() + post_patch, credentials_patch, prepare_patch = _patched_bedrock_post( + guardrail, _blocking_bedrock_httpx_response("violence") + ) + + with post_patch as mock_post, credentials_patch, prepare_patch as mock_prepare: + with pytest.raises(HTTPException) as exc_info: + await guardrail.async_scan_request_attachments( + data=_image_only_chat_request(), call_type=CallTypes.acompletion.value + ) + + assert exc_info.value.status_code == 400 + assert mock_post.await_count == 1 + sent_body = mock_prepare.call_args.kwargs["data"] + assert sent_body["source"] == "INPUT" + assert sent_body["content"] == ({"image": {"format": "png", "source": {"bytes": _ATTACHMENT_PNG_B64}}},) + + +@pytest.mark.asyncio +async def test_attachment_scan_lets_a_passing_image_through(): + guardrail = _attachment_guardrail() + data = _image_only_chat_request() + post_patch, credentials_patch, prepare_patch = _patched_bedrock_post( + guardrail, _passing_bedrock_httpx_response("ok") + ) + + with post_patch as mock_post, credentials_patch, prepare_patch: + await guardrail.async_scan_request_attachments(data=data, call_type=CallTypes.acompletion.value) + + assert mock_post.await_count == 1 + logged = data["metadata"]["standard_logging_guardrail_information"] + assert [entry["guardrail_status"] for entry in logged] == ["success"] + + +@pytest.mark.asyncio +async def test_attachment_scan_blocks_unscannable_attachment_without_calling_bedrock(): + guardrail = _attachment_guardrail() + data = _pdf_chat_request() + + with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as mock_post: + with pytest.raises(HTTPException) as exc_info: + await guardrail.async_scan_request_attachments(data=data, call_type=CallTypes.acompletion.value) + + assert exc_info.value.status_code == 400 + assert exc_info.value.detail["error"] == "Violated guardrail policy" + assert "cannot scan 1 attachment(s) (file)" in exc_info.value.detail["reason"] + assert exc_info.value.detail["guardrailIdentifier"] == "gid" + mock_post.assert_not_awaited() + logged = data["metadata"]["standard_logging_guardrail_information"] + assert logged[0]["guardrail_status"] == "guardrail_intervened" + assert logged[0]["guardrail_response"] == {"unscannable_attachments": ["file"]} + + +@pytest.mark.asyncio +async def test_attachment_scan_skip_unscannable_attachments_lets_documents_through(): + guardrail = _attachment_guardrail(skip_unscannable_attachments=True) + data = _pdf_chat_request() + + with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as mock_post: + result = await guardrail.async_scan_request_attachments(data=data, call_type=CallTypes.acompletion.value) + + assert result is None + assert data == _pdf_chat_request() + mock_post.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_attachment_scan_block_returns_a_response_when_exceptions_are_disabled(): + guardrail = _attachment_guardrail(disable_exception_on_block=True) + + with pytest.raises(ModifyResponseException) as exc_info: + await guardrail.async_scan_request_attachments(data=_pdf_chat_request(), call_type=CallTypes.acompletion.value) + + assert "cannot scan 1 attachment(s) (file)" in exc_info.value.message + + +@pytest.mark.asyncio +async def test_attachment_scan_treats_images_as_unscannable_in_checks_mode(): + guardrail = BedrockGuardrail(guardrail_name="bedrock-checks", checks={"contentFilter": {}}) + + with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as mock_post: + with pytest.raises(HTTPException) as exc_info: + await guardrail.async_scan_request_attachments( + data=_image_only_chat_request(), call_type=CallTypes.acompletion.value + ) + + assert "cannot scan 1 attachment(s) (image)" in exc_info.value.detail["reason"] + mock_post.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_attachment_scan_does_nothing_for_text_only_requests(): + guardrail = _attachment_guardrail() + data = {"model": "claude", "messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}]}]} + + with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as mock_post: + await guardrail.async_scan_request_attachments(data=data, call_type=CallTypes.acompletion.value) + + mock_post.assert_not_awaited() + assert "metadata" not in data + + +@pytest.mark.asyncio +async def test_during_call_hook_scans_image_only_turn(): + guardrail = _attachment_guardrail(event_hook=GuardrailEventHooks.during_call, default_on=True) + post_patch, credentials_patch, prepare_patch = _patched_bedrock_post( + guardrail, _blocking_bedrock_httpx_response("violence") + ) + + with post_patch as mock_post, credentials_patch, prepare_patch: + with pytest.raises(HTTPException): + await guardrail.async_moderation_hook( + data=_image_only_chat_request(), + user_api_key_dict=UserAPIKeyAuth(), + call_type=CallTypes.acompletion.value, + ) + + assert mock_post.await_count == 1 + + +@pytest.mark.asyncio +async def test_attachment_scan_in_checks_mode_never_posts_images_even_when_skipping(): + guardrail = BedrockGuardrail( + guardrail_name="bedrock-checks", checks={"contentFilter": {}}, skip_unscannable_attachments=True + ) + + data = _image_only_chat_request() + + with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as mock_post: + result = await guardrail.async_scan_request_attachments(data=data, call_type=CallTypes.acompletion.value) + + assert result is None + assert data == _image_only_chat_request() + mock_post.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_attachment_scan_sends_at_most_twenty_images_per_call(): + guardrail = _attachment_guardrail() + data = _image_only_chat_request() + data["messages"][0]["content"] = data["messages"][0]["content"] * 25 + post_patch, credentials_patch, prepare_patch = _patched_bedrock_post( + guardrail, _passing_bedrock_httpx_response("ok") + ) + + with post_patch as mock_post, credentials_patch, prepare_patch as mock_prepare: + await guardrail.async_scan_request_attachments(data=data, call_type=CallTypes.acompletion.value) + + assert mock_post.await_count == 2 + assert [len(call.kwargs["data"]["content"]) for call in mock_prepare.call_args_list] == [20, 5] + + +@pytest.mark.asyncio +async def test_during_call_hook_skips_attachment_scan_when_guardrail_is_not_requested(): + guardrail = _attachment_guardrail(event_hook=GuardrailEventHooks.during_call, default_on=False) + + with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as mock_post: + result = await guardrail.async_moderation_hook( + data=_pdf_chat_request(), user_api_key_dict=UserAPIKeyAuth(), call_type="acompletion" + ) + + assert result is None + assert mock_post.await_count == 0 + + +@pytest.mark.asyncio +async def test_attachment_scan_honors_scan_only_tool_results(): + guardrail = _attachment_guardrail() + guardrail.scan_only_tool_results = True + data = _pdf_chat_request() + + with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as mock_post: + result = await guardrail.async_scan_request_attachments(data=data, call_type=CallTypes.acompletion.value) + + assert result is None + assert mock_post.await_count == 0 + assert data == _pdf_chat_request() + + +@pytest.mark.asyncio +async def test_native_pre_call_hook_blocks_unscannable_attachment(): + guardrail = _attachment_guardrail(event_hook=GuardrailEventHooks.pre_call, default_on=True) + + with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as mock_post: + with pytest.raises(HTTPException) as exc_info: + await guardrail.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=MagicMock(), + data=_pdf_chat_request(), + call_type="acompletion", + ) + + assert exc_info.value.status_code == 400 + assert "cannot scan 1 attachment(s) (file)" in exc_info.value.detail["reason"] + assert mock_post.await_count == 0 + + +@pytest.mark.asyncio +async def test_attachment_scan_debug_log_omits_image_bytes(): + guardrail = _attachment_guardrail() + post_patch, credentials_patch, prepare_patch = _patched_bedrock_post( + guardrail, _passing_bedrock_httpx_response("ok") + ) + + with ( + post_patch, + credentials_patch, + prepare_patch, + patch("litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails.verbose_proxy_logger.debug") as mock_debug, + ): + await guardrail.async_scan_request_attachments( + data=_image_only_chat_request(), call_type=CallTypes.acompletion.value + ) + + logged = " ".join(str(call.args) for call in mock_debug.call_args_list) + assert _ATTACHMENT_PNG_B64 not in logged + assert f"<{len(_ATTACHMENT_PNG_B64)} base64 chars>" in logged + + +@pytest.mark.asyncio +async def test_text_only_debug_log_prints_the_signed_request_body(): + guardrail = _attachment_guardrail() + post_patch, credentials_patch, prepare_patch = _patched_bedrock_post( + guardrail, _passing_bedrock_httpx_response("ok") + ) + captured_records: list[logging.LogRecord] = [] + + class _RecordingHandler(logging.Handler): + def emit(self, record: logging.LogRecord) -> None: + captured_records.append(record) + + handler = _RecordingHandler(level=logging.DEBUG) + previous_level = verbose_proxy_logger.level + verbose_proxy_logger.addHandler(handler) + verbose_proxy_logger.setLevel(logging.DEBUG) + try: + with post_patch, credentials_patch, prepare_patch: + await guardrail.make_bedrock_api_request( + source="INPUT", messages=[{"role": "user", "content": "hello"}], request_data={} + ) + finally: + verbose_proxy_logger.removeHandler(handler) + verbose_proxy_logger.setLevel(previous_level) + + body_lines = [ + record.getMessage() for record in captured_records if record.getMessage().startswith("Bedrock AI request body") + ] + assert len(body_lines) == 1 + assert "'content': ({'text': {'text': 'hello'}},)" in body_lines[0] + + +def _text_document_messages_request(text: str) -> dict: + return { + "model": "claude", + "messages": [ + { + "role": "user", + "content": [ + {"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": text}}, + {"type": "text", "text": "summarize"}, + ], + } + ], + } + + +def _anonymizing_bedrock_httpx_response() -> MagicMock: + response = MagicMock() + response.status_code = 200 + response.json.return_value = { + "action": "GUARDRAIL_INTERVENED", + "outputs": [{"text": "SSN {US_SOCIAL_SECURITY_NUMBER}"}], + "assessments": [ + { + "sensitiveInformationPolicy": { + "piiEntities": [{"type": "US_SOCIAL_SECURITY_NUMBER", "action": "ANONYMIZED"}] + } + } + ], + "usage": {"contentPolicyUnits": 1}, + } + return response + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "response, blocked", + [ + pytest.param(_passing_bedrock_httpx_response("ok"), False, id="pass"), + pytest.param(_blocking_bedrock_httpx_response("deny"), True, id="block"), + pytest.param(_anonymizing_bedrock_httpx_response(), True, id="anonymize"), + ], +) +async def test_attachment_scan_sends_text_document_as_text(response, blocked): + guardrail = _attachment_guardrail() + post_patch, credentials_patch, prepare_patch = _patched_bedrock_post(guardrail, response) + + with post_patch as mock_post, credentials_patch, prepare_patch as mock_prepare: + if blocked: + with pytest.raises(HTTPException) as exc_info: + await guardrail.async_scan_request_attachments( + data=_text_document_messages_request("SSN 123-45-6789"), + call_type=CallTypes.anthropic_messages.value, + ) + assert exc_info.value.status_code == 400 + else: + result = await guardrail.async_scan_request_attachments( + data=_text_document_messages_request("SSN 123-45-6789"), + call_type=CallTypes.anthropic_messages.value, + ) + assert result is None + + assert mock_post.await_count == 1 + sent_body = mock_prepare.call_args.kwargs["data"] + assert sent_body["source"] == "INPUT" + assert sent_body["content"] == ({"text": {"text": "SSN 123-45-6789"}},) + + +class _CustomApplyGuardrail(BedrockGuardrail): + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + return inputs + + +@pytest.mark.asyncio +async def test_attachment_scan_skipped_when_subclass_overrides_apply_guardrail(): + guardrail = _CustomApplyGuardrail( + guardrail_name="bedrock-attachments", guardrailIdentifier="gid", guardrailVersion="DRAFT" + ) + + with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as mock_post: + pdf_result = await guardrail.async_scan_request_attachments( + data=_pdf_chat_request(), call_type=CallTypes.acompletion.value + ) + image_result = await guardrail.async_scan_request_attachments( + data=_image_only_chat_request(), call_type=CallTypes.acompletion.value + ) + + assert pdf_result is None + assert image_result is None + assert mock_post.await_count == 0 + + +@pytest.mark.asyncio +async def test_attachment_scan_skipped_when_instance_replaces_make_bedrock_api_request(): + guardrail = BedrockGuardrail( + guardrail_name="bedrock-attachments", guardrailIdentifier="gid", guardrailVersion="DRAFT" + ) + guardrail.make_bedrock_api_request = AsyncMock(return_value={"action": "NONE"}) + + with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as mock_post: + pdf_result = await guardrail.async_scan_request_attachments( + data=_pdf_chat_request(), call_type=CallTypes.acompletion.value + ) + + assert pdf_result is None + assert mock_post.await_count == 0 + + +class _CustomRequestGuardrail(BedrockGuardrail): + async def make_bedrock_api_request(self, source, messages=None, response=None, request_data=None, **kwargs): + return {"action": "NONE"} + + +@pytest.mark.asyncio +async def test_attachment_scan_skipped_when_subclass_overrides_make_bedrock_api_request(): + guardrail = _CustomRequestGuardrail( + guardrail_name="bedrock-attachments", guardrailIdentifier="gid", guardrailVersion="DRAFT" + ) + + with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as mock_post: + pdf_result = await guardrail.async_scan_request_attachments( + data=_pdf_chat_request(), call_type=CallTypes.acompletion.value + ) + image_result = await guardrail.async_scan_request_attachments( + data=_image_only_chat_request(), call_type=CallTypes.acompletion.value + ) + + assert pdf_result is None + assert image_result is None + assert mock_post.await_count == 0 + + + + @pytest.mark.asyncio async def test_bedrock_guardrails_streaming_request_body_mock(aws_test_credentials): """Test that the exact request body sent to Bedrock matches expected format when using streaming""" diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_headroom.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_headroom.py index 6f4b1816150..5bdcdddf4b7 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_headroom.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_headroom.py @@ -2008,6 +2008,45 @@ async def test_apply_guardrail_sends_textless_parts_rows_unflattened( assert wire_messages[1]["content"] == "D" * 5000 +@pytest.mark.asyncio +async def test_sdk_echoed_null_tool_calls_never_reach_compression_service(guardrail: HeadroomGuardrail): + """LIT-6921: OpenAI SDK assistant messages echoed back as history carry + ``"tool_calls": null``, which headroom-ai 0.27-0.30 answer with a 503. The + chat handler sends no row carrying that null and forwards the echoed rows + unchanged.""" + from openai.types.chat import ChatCompletionMessage + + from litellm.llms.openai.chat.guardrail_translation.handler import OpenAIChatCompletionsHandler + + def _sdk_assistant(**fields: object) -> dict: + return ChatCompletionMessage.model_validate({"role": "assistant", **fields}).model_dump() + + tool_call = {"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}} + history = [ + {"role": "user", "content": "E" * 5000}, + _sdk_assistant(content=None, tool_calls=[tool_call]), + {"role": "tool", "tool_call_id": "call_1", "content": "R" * 5000}, + _sdk_assistant(content="summary"), + {"role": "user", "content": "F" * 5000}, + _sdk_assistant(content="noted"), + {"role": "user", "content": "and now?"}, + ] + assert history[3]["tool_calls"] is None + sent: dict = {} + + def _echo(**kwargs): + sent["messages"] = kwargs["json"]["messages"] + return _make_compress_response(json.loads(json.dumps(sent["messages"]))) + + data = {"model": "claude-sonnet-5", "messages": json.loads(json.dumps(history))} + with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock, side_effect=_echo): + result = await OpenAIChatCompletionsHandler().process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert [row["role"] for row in sent["messages"]] == ["user", "tool", "user"] + assert all(row.get("tool_calls", []) is not None for row in sent["messages"]) + assert [result["messages"][index] for index in (1, 3, 5)] == [history[index] for index in (1, 3, 5)] + + @pytest.mark.asyncio async def test_fail_open_returns_original_parts_shapes(): guardrail = _make_guardrail(unreachable_fallback="fail_open") diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py b/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py index e797304a1c6..3efbe6475b2 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py @@ -49,6 +49,7 @@ from litellm.types.guardrails import GuardrailEventHooks from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import ( CallTypes, + CallTypesLiteral, Delta, GenericGuardrailAPIInputs, ModelResponse, @@ -3011,3 +3012,44 @@ class TestTranslationMappingsAreReadLive: assert not [ name for name, value in vars(unified_module).items() if isinstance(value, dict) and CallTypes.aocr in value ] + + +class AttachmentScanningGuardrail(RecordingGuardrail): + """Records the raw request it is handed for an attachment scan.""" + + def __init__(self) -> None: + super().__init__() + self.attachment_scans: list[dict[str, object]] = [] + + async def async_scan_request_attachments(self, data: dict, call_type: CallTypesLiteral) -> None: + self.attachment_scans.append({"call_type": call_type, "apply_calls_so_far": len(self.apply_calls)}) + + +@pytest.mark.asyncio +async def test_pre_call_hook_scans_attachments_before_text_extraction(): + guardrail = AttachmentScanningGuardrail() + + await UnifiedLLMGuardrails().async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + cache=DualCache(), + data={"guardrail_to_apply": guardrail, "model": "gpt-4o", "input": "hello"}, + call_type=CallTypes.aresponses.value, + ) + + assert guardrail.attachment_scans == [{"call_type": CallTypes.aresponses.value, "apply_calls_so_far": 0}] + assert len(guardrail.apply_calls) == 1 + + +@pytest.mark.asyncio +async def test_pre_call_hook_skips_attachment_scan_when_guardrail_has_none(): + guardrail = RecordingGuardrail() + guardrail.async_scan_request_attachments = None # instance attribute, not a method on the class + + await UnifiedLLMGuardrails().async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + cache=DualCache(), + data={"guardrail_to_apply": guardrail, "model": "gpt-4o", "input": "hello"}, + call_type=CallTypes.aresponses.value, + ) + + assert len(guardrail.apply_calls) == 1 diff --git a/tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py b/tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py index c206a91bd81..dbbc31279c3 100644 --- a/tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py +++ b/tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py @@ -45,8 +45,8 @@ def test_get_blog_posts_returns_response_shape(client): assert post["url"] == "https://www.litellm.ai/blog/test" -def test_get_blog_posts_limits_to_five(client): - """Endpoint returns at most 5 posts.""" +def test_get_blog_posts_returns_every_fetched_post(client): + """The parser owns the cap (BLOG_POSTS_MAX), so the endpoint passes every fetched post through.""" many_posts = [ { "title": f"Post {i}", @@ -64,7 +64,7 @@ def test_get_blog_posts_limits_to_five(client): response = client.get("/public/litellm_blog_posts") assert response.status_code == 200 - assert len(response.json()["posts"]) == 5 + assert [p["title"] for p in response.json()["posts"]] == [f"Post {i}" for i in range(10)] def test_get_blog_posts_returns_local_backup_on_failure(client): diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 0f66674daf9..3eb542f223e 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -3890,14 +3890,19 @@ class TestStreamingOverheadHeader: class TestDDSpanTaggerTagRequest: """Tests for DDSpanTagger.tag_request - key/model DD span tagging.""" - def _make_user_api_key_dict(self, key_alias=None, token=None): + def _make_user_api_key_dict(self, key_alias=None, token=None, user_email=None): from litellm.proxy._types import UserAPIKeyAuth d = UserAPIKeyAuth() d.key_alias = key_alias d.token = token + d.user_email = user_email return d + def _tracer_with_active_span_writing_to(self, tags: dict[str, str]) -> SimpleNamespace: + span = SimpleNamespace(set_tag_str=tags.__setitem__) + return SimpleNamespace(current_span=lambda: span) + def test_tags_key_alias_and_model(self): """key_alias and requested_model are set on the span when present.""" user_key = self._make_user_api_key_dict(key_alias="my-prod-key", token="hashed123") @@ -3936,6 +3941,32 @@ class TestDDSpanTaggerTagRequest: mock_set_tag.assert_called_once_with("litellm.requested_model", "claude-3-5-sonnet") + def test_tags_user_email(self): + """user_email is tagged so JWT-authenticated requests are traceable per person.""" + user_key = self._make_user_api_key_dict(user_email="user@example.com") + tags: dict[str, str] = {} + + with patch("litellm.litellm_core_utils.dd_tracing.tracer", self._tracer_with_active_span_writing_to(tags)): + DDSpanTagger.tag_request( + user_api_key_dict=user_key, + requested_model=None, + ) + + assert tags == {"litellm.user_email": "user@example.com"} + + def test_no_user_email_tag_when_absent(self): + """No user email tag when the authenticated identity has no email.""" + user_key = self._make_user_api_key_dict(key_alias="my-prod-key", user_email=None) + tags: dict[str, str] = {} + + with patch("litellm.litellm_core_utils.dd_tracing.tracer", self._tracer_with_active_span_writing_to(tags)): + DDSpanTagger.tag_request( + user_api_key_dict=user_key, + requested_model="gpt-4o", + ) + + assert tags == {"litellm.key_alias": "my-prod-key", "litellm.requested_model": "gpt-4o"} + class TestHasAttributeErrorInChain: """Tests for _has_attribute_error_in_chain helper.""" diff --git a/tests/unit/test_get_blog_posts.py b/tests/unit/test_get_blog_posts.py index 241dce23633..5686f45efd3 100644 --- a/tests/unit/test_get_blog_posts.py +++ b/tests/unit/test_get_blog_posts.py @@ -2,17 +2,18 @@ import time from unittest.mock import MagicMock, patch +from xml.etree import ElementTree import pytest import litellm from litellm.litellm_core_utils.get_blog_posts import ( + BLOG_POSTS_MAX, BlogPost, BlogPostsResponse, GetBlogPosts, get_blog_posts, ) -from xml.etree import ElementTree SAMPLE_RSS = """\ @@ -71,6 +72,18 @@ def test_parse_rss_to_posts_multiple(): assert posts[1]["title"] == "Second Post" +def test_parse_rss_to_posts_default_cap(): + items = "".join( + f"Post {i}https://docs.litellm.ai/blog/{i}" + f"DThu, 0{1 + i % 9} Jan 2026 00:00:00 GMT" + for i in range(BLOG_POSTS_MAX + 1) + ) + posts = GetBlogPosts.parse_rss_to_posts(f"{items}") + assert len(posts) == BLOG_POSTS_MAX + assert posts[0]["title"] == "Post 0" + assert posts[-1]["title"] == f"Post {BLOG_POSTS_MAX - 1}" + + def test_parse_rss_to_posts_invalid_xml(): with pytest.raises(ElementTree.ParseError): GetBlogPosts.parse_rss_to_posts("not xml") @@ -82,9 +95,7 @@ def test_parse_rss_to_posts_missing_channel(): def test_validate_blog_posts_valid(): - posts = [ - {"title": "T", "description": "D", "date": "2026-01-01", "url": "https://x.com"} - ] + posts = [{"title": "T", "description": "D", "date": "2026-01-01", "url": "https://x.com"}] assert GetBlogPosts.validate_blog_posts(posts) is True @@ -108,7 +119,7 @@ def test_get_blog_posts_success(): ): posts = get_blog_posts(url=litellm.blog_posts_url) - assert len(posts) == 1 + assert len(posts) == 2 assert posts[0]["title"] == "Test Post" @@ -163,9 +174,7 @@ def test_get_blog_posts_ttl_cache_not_refetched(): m.raise_for_status = MagicMock() return m - with patch( - "litellm.litellm_core_utils.get_blog_posts.httpx.get", side_effect=mock_get - ): + with patch("litellm.litellm_core_utils.get_blog_posts.httpx.get", side_effect=mock_get): posts = get_blog_posts(url=litellm.blog_posts_url) assert call_count == 0 # cache hit, no fetch @@ -196,7 +205,7 @@ def test_get_blog_posts_ttl_expired_refetches(): posts = get_blog_posts(url=litellm.blog_posts_url) mock_get.assert_called_once() - assert len(posts) == 1 + assert len(posts) == 2 def test_get_blog_posts_local_env_var_skips_remote(monkeypatch): @@ -219,9 +228,5 @@ def test_blog_post_pydantic_model(): def test_blog_posts_response_pydantic_model(): - resp = BlogPostsResponse( - posts=[ - BlogPost(title="T", description="D", date="2026-01-01", url="https://x.com") - ] - ) + resp = BlogPostsResponse(posts=[BlogPost(title="T", description="D", date="2026-01-01", url="https://x.com")]) assert len(resp.posts) == 1 diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 3f5450ea36b..8c9d1b5642d 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -2318,6 +2318,31 @@ def test_model_group_info_cost_none_for_unpriced_deployment_but_zero_when_declar assert priced.output_cost_per_token is not None and priced.output_cost_per_token > 0 +def test_model_group_info_description_from_model_info(): + """ + model_info.description set in the config should surface on ModelGroupInfo, + including when only a later deployment in the group carries it. + """ + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": {"model": "gpt-4", "api_key": "fake"}, + "model_info": {}, + }, + { + "model_name": "gpt-4", + "litellm_params": {"model": "gpt-4", "api_key": "fake2"}, + "model_info": {"description": "State-of-the-art language model."}, + }, + ] + ) + + result = router.get_model_group_info("gpt-4") + assert result is not None + assert result.description == "State-of-the-art language model." + + def _alias_cost_router() -> Router: return Router( model_list=[ diff --git a/ui/litellm-dashboard/src/app/(dashboard)/home/_components/HomePage.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/home/_components/HomePage.integration.test.tsx new file mode 100644 index 00000000000..ad3f80b7d78 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/home/_components/HomePage.integration.test.tsx @@ -0,0 +1,149 @@ +import { render, screen, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { DailyActivityAggregatedResponse } from "@/components/UsagePage/dailyActivityApi"; +import { EMPTY_DAILY_ACTIVITY_METADATA } from "@/components/UsagePage/dailyActivityApi"; +import { all_admin_roles } from "@/utils/roles"; +import HomePage from "./HomePage"; +import { WHATS_NEW_ITEMS } from "./homeContent"; + +const auth = vi.hoisted(() => ({ + accessToken: "sk-test", + userRole: "", + userId: "u1", + isViewOnly: false, +})); +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ default: () => auth })); + +const network = vi.hoisted(() => ({ + aggregated: vi.fn(), + gateway: vi.fn(), + userInfo: vi.fn(), +})); +vi.mock("@/components/networking", async (importOriginal) => ({ + ...(await importOriginal()), + dailyActivityAggregatedCall: (...args: unknown[]) => network.aggregated(...args), + gatewayDailyActivityCall: (...args: unknown[]) => network.gateway(...args), + userGetInfoV2: (...args: unknown[]) => network.userInfo(...args), + getProxyBaseUrl: () => "http://proxy.test", +})); + +const blogPosts = { posts: [{ title: "Post one", description: "d", date: "2026-10-09", url: "https://x/1" }] }; + +const dayResult = (date: string, model: string, tokens: number) => { + const metrics = { + spend: 1, + prompt_tokens: tokens, + completion_tokens: 0, + total_tokens: tokens, + api_requests: 1, + successful_requests: 1, + failed_requests: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }; + return { + date, + metrics, + breakdown: { + models: {}, + model_groups: { [model]: { metrics, metadata: {}, api_key_breakdown: {} } }, + providers: {}, + api_keys: {}, + entities: {}, + mcp_servers: {}, + }, + }; +}; + +const activity: DailyActivityAggregatedResponse = { + results: [ + dayResult("2026-10-08", "gpt-6.1-sol", 900), + dayResult("2026-10-09", "claude-haiku-5-5", 100), + ] as unknown as DailyActivityAggregatedResponse["results"], + metadata: { + ...EMPTY_DAILY_ACTIVITY_METADATA, + total_spend: 501.74, + total_api_requests: 5118, + total_successful_requests: 5075, + total_failed_requests: 43, + total_tokens: 98_600_000, + }, +}; + +const renderHome = () => + render( + + + , + ); + +describe("HomePage", () => { + beforeEach(() => { + auth.isViewOnly = false; + auth.userRole = all_admin_roles[0]!; + network.aggregated.mockReset().mockResolvedValue(activity); + network.gateway.mockReset().mockResolvedValue({ total_successful_requests: 0, total_failed_requests: 0 }); + network.userInfo.mockReset().mockResolvedValue({ user_info: { max_budget: null } }); + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response(JSON.stringify(blogPosts), { status: 200 }))); + }); + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("shows the usage totals from the daily activity API and links Create New Virtual Key to the key creation flow", async () => { + renderHome(); + expect(screen.getByRole("heading", { level: 1, name: "Home" })).toBeInTheDocument(); + expect(await screen.findByText("$501.74")).toBeInTheDocument(); + expect(screen.getByText("5,118")).toBeInTheDocument(); + expect(screen.getByText("98.6M")).toBeInTheDocument(); + expect(screen.getByRole("link", { name: /create new virtual key/i })).toHaveAttribute( + "href", + expect.stringMatching(/\/api-keys\?create=true$/), + ); + expect(network.aggregated).toHaveBeenCalledWith("user", expect.objectContaining({ accessToken: "sk-test" })); + }); + + it("hides Create New Virtual Key from view-only users", () => { + auth.isViewOnly = true; + renderHome(); + expect(screen.queryByRole("link", { name: /create new virtual key/i })).not.toBeInTheDocument(); + }); + + it("shows an error with a retry instead of zero totals when the usage request fails", async () => { + network.aggregated.mockRejectedValueOnce(new Error("boom")); + renderHome(); + expect(await screen.findByText("Failed to load usage")).toBeInTheDocument(); + expect(screen.queryByText("$0.00")).not.toBeInTheDocument(); + await userEvent.click(screen.getByRole("button", { name: "Retry" })); + expect(await screen.findByText("$501.74")).toBeInTheDocument(); + expect(network.aggregated).toHaveBeenCalledTimes(2); + }); + + it("lists the curated launches newest first with their links", () => { + renderHome(); + const section = screen.getByTestId("home-whats-new"); + const links = within(section).getAllByRole("link"); + expect(links.map((link) => link.getAttribute("href"))).toEqual(WHATS_NEW_ITEMS.map((item) => item.href)); + const dates = WHATS_NEW_ITEMS.map((item) => item.publishedOn); + expect(dates).toEqual([...dates].sort().reverse()); + }); + + it("ranks model groups by token share on the leaderboard", async () => { + renderHome(); + const board = screen.getByTestId("home-leaderboard"); + expect(await within(board).findByText("gpt-6.1-sol")).toBeInTheDocument(); + const rows = within(board).getAllByRole("listitem"); + expect(within(rows[0]!).getByText("gpt-6.1-sol")).toBeInTheDocument(); + expect(within(rows[0]!).getByText("90.0%")).toBeInTheDocument(); + expect(within(rows[1]!).getByText("claude-haiku-5-5")).toBeInTheDocument(); + }); + + it("renders the blog posts fetched from /public/litellm_blog_posts in the Updates feed", async () => { + renderHome(); + const updates = screen.getByTestId("home-updates"); + expect(await within(updates).findByRole("link", { name: /post one/i })).toHaveAttribute("href", "https://x/1"); + expect(fetch).toHaveBeenCalledWith("http://proxy.test/public/litellm_blog_posts"); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/home/_components/HomePage.tsx b/ui/litellm-dashboard/src/app/(dashboard)/home/_components/HomePage.tsx new file mode 100644 index 00000000000..f4519958e9d --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/home/_components/HomePage.tsx @@ -0,0 +1,108 @@ +"use client"; + +import { ArrowRight, Plus, Sparkles } from "lucide-react"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { Button } from "@/components/ui/button"; +import { Card } from "@/components/ui/card"; +import { Page } from "@/components/shared/Page"; +import { PageHeader, PageHeaderControls, PageHeaderTitle } from "@/components/shared/PageHeader"; +import { uiHref } from "@/utils/uiHref"; +import { Leaderboard, useBreakdown } from "@/app/(dashboard)/usage/_components/components/overview/BreakdownChart"; +import { Panel } from "@/app/(dashboard)/usage/_components/components/overview/Primitives"; +import UsageStatStrip from "@/app/(dashboard)/usage/_components/components/overview/UsageStatStrip"; +import { formatPublishedOn, WHATS_NEW_ITEMS } from "./homeContent"; +import UpdatesFeed from "./UpdatesFeed"; +import { HOME_USAGE_DAYS, useHomeUsage } from "./useHomeUsage"; + +const LEADERBOARD = { metric: "tokens", dimension: "model_groups" } as const; + +export default function HomePage() { + const { isViewOnly } = useAuthorized(); + const usage = useHomeUsage(); + const { series, ranking } = useBreakdown(usage.results, LEADERBOARD, 8); + + return ( + + + Home + {!isViewOnly && ( + + + + )} + + +
+ {usage.failed ? ( +
+ Failed to load usage + +
+ ) : ( + + )} +
+ +
+
+
+
+

+ + What's new in LiteLLM? +

+
+
+ {WHATS_NEW_ITEMS.map((item) => { + const Icon = item.icon; + return ( + + +
+ +
+
{item.title}
+

{item.description}

+
+ {formatPublishedOn(item.publishedOn)} + +
+ + + ); + })} +
+
+ + + + +
+ + +
+ + ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/home/_components/UpdatesFeed.tsx b/ui/litellm-dashboard/src/app/(dashboard)/home/_components/UpdatesFeed.tsx new file mode 100644 index 00000000000..ddb135d6f4d --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/home/_components/UpdatesFeed.tsx @@ -0,0 +1,93 @@ +"use client"; + +import { ArrowUpRight, Newspaper } from "lucide-react"; +import { useBlogPosts, type BlogPost } from "@/app/(dashboard)/hooks/blogPosts/useBlogPosts"; +import { useDisableBlogPosts } from "@/app/(dashboard)/hooks/useDisableBlogPosts"; +import { Button } from "@/components/ui/button"; +import { Skeleton } from "@/components/ui/skeleton"; + +export const BLOG_URL = "https://docs.litellm.ai/blog"; +const MAX_ROWS = 6; +const DAY_MS = 86_400_000; + +const relativeTime = new Intl.RelativeTimeFormat("en", { numeric: "auto" }); + +export const formatRelativeDate = (isoDate: string, now: Date = new Date()): string => { + const published = new Date(`${isoDate}T00:00:00`); + if (Number.isNaN(published.getTime())) return isoDate; + const days = Math.max(0, Math.round((now.getTime() - published.getTime()) / DAY_MS)); + if (days < 7) return relativeTime.format(-days, "day"); + if (days < 30) return relativeTime.format(-Math.round(days / 7), "week"); + if (days < 365) return relativeTime.format(-Math.round(days / 30), "month"); + return relativeTime.format(-Math.round(days / 365), "year"); +}; + +const Row = ({ children }: { children: React.ReactNode }) => ( +
  • + + + +
    {children}
    +
    • +); + +/** Latest posts from the LiteLLM blog RSS feed, via /public/litellm_blog_posts, one row per post. */ +export default function UpdatesFeed({ className }: { className?: string }) { + const disabled = useDisableBlogPosts(); + const { data, isLoading, isError, refetch } = useBlogPosts(); + if (disabled) return null; + + const posts = data?.posts.slice(0, MAX_ROWS) ?? []; + + return ( +
    +
    +

    + Updates +

    + +
    + {isLoading && ( +
      + {[0, 1, 2, 3].map((i) => ( + + + + + + ))} +
    + )} + {isError && ( +
    + Failed to load posts + +
    + )} + {data && posts.length === 0 &&

    No posts yet

    } + {posts.length > 0 && ( +
      + {posts.map((post: BlogPost) => ( + + {formatRelativeDate(post.date)} + + {post.title} + + {post.description} + + ))} +
    + )} +
    + ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/home/_components/homeContent.ts b/ui/litellm-dashboard/src/app/(dashboard)/home/_components/homeContent.ts new file mode 100644 index 00000000000..41d1aac25de --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/home/_components/homeContent.ts @@ -0,0 +1,40 @@ +import { Box, Scale, Zap, type LucideIcon } from "lucide-react"; + +export interface Launch { + readonly icon: LucideIcon; + readonly title: string; + readonly description: string; + readonly href: string; + readonly publishedOn: string; +} + +const LAUNCHES: readonly Launch[] = [ + { + icon: Box, + title: "Claude Haiku 5.5", + description: "Anthropic's newest small model, with day 0 pricing on Anthropic, Bedrock and Vertex AI.", + href: "https://docs.litellm.ai/blog/claude-haiku-5-5", + publishedOn: "2026-10-07", + }, + { + icon: Scale, + title: "Decision Models", + description: "Call decision models at /v1/decisions and try them in the Decisions playground.", + href: "https://docs.litellm.ai/docs/decisions", + publishedOn: "2026-10-07", + }, + { + icon: Zap, + title: "OpenAI Ultrafast", + description: "service_tier: ultrafast on GPT-6 Astra and GPT-6.1 Sol, including /ultrafast in Codex.", + href: "https://docs.litellm.ai/docs/providers/openai/ultrafast", + publishedOn: "2026-10-06", + }, +]; + +export const WHATS_NEW_ITEMS: readonly Launch[] = [...LAUNCHES].sort((a, b) => + b.publishedOn.localeCompare(a.publishedOn), +); + +export const formatPublishedOn = (isoDate: string): string => + new Date(`${isoDate}T00:00:00`).toLocaleDateString("en-US", { month: "short", day: "numeric" }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/home/_components/useHomeUsage.ts b/ui/litellm-dashboard/src/app/(dashboard)/home/_components/useHomeUsage.ts new file mode 100644 index 00000000000..fe6931bb6f5 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/home/_components/useHomeUsage.ts @@ -0,0 +1,86 @@ +import { useEffect, useMemo, useState } from "react"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { useCurrentUser } from "@/app/(dashboard)/hooks/users/useCurrentUser"; +import { gatewayDailyActivityCall } from "@/components/networking"; +import { toDailyData } from "@/components/UsagePage/dailyActivityApi"; +import type { DailyData } from "@/components/UsagePage/types"; +import { all_admin_roles } from "@/utils/roles"; +import { useAggregatedDailyActivity } from "@/app/(dashboard)/usage/_components/hooks/useAggregatedDailyActivity"; +import { ENTITY_API } from "@/app/(dashboard)/usage/_components/components/EntityUsage/entityFetchFns"; +import type { GatewayActivity } from "@/app/(dashboard)/usage/_components/components/gatewayActivity"; +import { EMPTY_DAILY_ACTIVITY_METADATA, type DailyActivityRequest } from "@/components/UsagePage/dailyActivityApi"; +import { + overviewTotals, + type OverviewTotals, +} from "@/app/(dashboard)/usage/_components/components/overview/overviewData"; + +export const HOME_USAGE_DAYS = 7; + +export interface HomeUsage { + results: readonly DailyData[]; + totals: OverviewTotals; + loading: boolean; + requestCountsPending: boolean; + failed: boolean; + budget: number | null; + retry: () => void; +} + +/** Last HOME_USAGE_DAYS of the signed-in user's view of usage: deployment-wide for admins, their own otherwise. */ +export function useHomeUsage(): HomeUsage { + const { accessToken, userRole, userId } = useAuthorized(); + const { data: currentUser } = useCurrentUser(); + const isAdmin = all_admin_roles.includes(userRole || ""); + const [attempt, setAttempt] = useState(0); + const range = useMemo(() => { + const endTime = new Date(); + const startTime = new Date(endTime.getTime() - HOME_USAGE_DAYS * 24 * 60 * 60 * 1000); + return { startTime, endTime }; + }, []); + + const request = useMemo( + () => + accessToken && (isAdmin || userId) + ? { accessToken, ...range, entityIds: isAdmin ? null : [userId as string] } + : null, + [accessToken, isAdmin, range, userId], + ); + const { data, loading, failed } = useAggregatedDailyActivity({ + fetch: () => ENTITY_API.user.aggregated(request as DailyActivityRequest), + enabled: request !== null, + deps: [accessToken, isAdmin, userId, range.startTime.toISOString(), attempt], + }); + + const [gateway, setGateway] = useState(null); + useEffect(() => { + if (!isAdmin || !accessToken) return; + let cancelled = false; + gatewayDailyActivityCall(accessToken, range.startTime, range.endTime) + .then((activity: GatewayActivity) => { + const recorded = activity.total_successful_requests + activity.total_failed_requests > 0; + if (!cancelled) setGateway(recorded ? activity : null); + }) + .catch(() => { + if (!cancelled) setGateway(null); + }); + return () => { + cancelled = true; + }; + }, [isAdmin, accessToken, range, attempt]); + + const results = useMemo(() => toDailyData(data), [data]); + const totals = useMemo( + () => overviewTotals(data.metadata ?? EMPTY_DAILY_ACTIVITY_METADATA, gateway), + [data.metadata, gateway], + ); + + return { + results, + totals, + loading, + requestCountsPending: loading && gateway === null, + failed, + budget: currentUser?.max_budget ?? null, + retry: () => setAttempt((n) => n + 1), + }; +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/home/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/home/page.tsx new file mode 100644 index 00000000000..06e3b000816 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/home/page.tsx @@ -0,0 +1,13 @@ +"use client"; + +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import LoadingScreen from "@/components/common_components/LoadingScreen"; +import HomePage from "./_components/HomePage"; + +export default function HomeRoutePage() { + const { isLoading, isAuthorized } = useAuthorized(); + if (isLoading || !isAuthorized) { + return ; + } + return ; +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.ts b/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.ts index 09cc4ef0a75..f4489625dfe 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.ts @@ -2,6 +2,7 @@ import { uiHref } from "@/utils/uiHref"; const LEGACY_PAGE_ROUTES: ReadonlyMap = new Map( Object.entries({ + home: "home", "api-keys": "api-keys", models: "models-and-endpoints", api_ref: "api-reference", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/page.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/page.test.tsx index aaad0d072e5..f5baff0e727 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/page.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/page.test.tsx @@ -37,8 +37,8 @@ vi.mock("@/contexts/AuthContext", () => ({ }), })); vi.mock("@/app/(dashboard)/hooks/keys/useKeys", () => ({ useKeys: mockUseKeys })); -vi.mock("@/app/(dashboard)/api-keys/ApiKeysDashboard", () => ({ - default: () =>
    , +vi.mock("@/app/(dashboard)/home/_components/HomePage", () => ({ + default: () =>
    , })); vi.mock("@/components/common_components/LoadingScreen", () => ({ default: () =>
    , @@ -86,7 +86,7 @@ describe("dashboard landing", () => { (role) => { state.userRole = role; render(); - expect(screen.getByTestId("api-keys-dashboard")).toBeInTheDocument(); + expect(screen.getByTestId("home-page")).toBeInTheDocument(); expect(screen.queryByTestId("loading-screen")).not.toBeInTheDocument(); expect(mockReplace).not.toHaveBeenCalled(); expect(mockUiHref).not.toHaveBeenCalledWith("connect"); @@ -96,7 +96,7 @@ describe("dashboard landing", () => { it("lands a user who already owns a key on the keys dashboard", () => { state.keys = [{ token: "sk-abc" }]; render(); - expect(screen.getByTestId("api-keys-dashboard")).toBeInTheDocument(); + expect(screen.getByTestId("home-page")).toBeInTheDocument(); expect(mockReplace).not.toHaveBeenCalled(); }); @@ -110,7 +110,7 @@ describe("dashboard landing", () => { render(); expect(mockReplace).toHaveBeenCalledWith("/mocked-ui/logs"); expect(screen.getByTestId("loading-screen")).toBeInTheDocument(); - expect(screen.queryByTestId("api-keys-dashboard")).not.toBeInTheDocument(); + expect(screen.queryByTestId("home-page")).not.toBeInTheDocument(); }); it("carries the MCP env-var deep link's other params through the legacy redirect", () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/page.tsx index 3a38958dd66..94f4ba79a25 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/page.tsx @@ -1,6 +1,6 @@ "use client"; -import ApiKeysDashboard from "@/app/(dashboard)/api-keys/ApiKeysDashboard"; +import HomePage from "@/app/(dashboard)/home/_components/HomePage"; import LoadingScreen from "@/components/common_components/LoadingScreen"; import { proxyBaseUrl } from "@/components/networking"; import { useAuth } from "@/contexts/AuthContext"; @@ -89,7 +89,7 @@ function CreateKeyPageContent() { return ; } - return ; + return ; } export default function CreateKeyPage() { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/UsageOverview.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/UsageOverview.tsx index 8b7c2d5ec45..afd6e5ba48b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/UsageOverview.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/UsageOverview.tsx @@ -6,7 +6,7 @@ * matching colors. */ -import React, { type ReactNode, useMemo, useState } from "react"; +import { type ReactNode, useMemo, useState } from "react"; import type { DailyData } from "@/components/UsagePage/types"; import { KeyRound, Users } from "lucide-react"; import { StackedUsageChart, type StackedUsageScale } from "@/components/shared/charts"; @@ -14,18 +14,15 @@ import { bucketSeries, bucketTotals, labelForDate, - dailyTotals, - formatCompact, - formatLatency, formatMetricValue, - formatUsd, type Granularity, type OverviewTotals, type UsageMetric, } from "./overviewData"; import { BreakdownControls, Leaderboard, useBreakdown, type BreakdownState } from "./BreakdownChart"; import { cn } from "@/lib/cva.config"; -import { ChartSkeleton, Panel, PANEL_INSET_X, Segmented, Sparkline, Stat } from "./Primitives"; +import { ChartSkeleton, Panel, PANEL_INSET_X, Segmented } from "./Primitives"; +import UsageStatStrip from "./UsageStatStrip"; interface UsageOverviewProps { results: readonly DailyData[]; @@ -41,7 +38,6 @@ interface UsageOverviewProps { } const METRIC_NOUN: Record = { spend: "spend", tokens: "tokens", requests: "requests" }; -const BRAND = "#2b3fd6"; const GRANULARITY_OPTIONS = [ { value: "day", label: "Daily" }, { value: "week", label: "Weekly" }, @@ -69,57 +65,17 @@ export default function UsageOverview({ const { series: dailySeries, ranking } = useBreakdown(results, state, 8); const series = useMemo(() => bucketSeries(dailySeries, granularity), [dailySeries, granularity]); const totalsByBucket = useMemo(() => bucketTotals(series), [series]); - const daily = useMemo(() => dailyTotals(results), [results]); const format = (value: number) => formatMetricValue(value, state.metric); return (
    -
    - }> - - - }> - - {!requestCountsPending && ( -
    - - {totals.successful.toLocaleString()} ok - - - 0 ? "text-destructive tabular-nums" : "tabular-nums"}> - {totals.failed.toLocaleString()} - {" "} - failed - -
    - )} -     - }> - - - -
    - - -
    -     -
    + ); } - -function StatCell({ children, trend }: { children: React.ReactNode; trend?: React.ReactNode }) { - return ( -
    -
    {children}
    - {trend} -
    - ); -} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/UsageStatStrip.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/UsageStatStrip.tsx new file mode 100644 index 00000000000..318b7758370 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/UsageStatStrip.tsx @@ -0,0 +1,90 @@ +"use client"; + +import React, { useMemo } from "react"; +import type { DailyData } from "@/components/UsagePage/types"; +import { cn } from "@/lib/cva.config"; +import { dailyTotals, formatCompact, formatLatency, formatUsd, type OverviewTotals } from "./overviewData"; +import { PANEL_INSET_X, Sparkline, Stat } from "./Primitives"; + +const BRAND = "#2b3fd6"; + +interface UsageStatStripProps { + results: readonly DailyData[]; + totals: OverviewTotals; + loading: boolean; + requestCountsPending: boolean; + budget: number | null; +} + +/** Spend, requests, tokens and averages for a range, each with a daily sparkline. */ +export default function UsageStatStrip({ + results, + totals, + loading, + requestCountsPending, + budget, +}: UsageStatStripProps) { + const daily = useMemo(() => dailyTotals(results), [results]); + return ( +
    + }> + + + }> + + {!requestCountsPending && ( +
    + + {totals.successful.toLocaleString()} ok + + + 0 ? "text-destructive tabular-nums" : "tabular-nums"}> + {totals.failed.toLocaleString()} + {" "} + failed + +
    + )} +     + }> + + + +
    + + +
    +     +
    + ); +} + +function StatCell({ children, trend }: { children: React.ReactNode; trend?: React.ReactNode }) { + return ( +
    +
    {children}
    + {trend} +
    + ); +} diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx index e5dcb8ebc73..41b57a78eef 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx @@ -684,6 +684,12 @@ const ModelHubTable: React.FC = ({ ))}
    + {selectedModel.description && ( +
    +

    Description:

    +

    {selectedModel.description}

    +
    + )}
    diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx index fc71a9340ac..3d6423f2d44 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx @@ -31,6 +31,7 @@ export interface ModelHubData { supports_function_calling: boolean; supported_openai_params?: string[]; is_public_model_group: boolean; + description?: string; [key: string]: any; } diff --git a/ui/litellm-dashboard/src/components/CommandPalette/utils.test.ts b/ui/litellm-dashboard/src/components/CommandPalette/utils.test.ts index 5bb61756f80..c05e4db8924 100644 --- a/ui/litellm-dashboard/src/components/CommandPalette/utils.test.ts +++ b/ui/litellm-dashboard/src/components/CommandPalette/utils.test.ts @@ -5,7 +5,7 @@ import { flattenNavItems, matchNavItems, paletteScopeForRoute, shortcutLabel, ty describe("paletteScopeForRoute", () => { it.each([ ["api-keys", "keys"], - ["", "keys"], + ["", "global"], ["teams", "global"], ] as const)("uses the expected palette scope for %s", (routeSegment, expected) => { expect(paletteScopeForRoute(routeSegment)).toBe(expected); diff --git a/ui/litellm-dashboard/src/components/CommandPalette/utils.ts b/ui/litellm-dashboard/src/components/CommandPalette/utils.ts index ad02e155b0e..396b92a817e 100644 --- a/ui/litellm-dashboard/src/components/CommandPalette/utils.ts +++ b/ui/litellm-dashboard/src/components/CommandPalette/utils.ts @@ -12,7 +12,7 @@ export interface PaletteNavItem { } export const paletteScopeForRoute = (routeSegment: string): PaletteScope => - routeSegment === "" || routeSegment === "api-keys" ? "keys" : "global"; + routeSegment === "api-keys" ? "keys" : "global"; const matchRank = (item: PaletteNavItem, query: string): number => { const label = item.label.toLowerCase(); diff --git a/ui/litellm-dashboard/src/components/DashboardHeader.test.tsx b/ui/litellm-dashboard/src/components/DashboardHeader.test.tsx index 822cec9deb3..6c64d09009c 100644 --- a/ui/litellm-dashboard/src/components/DashboardHeader.test.tsx +++ b/ui/litellm-dashboard/src/components/DashboardHeader.test.tsx @@ -59,11 +59,11 @@ describe("DashboardHeader breadcrumb", () => { expect(screen.getByText("Models + Endpoints")).toBeInTheDocument(); }); - it("titles the dashboard root as Virtual Keys", () => { + it("titles the dashboard root as Home", () => { state.pathname = "/ui/"; renderDashboardHeader(); - expect(screen.getByText("Virtual Keys")).toBeInTheDocument(); + expect(screen.getByText("Home")).toBeInTheDocument(); }); it("roots the breadcrumb in the AI Gateway selector (with a Chat option) and drops the static section crumb when the selector is available", async () => { diff --git a/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx b/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx index de987360791..2fae8044d6b 100644 --- a/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx @@ -25,6 +25,7 @@ export interface ModelGroupInfo { health_status?: string; health_response_time?: number; health_checked_at?: string; + description?: string; [key: string]: any; } diff --git a/ui/litellm-dashboard/src/components/leftnav.test.tsx b/ui/litellm-dashboard/src/components/leftnav.test.tsx index cdb74c73a80..7ef951e1041 100644 --- a/ui/litellm-dashboard/src/components/leftnav.test.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.test.tsx @@ -247,6 +247,7 @@ describe("Sidebar (leftnav)", () => { renderWithProviders(); const topLevelLabels = [ + "Home", "Virtual Keys", "Playground", "Models + Endpoints", @@ -660,10 +661,10 @@ describe("Sidebar (leftnav)", () => { expect(screen.getByRole("link", { name: "Virtual Keys" })).not.toHaveAttribute("data-active"); }); - it("marks Virtual Keys active at the dashboard root", () => { + it("marks Home active at the dashboard root", () => { navState.pathname = "/ui/"; renderWithProviders(); - expect(screen.getByRole("link", { name: "Virtual Keys" })).toHaveAttribute("data-active", "true"); + expect(screen.getByRole("link", { name: "Home" })).toHaveAttribute("data-active", "true"); }); it("expands the parent group of the current nested route and marks the child active", () => { @@ -745,8 +746,8 @@ describe("getBreadcrumb", () => { expect(getBreadcrumb("/ui/old-usage")).toEqual({ section: "Developer Tools", title: "Old Usage" }); }); - it("titles the dashboard root as Virtual Keys", () => { - expect(getBreadcrumb("/ui/")).toEqual({ section: "AI Gateway", title: "Virtual Keys" }); + it("titles the dashboard root as Home", () => { + expect(getBreadcrumb("/ui/")).toEqual({ section: "AI Gateway", title: "Home" }); }); it("resolves a nested child route to its parent section", () => { diff --git a/ui/litellm-dashboard/src/components/leftnav.tsx b/ui/litellm-dashboard/src/components/leftnav.tsx index 256fe578912..9a6ec334b56 100644 --- a/ui/litellm-dashboard/src/components/leftnav.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.tsx @@ -40,6 +40,7 @@ import { FlaskConical, Folder, HeartPulse, + House, KeyRound, LayoutGrid, Network, @@ -207,6 +208,7 @@ const menuGroups: MenuGroup[] = [ { groupLabel: "AI GATEWAY", items: [ + { key: "home", page: "home", label: "Home", icon: }, { key: "api-keys", page: "api-keys", label: "Virtual Keys", icon: }, { key: "llm-playground", @@ -468,7 +470,7 @@ const menuGroups: MenuGroup[] = [ }, ]; -const HOME_ROUTE = "api-keys"; +const HOME_ROUTE = "home"; export const routeOf = (item: MenuItem): string => item.route ?? item.page; diff --git a/ui/litellm-dashboard/src/components/page_metadata.ts b/ui/litellm-dashboard/src/components/page_metadata.ts index 8eb14fc2151..72bed431421 100644 --- a/ui/litellm-dashboard/src/components/page_metadata.ts +++ b/ui/litellm-dashboard/src/components/page_metadata.ts @@ -5,6 +5,7 @@ // Page descriptions for UI Settings configuration export const pageDescriptions: Record = { + home: "Usage at a glance, recent launches and blog updates", "api-keys": "Manage virtual keys for API access and authentication", "llm-playground": "Interactive playground for testing LLM requests", models: "Configure and manage LLM models and endpoints", diff --git a/ui/litellm-dashboard/src/components/public_model_hub.tsx b/ui/litellm-dashboard/src/components/public_model_hub.tsx index 6102ed9556a..20daee4d006 100644 --- a/ui/litellm-dashboard/src/components/public_model_hub.tsx +++ b/ui/litellm-dashboard/src/components/public_model_hub.tsx @@ -714,6 +714,12 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded })} + {selectedModel.description && ( +
    +

    Description:

    +

    {selectedModel.description}

    +
    + )} {/* Wildcard Routing Note */} diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 3f786a38765..9853a71a2ed 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -26874,7 +26874,7 @@ export interface components { skip_tool_message_in_guardrail?: boolean | null; /** * Skip Unscannable Attachments - * @description Implemented by guardrail='model_armor'. When True, attachment references that carry no inline bytes (file_id, gs://, or http(s) URLs) pass through unscanned instead of blocking, while fail_on_error still governs real Model Armor API errors. Default False blocks them. + * @description Implemented by guardrail='model_armor' and guardrail='bedrock'. When True, attachments the guardrail cannot scan pass through unscanned instead of blocking. For Model Armor these are references with no inline bytes (file_id, gs://, or http(s) URLs), and fail_on_error still governs real Model Armor API errors. For Bedrock these are documents, files, audio, video, and images that are not inline PNG or JPEG up to 4 MB. Default False blocks them. * @default false */ skip_unscannable_attachments: boolean | null; @@ -36518,7 +36518,7 @@ export interface components { skip_tool_message_in_guardrail?: boolean | null; /** * Skip Unscannable Attachments - * @description Implemented by guardrail='model_armor'. When True, attachment references that carry no inline bytes (file_id, gs://, or http(s) URLs) pass through unscanned instead of blocking, while fail_on_error still governs real Model Armor API errors. Default False blocks them. + * @description Implemented by guardrail='model_armor' and guardrail='bedrock'. When True, attachments the guardrail cannot scan pass through unscanned instead of blocking. For Model Armor these are references with no inline bytes (file_id, gs://, or http(s) URLs), and fail_on_error still governs real Model Armor API errors. For Bedrock these are documents, files, audio, video, and images that are not inline PNG or JPEG up to 4 MB. Default False blocks them. * @default false */ skip_unscannable_attachments: boolean | null; @@ -37726,6 +37726,8 @@ export interface components { ModelGroupInfoProxy: { /** Configurable Clientside Auth Params */ configurable_clientside_auth_params?: (string | components["schemas"]["ConfigurableClientsideParamsCustomAuth-Output"])[] | null; + /** Description */ + description?: string | null; /** Health Checked At */ health_checked_at?: string | null; /** Health Response Time */ diff --git a/ui/litellm-dashboard/tests/CreateKeyPage.expiredToken.test.tsx b/ui/litellm-dashboard/tests/CreateKeyPage.expiredToken.test.tsx index 2c57833102d..1a31d85eed5 100644 --- a/ui/litellm-dashboard/tests/CreateKeyPage.expiredToken.test.tsx +++ b/ui/litellm-dashboard/tests/CreateKeyPage.expiredToken.test.tsx @@ -105,7 +105,7 @@ vi.mock("@/utils/returnUrlUtils", async (importOriginal) => { // Super-light stubs for all heavy components so rendering doesn't explode vi.mock("@/components/navbar", () => ({ default: stub("navbar") })); -vi.mock("@/app/(dashboard)/api-keys/ApiKeysDashboard", () => ({ default: stub("api-keys-dashboard") })); +vi.mock("@/app/(dashboard)/home/_components/HomePage", () => ({ default: stub("home-page") })); vi.mock("@/components/templates/model_dashboard", () => ({ default: stub("model-dashboard") })); vi.mock("@/components/teams", () => ({ default: stub("teams") })); vi.mock("@/app/(dashboard)/organizations/_components/organizations", () => ({ @@ -271,7 +271,7 @@ describe("CreateKeyPage auth behavior", () => { // And the default page content appears (ApiKeysDashboard stub; chrome now lives in the layout) await waitFor(() => { - expect(screen.getByTestId("api-keys-dashboard")).toBeInTheDocument(); + expect(screen.getByTestId("home-page")).toBeInTheDocument(); }); });