scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-11 03:38:38 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 82

refactor(rust): share provider constants via llms-types and restructure AGENTS.md (#45850)

Browse source 
* wip

* refactor(rust): share Bedrock paths and Converse/Invoke wire types via llms-types

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* refactor(rust): share Vertex AI locations, base URL and rawPredict paths via llms-types

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* refactor(rust): give every llms-types format and provider its own folder

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* refactor(rust): keep single-adapter projections in llms

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* docs(rust): restructure llms and llms-types AGENTS.md into rules and references

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* refactor(rust): keep provider mod.rs files as re-export entrypoints

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* docs(rust): split llms and llms-types AGENTS.md rules into topic sections

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test(rust): move llms-types tests inline and default the workspace to inline tests

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test(rust): move llms tests inline beside the code they cover

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* docs(rust): cover every provider format with AGENTS.md and keep each upstream URL in one file

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* ci: drop the check_rust_agents_md step

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* chore: regenerate greptile files.json for the new AGENTS.md files

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* refactor(rust): make every llms-types mod.rs a re-export entrypoint and drop check_rust_agents_md

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* refactor(rust): register Messages providers in one match and drop the nested AGENTS.md list

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
yujonglee 2026-10-10 16:53:35 -07:00 • committed by GitHub
parent 31e2ce1427
commit 705bd84523
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
116 changed files with 6083 additions and 5545 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

152
.greptile/files.json
View file

@ -196,6 +196,13 @@
"litellm-rust/crates/llms/src/anthropic/batches/**"
]
},
{
"path": "litellm-rust/crates/llms/src/anthropic/chat/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/anthropic/chat/",
"scope": [
"litellm-rust/crates/llms/src/anthropic/chat/**"
]
},
{
"path": "litellm-rust/crates/llms/src/anthropic/count_tokens/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/anthropic/count_tokens/",
@ -224,6 +231,13 @@
"litellm-rust/crates/llms/src/azure_ai/messages/**"
]
},
{
"path": "litellm-rust/crates/llms/src/azure_ai/ocr/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/azure_ai/ocr/",
"scope": [
"litellm-rust/crates/llms/src/azure_ai/ocr/**"
]
},
{
"path": "litellm-rust/crates/llms/src/base_llm/messages/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/base_llm/messages/",
@ -231,6 +245,27 @@
"litellm-rust/crates/llms/src/base_llm/messages/**"
]
},
{
"path": "litellm-rust/crates/llms/src/bedrock/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/bedrock/",
"scope": [
"litellm-rust/crates/llms/src/bedrock/**"
]
},
{
"path": "litellm-rust/crates/llms/src/bedrock/audio_transcription/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/bedrock/audio_transcription/",
"scope": [
"litellm-rust/crates/llms/src/bedrock/audio_transcription/**"
]
},
{
"path": "litellm-rust/crates/llms/src/bedrock/chat/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/bedrock/chat/",
"scope": [
"litellm-rust/crates/llms/src/bedrock/chat/**"
]
},
{
"path": "litellm-rust/crates/llms/src/bedrock/messages/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/bedrock/messages/",
@ -238,6 +273,62 @@
"litellm-rust/crates/llms/src/bedrock/messages/**"
]
},
{
"path": "litellm-rust/crates/llms/src/cohere/ocr/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/cohere/ocr/",
"scope": [
"litellm-rust/crates/llms/src/cohere/ocr/**"
]
},
{
"path": "litellm-rust/crates/llms/src/deepseek/messages/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/deepseek/messages/",
"scope": [
"litellm-rust/crates/llms/src/deepseek/messages/**"
]
},
{
"path": "litellm-rust/crates/llms/src/mistral/ocr/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/mistral/ocr/",
"scope": [
"litellm-rust/crates/llms/src/mistral/ocr/**"
]
},
{
"path": "litellm-rust/crates/llms/src/openai/responses/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/openai/responses/",
"scope": [
"litellm-rust/crates/llms/src/openai/responses/**"
]
},
{
"path": "litellm-rust/crates/llms/src/openai_like/chat/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/openai_like/chat/",
"scope": [
"litellm-rust/crates/llms/src/openai_like/chat/**"
]
},
{
"path": "litellm-rust/crates/llms/src/reducto/ocr/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/reducto/ocr/",
"scope": [
"litellm-rust/crates/llms/src/reducto/ocr/**"
]
},
{
"path": "litellm-rust/crates/llms/src/vertex_ai/messages/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/vertex_ai/messages/",
"scope": [
"litellm-rust/crates/llms/src/vertex_ai/messages/**"
]
},
{
"path": "litellm-rust/crates/llms/src/vertex_ai/ocr/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/vertex_ai/ocr/",
"scope": [
"litellm-rust/crates/llms/src/vertex_ai/ocr/**"
]
},
{
"path": "litellm-rust/crates/llms-types/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms-types/",
@ -246,10 +337,24 @@
]
},
{
"path": "litellm-rust/crates/llms-types/src/formats/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms-types/src/formats/",
"path": "litellm-rust/crates/llms-types/src/formats/audio_transcription/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms-types/src/formats/audio_transcription/",
"scope": [
"litellm-rust/crates/llms-types/src/formats/**"
"litellm-rust/crates/llms-types/src/formats/audio_transcription/**"
]
},
{
"path": "litellm-rust/crates/llms-types/src/formats/batches/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms-types/src/formats/batches/",
"scope": [
"litellm-rust/crates/llms-types/src/formats/batches/**"
]
},
{
"path": "litellm-rust/crates/llms-types/src/formats/chat_completions/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms-types/src/formats/chat_completions/",
"scope": [
"litellm-rust/crates/llms-types/src/formats/chat_completions/**"
]
},
{
@ -260,10 +365,45 @@
]
},
{
"path": "litellm-rust/crates/llms-types/src/providers/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms-types/src/providers/",
"path": "litellm-rust/crates/llms-types/src/formats/ocr/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms-types/src/formats/ocr/",
"scope": [
"litellm-rust/crates/llms-types/src/providers/**"
"litellm-rust/crates/llms-types/src/formats/ocr/**"
]
},
{
"path": "litellm-rust/crates/llms-types/src/formats/responses/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms-types/src/formats/responses/",
"scope": [
"litellm-rust/crates/llms-types/src/formats/responses/**"
]
},
{
"path": "litellm-rust/crates/llms-types/src/providers/anthropic/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms-types/src/providers/anthropic/",
"scope": [
"litellm-rust/crates/llms-types/src/providers/anthropic/**"
]
},
{
"path": "litellm-rust/crates/llms-types/src/providers/bedrock/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms-types/src/providers/bedrock/",
"scope": [
"litellm-rust/crates/llms-types/src/providers/bedrock/**"
]
},
{
"path": "litellm-rust/crates/llms-types/src/providers/minimax/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms-types/src/providers/minimax/",
"scope": [
"litellm-rust/crates/llms-types/src/providers/minimax/**"
]
},
{
"path": "litellm-rust/crates/llms-types/src/providers/vertex_ai/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms-types/src/providers/vertex_ai/",
"scope": [
"litellm-rust/crates/llms-types/src/providers/vertex_ai/**"
]
},
{

6
litellm-rust/AGENTS.md
View file

@ -11,9 +11,9 @@ Use the derived conversions directly (`<&'static str>::from(x)` / `.into()`, `st
## Test placement
- Never create a `tests.rs` (or `test.rs`) file under `src/`, and never `#[path = "tests.rs"] mod tests;`
- A test that reaches private items lives inline, in a `#[cfg(test)] mod tests { ... }` at the bottom of the file that owns those items
- A test that only uses the crate's public API lives in `crates/<crate>/tests/<subject>.rs`, next to `src/`
- Split a mixed test file along that line instead of widening visibility to move it
- Tests live inline by default, in a `#[cfg(test)] mod tests { ... }` at the bottom of the file that owns the code under test, whether or not they touch private items
- `crates/<crate>/tests/` is only for tests that drive several modules together end to end, such as a full request through a route. A test of one type or function never goes there
- A helper shared by several inline test modules goes in a `#[cfg(test)] mod test_support;` at the crate root, and only once a second module needs it
- A test for another crate's item belongs in that crate, not in a downstream one
- Never set `autotests = false` or hand-list `[[test]]` targets. Every file directly under `tests/` is discovered by cargo, and a shared helper goes in `tests/<name>/mod.rs` or `tests/<subject>/support.rs` so it is not picked up as a test crate of its own

70
litellm-rust/crates/inference-messages/src/common_utils.rs
View file

@ -15,51 +15,36 @@ use super::Error;
const HEADER_CONTEXT: &str = "messages";
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum MessagesProvider {
Anthropic,
AzureAi,
Bedrock,
Deepseek,
VertexAi,
#[derive(Clone, Copy)]
pub(crate) struct MessagesProvider {
provider: LlmProviders,
config: &'static dyn BaseMessagesConfig,
}
impl MessagesProvider {
pub(crate) fn as_str(self) -> &'static str {
match self {
Self::Anthropic => LlmProviders::Anthropic,
Self::AzureAi => LlmProviders::AzureAi,
Self::Bedrock => LlmProviders::Bedrock,
Self::Deepseek => LlmProviders::Deepseek,
Self::VertexAi => LlmProviders::VertexAi,
}
.into()
self.provider.into()
}
pub(crate) fn config(self) -> &'static dyn BaseMessagesConfig {
match self {
Self::Anthropic => &ANTHROPIC_MESSAGES_CONFIG,
Self::AzureAi => &AZURE_ANTHROPIC_MESSAGES_CONFIG,
Self::Bedrock => &BEDROCK_ANTHROPIC_MESSAGES_CONFIG,
Self::Deepseek => &DEEPSEEK_ANTHROPIC_MESSAGES_CONFIG,
Self::VertexAi => &VERTEX_ANTHROPIC_MESSAGES_CONFIG,
}
self.config
}
}
/// Python's `get_provider_anthropic_messages_config`: Vertex AI serves only its Claude
/// partner models on this route.
pub(crate) fn messages_provider(provider: LlmProviders, model: &str) -> Option<MessagesProvider> {
match provider {
LlmProviders::Anthropic => Some(MessagesProvider::Anthropic),
LlmProviders::AzureAi => Some(MessagesProvider::AzureAi),
LlmProviders::Bedrock => Some(MessagesProvider::Bedrock),
LlmProviders::Deepseek => Some(MessagesProvider::Deepseek),
let config: &'static dyn BaseMessagesConfig = match provider {
LlmProviders::Anthropic => &ANTHROPIC_MESSAGES_CONFIG,
LlmProviders::AzureAi => &AZURE_ANTHROPIC_MESSAGES_CONFIG,
LlmProviders::Bedrock => &BEDROCK_ANTHROPIC_MESSAGES_CONFIG,
LlmProviders::Deepseek => &DEEPSEEK_ANTHROPIC_MESSAGES_CONFIG,
LlmProviders::VertexAi if model.to_ascii_lowercase().contains("claude") => {
Some(MessagesProvider::VertexAi)
&VERTEX_ANTHROPIC_MESSAGES_CONFIG
}
_ => None,
}
_ => return None,
};
Some(MessagesProvider { provider, config })
}
pub(super) fn string_headers(
@ -74,24 +59,19 @@ mod tests {
use rstest::rstest;
use super::{MessagesProvider, messages_provider, string_headers, truncate_error_body};
use super::{messages_provider, string_headers, truncate_error_body};
use crate::Error;
use litellm_core_utils::get_llm_provider_logic::LlmProviders;
#[rstest]
#[case::anthropic("anthropic", MessagesProvider::Anthropic)]
#[case::azure_ai("azure_ai", MessagesProvider::AzureAi)]
#[case::bedrock("bedrock", MessagesProvider::Bedrock)]
#[case::deepseek("deepseek", MessagesProvider::Deepseek)]
#[case::vertex_ai("vertex_ai", MessagesProvider::VertexAi)]
fn provider_round_trips_through_its_python_name(
#[case] name: &str,
#[case] provider: MessagesProvider,
) {
assert_eq!(
messages_provider(name.parse::<LlmProviders>().unwrap(), "claude-sonnet-4-5"),
Some(provider)
);
#[case::anthropic("anthropic")]
#[case::azure_ai("azure_ai")]
#[case::bedrock("bedrock")]
#[case::deepseek("deepseek")]
#[case::vertex_ai("vertex_ai")]
fn provider_keeps_its_python_name(#[case] name: &str) {
let provider =
messages_provider(name.parse::<LlmProviders>().unwrap(), "claude-sonnet-4-5").unwrap();
assert_eq!(provider.as_str(), name);
}
@ -102,7 +82,7 @@ mod tests {
#[case] provider: LlmProviders,
#[case] model: &str,
) {
assert_eq!(messages_provider(provider, model), None);
assert!(messages_provider(provider, model).is_none());
}
#[test]

104
litellm-rust/crates/llms-types/AGENTS.md
View file

@ -1,75 +1,63 @@
The same ownership rule applies to Messages, Responses, Chat Completions, OCR, and other API formats. This crate owns their shared API data contracts. Adapter contracts and shared transformation machinery belong in `llms/src/base_llm/<format>/`, provider policy in `llms/src/<provider>/<format>/`, and call orchestration in `inference-<format>`. A provider originating a format, or several providers using a type, does not change these responsibilities. Existing model locations outside this crate are not exceptions to this rule for new shared API contracts
# rules
- `litellm-llms-types` owns shared API data contracts and their serialization
- A type belongs here when it describes a request, response, event, or value that consumers must agree on independently of how a call executes
- Being public, serializable, or used by several crates is not sufficient
- These are intended boundaries, not a claim that every existing item follows them
## Scope
- Organize public API contracts under `formats`: `messages`, `chat_completions`, `responses`, `ocr`, `audio_transcription`, and `batches`
- Use names such as `litellm_llms_types::formats::messages::MessagesRequest`, without an Anthropic prefix solely because Anthropic designed Messages
- Keep one canonical definition and import path when moving a contract, updating consumers together instead of adding duplicate models or compatibility re-exports
- This crate owns shared API data contracts and their serialization, nothing else
- A type belongs here when consumers must agree on a request, response, event or value independently of how a call executes
- Being public, serializable or used by several crates is not enough on its own
- Keep shared provider-specific wire types and extensions under `providers`
- Provider types may reuse format types. Format types must not depend on provider types
- A field belonging to an API format stays under `formats` even when provider support varies. Including it in a type does not promise provider support
- Add a typed provider extension when a consumer needs to interpret or construct it. Keep adapter-only projections in `llms` until a shared public data contract is needed
- Keep one authoritative representation of each field, preserving unknown fields without duplicating typed values in an extension map
- Provider capability checks, defaults, authentication, header selection, and transformations remain in `llms`
## Layout
- Keep format-independent data helpers such as `headers`, `recognized`, and `serde_compat` at the crate root
- `formats/<format>/` holds a provider-neutral API format, `providers/<provider>/` holds one provider's shared pieces
- Every format and provider is a folder with a `mod.rs` entrypoint and its own `AGENTS.md`. No standalone `<name>.rs` next to a folder
- A `mod.rs` only declares modules and glob re-exports them (`pub use request::*`). Constants go in `constants.rs`, types in a file named for what they describe (`request.rs`, `response.rs`, `streaming.rs`)
- A submodule stays namespaced (`pub mod streaming`) only when its names would clash with the parent's
- Format-independent helpers (`headers`, `recognized`, `serde_compat`, `json_schema`) stay at the crate root
- AGENTS.md files follow the convention in `../llms/AGENTS.md`. Nested files: `src/formats/<format>/` for every format, `src/providers/<provider>/` for every provider
- Shared request/response bodies, message and content-block enums, usage records, tool-call chunks, stream-event payloads, and protocol error bodies belong here
- This includes LiteLLM's normalized response contracts and extensions, not just exact upstream schemas
- `ChatCompletionsResponse` currently represents the response handed to the host, so replacing it with a supposedly more complete upstream schema must not silently change that contract
- Messages web-search result/error schemas and encrypted-content fields belong to the Messages format regardless of which providers implement them
## Formats
- Shared LiteLLM input data such as `ProviderSpecificHeader` and `ProviderSpecificHeaders` also belongs here
- Selecting entries for a provider belongs in `core-utils`, and applying headers belongs in `http`
- A genuinely provider-specific wire value may retain its provider name: `AnthropicBeta` and `BetaSet` describe the `anthropic-beta` header
- Parsing, formatting, deduplication, and value equality belong with those types
- Choosing required betas, OAuth companions, header precedence, or credentials belongs in `llms` and the auth crates
- Name types after the format, never the originator (`formats::messages::MessagesRequest`, not `AnthropicMessagesRequest`)
- A field defined by the format stays in the format type even when provider support varies
- Keep one canonical definition and import path. Move consumers with the type instead of adding re-exports
- LiteLLM's normalized contracts live here too. `ChatCompletionsResponse` is the response handed to the host, so do not swap it for a fuller upstream schema
- Allow deterministic constructors, accessors, serialization, schema generation, and validation of the represented data shape
- `ContentBlock::text`, `ResponsesWsEvent::model`, and `Recognized::known` are examples
- Exact value conversions such as `EffortLevel` to the matching `ReasoningEffort` are acceptable
- Clamping effort, choosing a thinking budget, rewriting content, mapping finish reasons, computing normalized usage, and translating between API formats are policy or transformations and belong outside this crate, even when they are pure functions
## Providers
- Keep call envelopes and execution state in their owning crates
- `MessagesCall`, `MessagesShaping`, prepared provider requests, and the response wrapper containing a live stream belong in `inference-messages`
- Provider config traits, `MessagesTransformContext`, `MessagesModelCapabilities`, `ThinkingBudgets`, `StreamShape`, and transformer state belong in `llms`
- Catalog records and pricing belong in `model-catalog`, which may reuse wire enums such as `ReasoningEffort`
- Host hooks, Python objects, credentials, clients, timeouts, and routing decisions do not become API payload types merely because they cross a crate boundary
- Legacy logging operation selection belongs in `callbacks-legacy-python`, not this crate
- A provider with more than one endpoint format puts shared endpoint paths, default headers and wire types used by several of its formats in `providers/<provider>/`
- Projections that only one adapter decodes (`InvokeChunkPayload`, the count-tokens and batches adapter structs, `ReplayedWebSearchResult`) stay in `llms/src/<provider>/<format>/`
- A typed provider extension of a format (MiniMax media blocks) belongs here once a consumer must construct or interpret it
- Provider types may use format types. Format types never depend on provider types
- Stream-event data belongs here, but live streams, decoders, framing, buffering, and stream lifecycle decisions do not
- Keep SSE and AWS framing in `framer`, provider decoding and conversion in `llms`, and call orchestration in the `inference-<fmt>` crates
- `ResponsesWsEvent` belongs here
- `ResponsesWsTransformResult` wraps the output of a provider transformation rather than a wire event and lives in `llms::base_llm::responses::transformation`
- Protocol error payloads may live here, while operational errors remain in the crate that raises them
## Behavior on types
- Keep provider-only wire envelopes and transformation-specific projections local until there is a shared API contract to expose
- `InvokeChunkPayload` and the permissive `ReplayedWebSearchResult` projection serve decoding or rendering and should not be promoted unchanged into public schemas
- Reuse existing shared message contracts inside batch and token-counting adapters without moving every adapter struct into this crate
- Shared normalization intermediates such as the prompt factory's `Conversation` stay with their algorithms in `core-utils`
- Allowed: deterministic constructors, accessors, serialization, schema generation, shape validation and exact value conversions
- Not allowed: clamping, budget selection, content rewriting, finish-reason mapping, usage normalization or cross-format translation. These belong in `llms`, even when pure
- Provider capability checks, defaults, auth and header selection belong in `llms`
- Keep this crate at the bottom of the dependency graph
- Use data, serialization, and optional schema libraries without depending on workspace execution crates, async runtimes, transport clients, or Python bindings
- No network or filesystem I/O, environment lookup, clock access, model catalog lookup, or global configuration reads belong here
- Defaults must describe the data contract, not select runtime policy
## What lives elsewhere
- Represent known discriminators such as `tool_use` and `server_tool_use` with enum variants
- Preserve each contract's existing treatment of unknown variants, extra fields, missing fields, and explicit nulls
- `Recognized<T>` deliberately retains values that fail typed parsing, including wrong-shaped values, so use it only where permissive passthrough is already part of the contract
- Typing an opaque field must neither reject previously accepted inputs nor accept malformed inputs previously rejected
- Do not silently discard unknown data or tighten a partial projection into a stricter public schema
- Call envelopes and execution state (`MessagesCall`, prepared requests, live streams) belong in `inference-<format>`
- Adapter contracts and transform state (`MessagesTransformContext`, `StreamShape`) belong in `llms`
- SSE and AWS framing belong in `framer`. Catalog records and pricing belong in `model-catalog`
- Host hooks, Python objects, credentials, clients and routing decisions are not payload types
- Test observable serialization, malformed-input rejection, unknown-value preservation, and value semantics in this crate
- Follow the workspace test-placement and `rstest` rules
- Test provider transformations, header policy, and stream execution in their owning crates
- Do not test import locations or Rust source structure as substitutes for behavior
## Dependencies
- Data, serde and optional schema libraries only. No workspace execution crates, async runtimes, transport clients or Python bindings
- No I/O, env lookup, clock access or global config reads. Defaults describe the data contract, not runtime policy
## Unknown data
- Model known discriminators as enum variants and keep each contract's existing handling of unknown variants, extra fields, missing fields and explicit nulls
- Use `Recognized<T>` only where permissive passthrough is already part of the contract
- Typing an opaque field must neither reject previously accepted input nor accept previously rejected input
## Tests
- Cover serialization, malformed-input rejection, unknown-value preservation and value semantics here
- Transformations and header policy are tested in their owning crates
# references
## json_schema.rs
- https://json-schema.org/draft/2020-12/json-schema-core

36
litellm-rust/crates/llms-types/src/formats/AGENTS.md
View file

@ -1,36 +0,0 @@
# references
These links describe shared format fields and discriminators. The official API reference is authoritative, and SDK sources only clarify shapes the reference leaves implicit. These contracts are partial typed projections, not exhaustive upstream schemas
## chat_completions.rs and chat_completions/content.rs
- https://developers.openai.com/api/reference/resources/chat
SDK wire definitions:
- https://github.com/openai/openai-python/blob/main/src/openai/types/chat/chat_completion_content_part_param.py
## responses/output.rs and responses/streaming_websocket.rs
- https://developers.openai.com/api/reference/resources/responses
- https://developers.openai.com/api/reference/resources/responses/streaming-events
SDK wire definitions:
- https://github.com/openai/openai-python/blob/main/src/openai/types/responses/response.py
- https://github.com/openai/openai-python/blob/main/src/openai/types/responses/response_output_item.py
- https://github.com/openai/openai-python/blob/main/src/openai/types/responses/mcp_tool_call_error.py
## batches.rs
- https://developers.openai.com/api/reference/resources/batches
## audio_transcription.rs
- https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions
## ocr.rs
- https://docs.mistral.ai/api/endpoint/ocr
Messages references live in `messages/AGENTS.md`. `LiteLLMOcrResponse` follows the Mistral OCR response shape, and `OcrBoundingBox` describes the corner coordinates providers copy into the normalized `bbox`. Normalized `tables` and `keyValuePairs` stay open because providers pass through different native shapes

9
litellm-rust/crates/llms-types/src/formats/audio_transcription/AGENTS.md Normal file
View file

@ -0,0 +1,9 @@
# rules
- The transcription contract follows OpenAI's `audio/transcriptions` shape, which is LiteLLM's normalized input and output for every transcription provider
- Providers without that endpoint translate in `llms/src/<provider>/audio_transcription/` (Bedrock builds a Converse request)
- Parameter support differs per provider and is filtered in the provider config, not by trimming the type
# references
- https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions

3
litellm-rust/crates/llms-types/src/formats/audio_transcription/mod.rs Normal file
View file

@ -0,0 +1,3 @@
mod response;
pub use response::*;

0
litellm-rust/crates/llms-types/src/formats/audio_transcription.rs → litellm-rust/crates/llms-types/src/formats/audio_transcription/response.rs
View file

9
litellm-rust/crates/llms-types/src/formats/batches/AGENTS.md Normal file
View file

@ -0,0 +1,9 @@
# rules
- The batch contract follows OpenAI's Batch object, which is LiteLLM's normalized batch shape for every provider
- Providers with their own batch API map into it in `llms/src/<provider>/batches/` (Anthropic Message Batches)
- Statuses and counts a provider lacks are derived or defaulted in the provider transformation, not added here
# references
- https://developers.openai.com/api/reference/resources/batches

3
litellm-rust/crates/llms-types/src/formats/batches/mod.rs Normal file
View file

@ -0,0 +1,3 @@
mod response;
pub use response::*;

0
litellm-rust/crates/llms-types/src/formats/batches.rs → litellm-rust/crates/llms-types/src/formats/batches/response.rs
View file

214
litellm-rust/crates/llms-types/src/formats/chat_completions.rs
View file

@ -1,214 +0,0 @@
use serde_json::{Map, Value};
use strum::IntoStaticStr;
mod content;
pub use content::{
ChatContentPart, ChatFile, ChatInputAudio, ChatLogprobs, ChatMediaUrl, ChatMediaUrlParameters,
ChatTokenLogprob, ChatTopLogprob, ChatVideoMetadata, PromptCacheBreakpoint, PromptCacheMode,
};
/// Reasoning effort level accepted or applied by the model.
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq, IntoStaticStr, strum::EnumString, strum::VariantArray)]
#[serde(rename_all = "snake_case")]
pub enum ReasoningEffort {
#[strum(serialize = "none")]
None,
#[strum(serialize = "minimal")]
Minimal,
#[strum(serialize = "low")]
Low,
#[strum(serialize = "medium")]
Medium,
#[strum(serialize = "high")]
High,
#[strum(serialize = "xhigh")]
Xhigh,
#[strum(serialize = "max")]
Max,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(untagged)]
pub enum ChatMessageContent {
Text(String),
Parts(Vec<Value>),
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatMessage {
pub role: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub content: Option<ChatMessageContent>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatCompletionToolCallFunctionChunk {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
pub arguments: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub provider_specific_fields: Option<Map<String, Value>>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatCompletionToolCallChunk {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub id: Option<String>,
#[serde(rename = "type")]
pub tool_type: String,
pub function: ChatCompletionToolCallFunctionChunk,
pub index: i64,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ChatCompletionThinkingBlock {
Thinking {
#[serde(default, skip_serializing_if = "Option::is_none")]
thinking: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
signature: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
cache_control: Option<Value>,
},
RedactedThinking {
#[serde(default, skip_serializing_if = "Option::is_none")]
data: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
cache_control: Option<Value>,
},
}
/// OpenAI `usage`, including the `prompt_tokens_details` split LiteLLM's Python
/// path reports so cost tracking sees the same numbers on either path.
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct PromptTokensDetails {
pub cached_tokens: u64,
pub cache_creation_tokens: u64,
pub text_tokens: u64,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ChatCompletionsUsage {
pub prompt_tokens: u64,
pub completion_tokens: u64,
pub total_tokens: u64,
pub prompt_tokens_details: PromptTokensDetails,
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatCompletionsChoiceMessage {
pub role: String,
// Whether an empty turn is `None` or `""` is the provider's choice, not a
// shared invariant: Anthropic's transform ends on `merged_text or None`
// while Converse assigns the joined string unconditionally. Each config
// mirrors its own, so keep this optional and serialize it even when None.
pub content: Option<String>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatCompletionsChoice {
pub index: u64,
pub message: ChatCompletionsChoiceMessage,
pub finish_reason: String,
}
/// The normalized response handed back to the host.
///
/// There is deliberately no `id`: Python mints the `chatcmpl-…` id on the
/// `ModelResponse` it already created, and echoing the provider's own id here
/// would change it. Pinned by `response_carries_no_id` in the Anthropic chat transformation tests.
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatCompletionsResponse {
pub created: u64,
pub model: String,
pub choices: Vec<ChatCompletionsChoice>,
pub usage: ChatCompletionsUsage,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ChatCompletionDelta {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub content: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub role: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tool_calls: Option<Vec<ChatCompletionToolCallChunk>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub reasoning_content: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub thinking_blocks: Option<Vec<ChatCompletionThinkingBlock>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub provider_specific_fields: Option<Map<String, Value>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatCompletionStreamingChoice {
pub index: u64,
pub delta: ChatCompletionDelta,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub finish_reason: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub logprobs: Option<Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatCompletionChunk {
pub id: String,
pub created: u64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub model: Option<String>,
pub object: String,
pub choices: Vec<ChatCompletionStreamingChoice>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub usage: Option<ChatCompletionsUsage>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub provider_specific_fields: Option<Map<String, Value>>,
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use strum::VariantArray;
use super::*;
#[rstest]
fn reasoning_effort_names_match_the_wire_and_parse_back(
#[values(
ReasoningEffort::None,
ReasoningEffort::Minimal,
ReasoningEffort::Low,
ReasoningEffort::Medium,
ReasoningEffort::High,
ReasoningEffort::Xhigh,
ReasoningEffort::Max
)]
effort: ReasoningEffort,
) {
assert_eq!(
serde_json::to_value(effort).unwrap(),
Value::String(<&'static str>::from(effort).to_string())
);
assert_eq!(<&'static str>::from(effort).parse(), Ok(effort));
assert!(ReasoningEffort::VARIANTS.contains(&effort));
}
#[rstest]
#[case::unknown("ultra")]
#[case::uppercase("HIGH")]
#[case::empty("")]
fn reasoning_effort_parse_rejects(#[case] value: &str) {
assert!(value.parse::<ReasoningEffort>().is_err());
}
}

15
litellm-rust/crates/llms-types/src/formats/chat_completions/AGENTS.md Normal file
View file

@ -0,0 +1,15 @@
# rules
- Chat Completions originated at OpenAI and is the most widely cloned LLM API. Types here describe the format, not OpenAI's deployment of it
- Spoken natively by OpenAI and OpenAI-compatible hosts, served by `llms/src/openai_like/chat/`
- Other providers translate into it from their own API (Anthropic Messages, Bedrock Converse) in `llms/src/<provider>/chat/`
- Compatible hosts commonly
- accept a subset of parameters, which is recorded per provider in `supported_openai_param_mappings`, not by trimming the type
- add top-level request or response fields, preserved through the existing extra-field maps instead of new typed fields
- return slightly different usage or finish reasons, normalized in the provider transformation
- `ChatCompletionsResponse` is LiteLLM's response contract to the host, a typed projection rather than the full upstream schema
# references
- https://developers.openai.com/api/reference/resources/chat
- https://github.com/openai/openai-python/blob/main/src/openai/types/chat/chat_completion_content_part_param.py

166
litellm-rust/crates/llms-types/src/formats/chat_completions/content.rs
View file

@ -153,3 +153,169 @@ pub struct ChatTopLogprob {
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[cfg(test)]
mod tests {
use crate::formats::{
chat_completions::{ChatContentPart, ChatLogprobs, ChatMediaUrl, PromptCacheMode},
messages::ContentSource,
};
use rstest::rstest;
use serde_json::{Value, json};
#[rstest]
#[case::parts(json!([
{"type":"text","text":"hello","cache_control":{"type":"ephemeral"},"prompt_cache_breakpoint":{"mode":"explicit"}},
{"type":"image_url","image_url":{"url":"https://example.test/image","detail":"high","format":"image/png"}},
{"type":"video_url","video_url":"https://example.test/video"},
{"type":"input_audio","input_audio":{"data":"AA==","format":"wav"}},
{"type":"file","file":{"file_id":"file_1","file_data":"JVBE","filename":"a.mp4","format":"video/mp4","detail":"low","video_metadata":{"fps":1,"start_offset":"1s","end_offset":"2s"}}},
{"type":"document","source":{"type":"text","media_type":"text/plain","data":"doc"},"title":"T","context":"C","citations":{"enabled":true}},
{"type":"refusal","refusal":"refused"}
]))]
fn content_parts_round_trip(#[case] wire: Value) {
let parts: Vec<ChatContentPart> = serde_json::from_value(wire.clone()).unwrap();
let [
ChatContentPart::Text {
text,
cache_control,
prompt_cache_breakpoint: Some(breakpoint),
extra: text_extra,
},
ChatContentPart::ImageUrl {
image_url: ChatMediaUrl::Parameters(image),
prompt_cache_breakpoint: None,
..
},
ChatContentPart::VideoUrl {
video_url: ChatMediaUrl::Url(video),
..
},
ChatContentPart::InputAudio { input_audio, .. },
ChatContentPart::File { file, .. },
ChatContentPart::Document {
source,
title: Some(title),
context: Some(context),
citations: Some(citations),
..
},
ChatContentPart::Refusal { refusal, .. },
] = parts.as_slice()
else {
panic!("expected typed content parts");
};
assert_eq!(text, "hello");
assert_eq!(
cache_control.as_ref().unwrap().cache_type.as_deref(),
Some("ephemeral")
);
assert_eq!(breakpoint.mode, PromptCacheMode::Explicit);
assert!(breakpoint.extra.is_empty());
assert!(text_extra.is_empty());
assert_eq!(image.url, "https://example.test/image");
assert_eq!(image.detail.as_deref(), Some("high"));
assert_eq!(image.format.as_deref(), Some("image/png"));
assert!(image.extra.is_empty());
assert_eq!(video, "https://example.test/video");
assert_eq!(input_audio.data, "AA==");
assert_eq!(input_audio.format, "wav");
assert_eq!(file.file_id.as_deref(), Some("file_1"));
assert_eq!(file.file_data.as_deref(), Some("JVBE"));
assert_eq!(file.filename.as_deref(), Some("a.mp4"));
assert_eq!(file.format.as_deref(), Some("video/mp4"));
assert_eq!(file.detail.as_deref(), Some("low"));
assert!(file.extra.is_empty());
let metadata = file.video_metadata.as_ref().unwrap();
assert_eq!(metadata.fps, Some(1.into()));
assert_eq!(metadata.start_offset.as_deref(), Some("1s"));
assert_eq!(metadata.end_offset.as_deref(), Some("2s"));
assert!(metadata.extra.is_empty());
let ContentSource::Text {
data, media_type, ..
} = source.as_ref()
else {
panic!("expected document text source");
};
assert_eq!(data, "doc");
assert_eq!(media_type, "text/plain");
assert_eq!(title, "T");
assert_eq!(context, "C");
assert_eq!(citations.enabled, Some(true));
assert_eq!(refusal, "refused");
assert_eq!(serde_json::to_value(parts).unwrap(), wire);
}
#[rstest]
fn logprobs_round_trip_with_tokens_and_alternatives() {
let wire = json!({
"content":[{"token":"hi","logprob":-1,"bytes":[104,105],"top_logprobs":[{"token":"hey","logprob":-2.5,"bytes":[104]}]}],
"refusal":[{"token":"refused","logprob":-3}]
});
let logprobs: ChatLogprobs = serde_json::from_value(wire.clone()).unwrap();
let token = &logprobs.content.as_ref().unwrap()[0];
assert_eq!(token.token, "hi");
assert_eq!(token.logprob, (-1).into());
assert_eq!(token.bytes.as_deref(), Some([104, 105].as_slice()));
let alternative = &token.top_logprobs.as_ref().unwrap()[0];
assert_eq!(alternative.token, "hey");
assert_eq!(alternative.bytes.as_deref(), Some([104].as_slice()));
assert_eq!(logprobs.refusal.as_ref().unwrap()[0].token, "refused");
assert!(logprobs.refusal.as_ref().unwrap()[0].top_logprobs.is_none());
assert_eq!(serde_json::to_value(logprobs).unwrap(), wire);
}
#[rstest]
#[case::text(json!({"type":"text","text":false}))]
#[case::missing_image(json!({"type":"image_url"}))]
#[case::audio_shape(json!({"type":"input_audio","input_audio":{"data":7,"format":"wav"}}))]
#[case::audio_missing_format(json!({"type":"input_audio","input_audio":{"data":"AA=="}}))]
#[case::image_missing_url(json!({"type":"image_url","image_url":{"detail":"high"}}))]
#[case::file_metadata(json!({"type":"file","file":{"video_metadata":{"fps":"fast"}}}))]
#[case::document_source(json!({"type":"document","source":{"type":"url","url":false}}))]
#[case::refusal_missing_text(json!({"type":"refusal"}))]
#[case::file_missing_file(json!({"type":"file"}))]
#[case::document_missing_source(json!({"type":"document"}))]
#[case::video_missing_url(json!({"type":"video_url"}))]
#[case::cache_control_shape(json!({"type":"text","text":"t","cache_control":"ephemeral"}))]
#[case::breakpoint_missing_mode(json!({"type":"text","text":"t","prompt_cache_breakpoint":{}}))]
#[case::breakpoint_unknown_mode(json!({"type":"text","text":"t","prompt_cache_breakpoint":{"mode":"auto"}}))]
#[case::unknown_tag(json!({"type":"future"}))]
fn content_parts_reject_malformed_typed_fields(#[case] wire: Value) {
assert!(serde_json::from_value::<ChatContentPart>(wire).is_err());
}
#[rstest]
fn partial_file_preserves_extensions_and_omits_null_optionals() {
let wire = json!({"type":"file","file":{"file_id":null,"filename":"a.pdf","extension":[1,null]},"future":true});
let part: ChatContentPart = serde_json::from_value(wire).unwrap();
let ChatContentPart::File {
file,
prompt_cache_breakpoint: None,
extra,
} = &part
else {
panic!("expected file")
};
assert!(file.file_id.is_none());
assert!(file.file_data.is_none());
assert_eq!(file.filename.as_deref(), Some("a.pdf"));
assert_eq!(extra["future"], json!(true));
assert_eq!(
serde_json::to_value(part).unwrap(),
json!({"type":"file","file":{"filename":"a.pdf","extension":[1,null]},"future":true})
);
}
#[rstest]
#[case::missing_token(json!({"logprob":-1}))]
#[case::missing_logprob(json!({"token":"hi"}))]
#[case::wrong_bytes(json!({"token":"hi","logprob":-1,"bytes":[256]}))]
fn token_logprobs_reject_malformed_fields(#[case] wire: Value) {
assert!(
serde_json::from_value::<crate::formats::chat_completions::ChatTokenLogprob>(wire)
.is_err()
);
}
}

9
litellm-rust/crates/llms-types/src/formats/chat_completions/mod.rs Normal file
View file

@ -0,0 +1,9 @@
mod content;
mod request;
mod response;
mod streaming;
pub use content::*;
pub use request::*;
pub use response::*;
pub use streaming::*;

77
litellm-rust/crates/llms-types/src/formats/chat_completions/request.rs Normal file
View file

@ -0,0 +1,77 @@
use serde_json::{Map, Value};
use strum::IntoStaticStr;
/// Reasoning effort level accepted or applied by the model.
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq, IntoStaticStr, strum::EnumString, strum::VariantArray)]
#[serde(rename_all = "snake_case")]
pub enum ReasoningEffort {
#[strum(serialize = "none")]
None,
#[strum(serialize = "minimal")]
Minimal,
#[strum(serialize = "low")]
Low,
#[strum(serialize = "medium")]
Medium,
#[strum(serialize = "high")]
High,
#[strum(serialize = "xhigh")]
Xhigh,
#[strum(serialize = "max")]
Max,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(untagged)]
pub enum ChatMessageContent {
Text(String),
Parts(Vec<Value>),
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatMessage {
pub role: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub content: Option<ChatMessageContent>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[cfg(test)]
mod tests {
use super::*;
use rstest::rstest;
use strum::VariantArray;
#[rstest]
fn reasoning_effort_names_match_the_wire_and_parse_back(
#[values(
ReasoningEffort::None,
ReasoningEffort::Minimal,
ReasoningEffort::Low,
ReasoningEffort::Medium,
ReasoningEffort::High,
ReasoningEffort::Xhigh,
ReasoningEffort::Max
)]
effort: ReasoningEffort,
) {
assert_eq!(
serde_json::to_value(effort).unwrap(),
Value::String(<&'static str>::from(effort).to_string())
);
assert_eq!(<&'static str>::from(effort).parse(), Ok(effort));
assert!(ReasoningEffort::VARIANTS.contains(&effort));
}
#[rstest]
#[case::unknown("ultra")]
#[case::uppercase("HIGH")]
#[case::empty("")]
fn reasoning_effort_parse_rejects(#[case] value: &str) {
assert!(value.parse::<ReasoningEffort>().is_err());
}
}

88
litellm-rust/crates/llms-types/src/formats/chat_completions/response.rs Normal file
View file

@ -0,0 +1,88 @@
use serde_json::{Map, Value};
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatCompletionToolCallFunctionChunk {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
pub arguments: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub provider_specific_fields: Option<Map<String, Value>>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatCompletionToolCallChunk {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub id: Option<String>,
#[serde(rename = "type")]
pub tool_type: String,
pub function: ChatCompletionToolCallFunctionChunk,
pub index: i64,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ChatCompletionThinkingBlock {
Thinking {
#[serde(default, skip_serializing_if = "Option::is_none")]
thinking: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
signature: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
cache_control: Option<Value>,
},
RedactedThinking {
#[serde(default, skip_serializing_if = "Option::is_none")]
data: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
cache_control: Option<Value>,
},
}
/// OpenAI `usage`, including the `prompt_tokens_details` split LiteLLM's Python
/// path reports so cost tracking sees the same numbers on either path.
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct PromptTokensDetails {
pub cached_tokens: u64,
pub cache_creation_tokens: u64,
pub text_tokens: u64,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ChatCompletionsUsage {
pub prompt_tokens: u64,
pub completion_tokens: u64,
pub total_tokens: u64,
pub prompt_tokens_details: PromptTokensDetails,
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatCompletionsChoiceMessage {
pub role: String,
// Whether an empty turn is `None` or `""` is the provider's choice, not a
// shared invariant: Anthropic's transform ends on `merged_text or None`
// while Converse assigns the joined string unconditionally. Each config
// mirrors its own, so keep this optional and serialize it even when None.
pub content: Option<String>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatCompletionsChoice {
pub index: u64,
pub message: ChatCompletionsChoiceMessage,
pub finish_reason: String,
}
/// The normalized response handed back to the host.
///
/// There is deliberately no `id`: Python mints the `chatcmpl-…` id on the
/// `ModelResponse` it already created, and echoing the provider's own id here
/// would change it. Pinned by `response_carries_no_id` in the Anthropic chat transformation tests.
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatCompletionsResponse {
pub created: u64,
pub model: String,
pub choices: Vec<ChatCompletionsChoice>,
pub usage: ChatCompletionsUsage,
}

48
litellm-rust/crates/llms-types/src/formats/chat_completions/streaming.rs Normal file
View file

@ -0,0 +1,48 @@
use serde_json::{Map, Value};
use crate::formats::chat_completions::{
ChatCompletionThinkingBlock, ChatCompletionToolCallChunk, ChatCompletionsUsage,
};
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ChatCompletionDelta {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub content: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub role: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tool_calls: Option<Vec<ChatCompletionToolCallChunk>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub reasoning_content: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub thinking_blocks: Option<Vec<ChatCompletionThinkingBlock>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub provider_specific_fields: Option<Map<String, Value>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatCompletionStreamingChoice {
pub index: u64,
pub delta: ChatCompletionDelta,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub finish_reason: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub logprobs: Option<Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatCompletionChunk {
pub id: String,
pub created: u64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub model: Option<String>,
pub object: String,
pub choices: Vec<ChatCompletionStreamingChoice>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub usage: Option<ChatCompletionsUsage>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub provider_specific_fields: Option<Map<String, Value>>,
}

26
litellm-rust/crates/llms-types/src/formats/messages/AGENTS.md
View file

@ -1,19 +1,25 @@
This directory owns shared Messages API data contracts and serialization: request and response bodies, messages, content blocks, usage, and stream-event payloads. Messages is a format independent of the provider that originated it. Keep one canonical definition of each shared contract here
# rules
Adapter contracts and execution inputs such as `MessagesTransformContext` belong in `llms/src/base_llm/messages`. Provider rewriting and interpretation belong in `llms/src/<provider>/messages`. Call envelopes, live streams, and call orchestration belong in `inference-messages`
Represent web-search results, encrypted-content fields, and thinking configuration as data here. Decisions to flatten results, remove encrypted content, select thinking budgets, or require beta headers belong to provider transformations. Data-shape validation belongs here, while model capability checks and request adaptation do not
- Messages originated at Anthropic and is served by many hosts. Types here describe the format, independent of any host
- Spoken by Anthropic, Bedrock InvokeModel, Vertex AI `rawPredict`, Azure AI Foundry, DeepSeek and MiniMax, each adapted in `llms/src/<provider>/messages/`
- Hosts extend it with
- their own content blocks or media sources, which get typed in `providers/<provider>/` once a consumer needs them (MiniMax image, video and mid-conversation system blocks)
- beta features gated by the `anthropic-beta` header, whose accepted values differ per host (`providers/anthropic/beta.rs`)
- Hosts deviate by
- moving `model` or the API version out of the body or into the URL (Bedrock, Vertex)
- framing the stream differently (AWS event stream on Bedrock instead of SSE)
- rejecting fields or discriminators (DeepSeek rejects `type: custom` tools and billing system blocks)
- Handle these in the provider adapter, never by loosening or forking a type here
- Request and response bodies, messages, content blocks, tools, usage and stream-event payloads live here
- Web-search results, encrypted-content fields and thinking configuration are data here
- Flattening results, stripping encrypted content, choosing thinking budgets and requiring betas are provider policy in `llms`
- `MessagesTransformContext` and other adapter inputs live in `llms/src/base_llm/messages/`. Call envelopes live in `inference-messages`
- Beta-only blocks and tools (MCP, compaction, tool changes, advisor, fallback, browser state, toolsets) follow the beta reference
# references
These upstream contracts document fields and discriminators. They are documentation links, not saved golden snapshots. Beta-only blocks and tools (MCP, compaction, tool changes, advisor, fallback, browser state, and toolsets) follow the beta reference
## Messages bodies, content, tools, and streaming
- https://platform.claude.com/docs/en/api/http/messages/create
- https://platform.claude.com/docs/en/api/messages/create.md
- https://platform.claude.com/docs/en/api/beta/messages/create.md
- https://platform.claude.com/docs/en/build-with-claude/streaming.md
- https://platform.claude.com/docs/en/build-with-claude/context-editing
Anthropic-compatible hosts document their deviations in the `llms/src/<provider>/messages` guides. Copy a host reference into `../../providers/AGENTS.md` only when that host gets a typed extension in `providers`

515
litellm-rust/crates/llms-types/src/formats/messages/content.rs
View file

@ -710,3 +710,518 @@ pub enum FallbackTrigger {
extra: Map<String, Value>,
},
}
#[cfg(test)]
mod tests {
use crate::formats::messages::AdvisorToolResultContent;
use crate::formats::messages::{BashCodeExecutionOutput, BashCodeExecutionToolResultContent};
use crate::formats::messages::{BlockContent, BrowserStateChange};
use crate::formats::messages::Citation;
use crate::formats::messages::CodeExecutionOutput;
use crate::formats::messages::CodeExecutionToolResultContent;
use crate::formats::messages::ContentSource;
use crate::formats::messages::FallbackTrigger;
use crate::formats::messages::{McpToolResultContent, McpToolResultText};
use crate::formats::messages::MessagesContentPart;
use crate::formats::messages::MessagesToolParam;
use crate::formats::messages::{TextEditorCodeExecutionToolResultContent, TextEditorFileType};
use crate::formats::messages::{ToolCaller, ToolChange, ToolChangeTarget};
use crate::formats::messages::ToolSearchReference;
use crate::formats::messages::ToolSearchToolResultContent;
use crate::formats::messages::WebFetchDocument;
use crate::formats::messages::WebFetchToolResultContent;
use crate::formats::messages::WebSearchToolResultContent;
use crate::test_support::*;
use rstest::rstest;
use serde_json::{Value, json};
#[rstest]
#[case::base64(json!({"type":"base64","media_type":"image/png","data":"AA=="}))]
#[case::url(json!({"type":"url","url":"https://example.test/image"}))]
#[case::file(json!({"type":"file","file_id":"file_1"}))]
#[case::text(json!({"type":"text","media_type":"text/plain","data":"document"}))]
#[case::content(json!({"type":"content","content":"nested"}))]
fn content_sources_round_trip(#[case] wire: Value) {
round_trip::<ContentSource>(wire);
}
fn part(wire: Value) -> MessagesContentPart {
round_trip::<MessagesContentPart>(wire)
}
#[rstest]
fn text_blocks_expose_every_citation_location() {
let block = part(json!({
"type":"text",
"text":"cited",
"citations":[
{"type":"char_location","cited_text":"a","document_index":0,"start_char_index":1,"end_char_index":6,"file_id":"file_1"},
{"type":"page_location","cited_text":"b","document_index":1,"start_page_number":1,"end_page_number":2},
{"type":"content_block_location","cited_text":"c","document_index":2,"start_block_index":0,"end_block_index":1},
{"type":"web_search_result_location","cited_text":"d","url":"https://example.test","encrypted_index":"opaque","title":"result"},
{"type":"search_result_location","cited_text":"e","search_result_index":3,"source":"kb","start_block_index":0,"end_block_index":2}
],
"cache_control":{"type":"ephemeral"}
}));
let MessagesContentPart::Text(text) = &block else {
panic!("expected text block");
};
assert_eq!(text.text, "cited");
let [
Citation::CharLocation(chars),
Citation::PageLocation(page),
Citation::ContentBlockLocation(blocks),
Citation::WebSearchResultLocation(search),
Citation::SearchResultLocation(result),
] = text.citations.as_deref().unwrap()
else {
panic!("expected one citation of each location type");
};
assert_eq!((chars.start_char_index, chars.end_char_index), (1, 6));
assert_eq!(chars.file_id.as_deref(), Some("file_1"));
assert_eq!((page.start_page_number, page.end_page_number), (1, 2));
assert_eq!(blocks.document_index, 2);
assert_eq!(search.encrypted_index, "opaque");
assert_eq!(result.search_result_index, 3);
assert_eq!(result.source, "kb");
}
#[rstest]
fn text_block_null_optionals_are_omitted() {
let block: MessagesContentPart = serde_json::from_value(
json!({"type":"text","text":"hi","citations":null,"cache_control":null,"future":null}),
)
.unwrap();
let MessagesContentPart::Text(text) = &block else {
panic!("expected text block");
};
assert!(text.citations.is_none());
assert!(text.cache_control.is_none());
assert_eq!(text.extra.get("future"), Some(&Value::Null));
assert_eq!(
serde_json::to_value(block).unwrap(),
json!({"type":"text","text":"hi","future":null})
);
}
#[rstest]
fn request_media_blocks_expose_sources() {
let MessagesContentPart::Image(image) = part(json!({
"type":"image",
"source":{"type":"base64","media_type":"image/png","data":"AA=="},
"transformations":{"oversized_image":"error"}
})) else {
panic!("expected image block");
};
assert!(
matches!(&image.source, ContentSource::Base64 { media_type, .. } if media_type == "image/png")
);
assert_eq!(image.extra["transformations"]["oversized_image"], "error");
let MessagesContentPart::Document(document) = part(json!({
"type":"document",
"source":{"type":"content","content":[
{"type":"text","text":"Section 1"},
{"type":"image","source":{"type":"url","url":"https://example.test/chart.png"}}
]},
"title":"Q3 report",
"context":"quarterly",
"citations":{"enabled":true}
})) else {
panic!("expected document block");
};
assert_eq!(document.title.as_deref(), Some("Q3 report"));
assert_eq!(document.citations.as_ref().unwrap().enabled, Some(true));
let ContentSource::Content {
content: BlockContent::Blocks(blocks),
..
} = &document.source
else {
panic!("expected content-block source");
};
assert!(matches!(
blocks.as_slice(),
[MessagesContentPart::Text(_), MessagesContentPart::Image(_)]
));
let MessagesContentPart::SearchResult(search) = part(json!({
"type":"search_result",
"source":"https://example.test/result",
"title":"result",
"content":[{"type":"text","text":"found"}],
"citations":{"enabled":false}
})) else {
panic!("expected search result block");
};
assert_eq!(search.source, "https://example.test/result");
assert!(
matches!(search.content.as_slice(), [MessagesContentPart::Text(text)] if text.text == "found")
);
}
#[rstest]
fn reasoning_and_tool_blocks_expose_required_fields() {
let MessagesContentPart::Thinking(thinking) =
part(json!({"type":"thinking","thinking":"plan","signature":"sig"}))
else {
panic!("expected thinking block");
};
assert_eq!(
(thinking.thinking.as_str(), thinking.signature.as_str()),
("plan", "sig")
);
let MessagesContentPart::RedactedThinking(redacted) =
part(json!({"type":"redacted_thinking","data":"opaque"}))
else {
panic!("expected redacted thinking block");
};
assert_eq!(redacted.data, "opaque");
let MessagesContentPart::ToolUse(tool_use) = part(json!({
"type":"tool_use",
"id":"toolu_1",
"name":"lookup",
"input":{"query":[1,null]},
"caller":{"type":"code_execution_20260120","tool_id":"srvtoolu_1"},
"toolset_name":"browser"
})) else {
panic!("expected tool use block");
};
assert_eq!(tool_use.input["query"], json!([1, null]));
assert!(
matches!(&tool_use.caller, Some(ToolCaller::CodeExecution20260120 { tool_id, .. }) if tool_id == "srvtoolu_1")
);
assert_eq!(tool_use.toolset_name.as_deref(), Some("browser"));
let MessagesContentPart::ToolResult(result) = part(json!({
"type":"tool_result",
"tool_use_id":"toolu_1",
"is_error":false,
"content":[
{"type":"tool_reference","tool_name":"lookup"},
{"type":"browser_state","tabs":[{"tab_id":"1","title":"","url":"","active":true}],
"state_changes":[{"type":"download_completed","download_id":"d1","url":"https://example.test/f","size_bytes":3}]}
]
})) else {
panic!("expected tool result block");
};
let Some(BlockContent::Blocks(blocks)) = &result.content else {
panic!("expected nested result blocks");
};
let [
MessagesContentPart::ToolReference(reference),
MessagesContentPart::BrowserState(browser),
] = blocks.as_slice()
else {
panic!("expected tool reference and browser state");
};
assert_eq!(reference.tool_name, "lookup");
assert_eq!(browser.tabs[0].active, Some(true));
assert!(matches!(
browser.state_changes.as_deref(),
Some([BrowserStateChange::DownloadCompleted {
size_bytes: Some(3),
path: None,
..
}])
));
let MessagesContentPart::ToolResult(text_result) =
part(json!({"type":"tool_result","tool_use_id":"toolu_2","content":"done"}))
else {
panic!("expected tool result block");
};
assert_eq!(text_result.content, Some(BlockContent::Text("done".into())));
assert!(text_result.is_error.is_none());
}
#[rstest]
fn server_tool_results_expose_nested_result_unions() {
let MessagesContentPart::ServerToolUse(server) = part(
json!({"type":"server_tool_use","id":"srvtoolu_1","name":"web_search","input":{"query":"rust"}}),
) else {
panic!("expected server tool use");
};
assert_eq!(server.name, "web_search");
let MessagesContentPart::WebSearchToolResult(search) = part(json!({
"type":"web_search_tool_result",
"tool_use_id":"srvtoolu_1",
"content":[{"type":"web_search_result","url":"https://example.test","title":"t","encrypted_content":"e","page_age":"1d"}]
})) else {
panic!("expected web search result");
};
let WebSearchToolResultContent::Results(results) = &search.content else {
panic!("expected search results");
};
assert_eq!(results[0].page_age.as_deref(), Some("1d"));
let MessagesContentPart::WebSearchToolResult(search_error) = part(json!({
"type":"web_search_tool_result",
"tool_use_id":"srvtoolu_1",
"content":{"type":"web_search_tool_result_error","error_code":"max_uses_exceeded"}
})) else {
panic!("expected web search error");
};
assert!(
matches!(&search_error.content, WebSearchToolResultContent::Error(error) if error.error_code == "max_uses_exceeded")
);
let MessagesContentPart::WebFetchToolResult(fetch) = part(json!({
"type":"web_fetch_tool_result",
"tool_use_id":"srvtoolu_2",
"content":{"type":"web_fetch_result","url":"https://example.test","retrieved_at":"2026-01-01T00:00:00Z",
"content":{"type":"document","source":{"type":"text","media_type":"text/plain","data":"page"}}}
})) else {
panic!("expected web fetch result");
};
let WebFetchToolResultContent::WebFetchResult(fetched) = &fetch.content else {
panic!("expected fetched page");
};
let WebFetchDocument::Document(document) = &fetched.content;
assert!(matches!(&document.source, ContentSource::Text { data, .. } if data == "page"));
assert!(fetched.extra.is_empty());
let MessagesContentPart::CodeExecutionToolResult(code) = part(json!({
"type":"code_execution_tool_result",
"tool_use_id":"srvtoolu_3",
"content":{"type":"encrypted_code_execution_result","encrypted_stdout":"opaque","stderr":"","return_code":-1,
"content":[{"type":"code_execution_output","file_id":"file_1"}]}
})) else {
panic!("expected code execution result");
};
let CodeExecutionToolResultContent::EncryptedCodeExecutionResult(encrypted) = &code.content
else {
panic!("expected encrypted execution result");
};
assert_eq!(encrypted.return_code, -1);
assert!(
matches!(encrypted.content.as_slice(), [CodeExecutionOutput::CodeExecutionOutput { file_id, .. }] if file_id == "file_1")
);
let MessagesContentPart::BashCodeExecutionToolResult(bash) = part(json!({
"type":"bash_code_execution_tool_result",
"tool_use_id":"srvtoolu_7",
"content":{"type":"bash_code_execution_result","stdout":"ok","stderr":"","return_code":0,
"content":[{"type":"bash_code_execution_output","file_id":"file_2"}]}
})) else {
panic!("expected bash code execution result");
};
let BashCodeExecutionToolResultContent::BashCodeExecutionResult(ran) = &bash.content else {
panic!("expected bash execution result");
};
assert_eq!((ran.stdout.as_str(), ran.return_code), ("ok", 0));
assert!(
matches!(ran.content.as_slice(), [BashCodeExecutionOutput::BashCodeExecutionOutput { file_id, .. }] if file_id == "file_2")
);
assert!(ran.extra.is_empty());
let MessagesContentPart::TextEditorCodeExecutionToolResult(editor) = part(json!({
"type":"text_editor_code_execution_tool_result",
"tool_use_id":"srvtoolu_4",
"content":{"type":"text_editor_code_execution_view_result","content":"fn main() {}","file_type":"text","num_lines":1}
})) else {
panic!("expected text editor result");
};
let TextEditorCodeExecutionToolResultContent::TextEditorCodeExecutionViewResult(view) =
&editor.content
else {
panic!("expected view result");
};
assert_eq!(view.file_type, TextEditorFileType::Text);
assert_eq!(view.num_lines, Some(1));
assert!(view.start_line.is_none());
let MessagesContentPart::ToolSearchToolResult(tool_search) = part(json!({
"type":"tool_search_tool_result",
"tool_use_id":"srvtoolu_5",
"content":{"type":"tool_search_tool_result_error","error_code":"unavailable","error_message":"down"}
})) else {
panic!("expected tool search result");
};
assert!(
matches!(&tool_search.content, ToolSearchToolResultContent::ToolSearchToolResultError(error) if error.error_message.as_deref() == Some("down"))
);
let MessagesContentPart::ToolSearchToolResult(found) = part(json!({
"type":"tool_search_tool_result",
"tool_use_id":"srvtoolu_8",
"content":{"type":"tool_search_tool_search_result","tool_references":[{"type":"tool_reference","tool_name":"lookup"}]}
})) else {
panic!("expected tool search result");
};
let ToolSearchToolResultContent::ToolSearchToolSearchResult(result) = &found.content else {
panic!("expected tool search hits");
};
let [ToolSearchReference::ToolReference(reference)] = result.tool_references.as_slice()
else {
panic!("expected one tool reference");
};
assert_eq!(reference.tool_name, "lookup");
assert!(reference.extra.is_empty() && result.extra.is_empty());
let MessagesContentPart::AdvisorToolResult(advisor) = part(json!({
"type":"advisor_tool_result",
"tool_use_id":"srvtoolu_6",
"content":{"type":"advisor_result","text":"advice"}
})) else {
panic!("expected advisor result");
};
assert!(
matches!(&advisor.content, AdvisorToolResultContent::AdvisorResult { text, stop_reason: None, .. } if text == "advice")
);
}
#[rstest]
fn beta_blocks_expose_mcp_compaction_and_fallback_fields() {
let MessagesContentPart::McpToolUse(mcp) = part(
json!({"type":"mcp_tool_use","id":"mcptoolu_1","name":"search","server_name":"kb","input":{}}),
) else {
panic!("expected MCP tool use");
};
assert_eq!(mcp.server_name, "kb");
let MessagesContentPart::McpToolResult(mcp_result) = part(
json!({"type":"mcp_tool_result","tool_use_id":"mcptoolu_1","is_error":true,"content":"failed"}),
) else {
panic!("expected MCP tool result");
};
assert_eq!(mcp_result.is_error, Some(true));
assert_eq!(
mcp_result.content,
Some(McpToolResultContent::Text("failed".into()))
);
let MessagesContentPart::McpToolResult(text_result) = part(json!({
"type":"mcp_tool_result",
"tool_use_id":"mcptoolu_3",
"content":[{"type":"text","text":"found"}]
})) else {
panic!("expected MCP tool result");
};
let Some(McpToolResultContent::Blocks(blocks)) = &text_result.content else {
panic!("expected MCP text blocks");
};
let [McpToolResultText::Text(text)] = blocks.as_slice() else {
panic!("expected one text block");
};
assert_eq!(text.text, "found");
let MessagesContentPart::McpToolResult(empty_result) =
part(json!({"type":"mcp_tool_result","tool_use_id":"mcptoolu_2"}))
else {
panic!("expected MCP tool result");
};
assert!(empty_result.content.is_none());
assert!(empty_result.extra.is_empty());
let MessagesContentPart::McpToolListing(listing) = part(json!({
"type":"mcp_tool_listing",
"mcp_server_name":"kb",
"tools":[{"name":"search","input_schema":{"type":"object"}}]
})) else {
panic!("expected MCP tool listing");
};
assert!(listing.tools[0].description.is_none());
let MessagesContentPart::Compaction(compaction) = part(json!({
"type":"compaction",
"content":"summary",
"encrypted_content":"opaque",
"tool_changes":[
{"type":"tool_addition","tool":{"type":"tool_definition","definition":{"name":"lookup","input_schema":{"type":"object"}}}},
{"type":"tool_removal","tool":{"type":"mcp_tool_reference","server_name":"kb","name":"search"}}
]
})) else {
panic!("expected compaction block");
};
let [
ToolChange::ToolAddition(addition),
ToolChange::ToolRemoval(removal),
] = compaction.tool_changes.as_deref().unwrap()
else {
panic!("expected tool addition and removal");
};
let ToolChangeTarget::ToolDefinition { definition, .. } = &addition.tool else {
panic!("expected inline tool definition");
};
assert!(
matches!(definition.as_ref(), MessagesToolParam::Custom(tool) if tool.name == "lookup")
);
assert!(
matches!(&removal.tool, ToolChangeTarget::McpToolReference { server_name, .. } if server_name == "kb")
);
let failed: MessagesContentPart =
serde_json::from_value(json!({"type":"compaction","content":null})).unwrap();
assert_eq!(failed, MessagesContentPart::Compaction(Default::default()));
let MessagesContentPart::Fallback(fallback) = part(json!({
"type":"fallback",
"from":{"model":"claude-opus-5-5"},
"to":{"model":"claude-sonnet-5-5"},
"trigger":{"type":"refusal","category":"cyber"}
})) else {
panic!("expected fallback block");
};
assert_eq!(fallback.from.model, "claude-opus-5-5");
assert_eq!(fallback.to.model, "claude-sonnet-5-5");
let Some(FallbackTrigger::Refusal { category, extra }) = &fallback.trigger else {
panic!("expected refusal trigger");
};
assert_eq!(category.as_deref(), Some("cyber"));
assert!(extra.is_empty() && fallback.extra.is_empty());
let MessagesContentPart::Fallback(untriggered) = part(json!({
"type":"fallback",
"from":{"model":"claude-opus-5-5"},
"to":{"model":"claude-sonnet-5-5"}
})) else {
panic!("expected fallback block");
};
assert!(untriggered.trigger.is_none());
}
#[rstest]
#[case::missing_tag(json!({"text":"hello"}))]
#[case::unknown_tag(json!({"type":"future_block","text":"hello"}))]
#[case::text_without_text(json!({"type":"text"}))]
#[case::wrong_text(json!({"type":"text","text":7}))]
#[case::citation_missing_location(json!({"type":"text","text":"a","citations":[{"type":"char_location","cited_text":"a","document_index":0}]}))]
#[case::unknown_citation(json!({"type":"text","text":"a","citations":[{"type":"future_location"}]}))]
#[case::thinking_without_signature(json!({"type":"thinking","thinking":"plan"}))]
#[case::tool_use_without_id(json!({"type":"tool_use","name":"lookup","input":{}}))]
#[case::tool_use_array_input(json!({"type":"tool_use","id":"t","name":"lookup","input":[]}))]
#[case::tool_result_without_id(json!({"type":"tool_result","content":"done"}))]
#[case::malformed_nested_block(json!({"type":"tool_result","tool_use_id":"t","content":[{"type":"text","text":7}]}))]
#[case::malformed_source(json!({"type":"document","source":{"type":"url","url":7}}))]
#[case::image_without_source(json!({"type":"image"}))]
#[case::search_result_without_title(json!({"type":"search_result","source":"s","content":[]}))]
#[case::web_search_result_without_url(json!({"type":"web_search_tool_result","tool_use_id":"t","content":[{"type":"web_search_result","title":"t","encrypted_content":"e"}]}))]
#[case::unknown_fetch_result(json!({"type":"web_fetch_tool_result","tool_use_id":"t","content":{"type":"future"}}))]
#[case::fractional_return_code(json!({"type":"bash_code_execution_tool_result","tool_use_id":"t","content":{"type":"bash_code_execution_result","stdout":"","stderr":"","return_code":0.5,"content":[]}}))]
#[case::unknown_file_type(json!({"type":"text_editor_code_execution_tool_result","tool_use_id":"t","content":{"type":"text_editor_code_execution_view_result","content":"","file_type":"video"}}))]
#[case::negative_line_count(json!({"type":"text_editor_code_execution_tool_result","tool_use_id":"t","content":{"type":"text_editor_code_execution_str_replace_result","new_lines":-1}}))]
#[case::unknown_tool_change(json!({"type":"compaction","tool_changes":[{"type":"tool_addition","tool":{"type":"future"}}]}))]
#[case::tool_search_non_reference(json!({"type":"tool_search_tool_result","tool_use_id":"t","content":{"type":"tool_search_tool_search_result","tool_references":[{"type":"text","text":"lookup"}]}}))]
#[case::web_fetch_text_content(json!({"type":"web_fetch_tool_result","tool_use_id":"t","content":{"type":"web_fetch_result","url":"https://example.test","content":{"type":"text","text":"page"}}}))]
#[case::bash_result_with_code_output(json!({"type":"bash_code_execution_tool_result","tool_use_id":"t","content":{"type":"bash_code_execution_result","stdout":"","stderr":"","return_code":0,"content":[{"type":"code_execution_output","file_id":"f"}]}}))]
#[case::code_result_with_bash_output(json!({"type":"code_execution_tool_result","tool_use_id":"t","content":{"type":"code_execution_result","stdout":"","stderr":"","return_code":0,"content":[{"type":"bash_code_execution_output","file_id":"f"}]}}))]
#[case::fallback_unknown_trigger(json!({"type":"fallback","from":{"model":"a"},"to":{"model":"b"},"trigger":{"type":"overload"}}))]
#[case::unknown_state_change(json!({"type":"browser_state","tabs":[],"state_changes":[{"type":"tab_closed","tab_id":"1"}]}))]
fn content_parts_reject_malformed_known_fields(#[case] wire: Value) {
assert!(serde_json::from_value::<MessagesContentPart>(wire).is_err());
}
#[rstest]
#[case::image(json!([{"type":"image","source":{"type":"url","url":"https://example.test/a.png"}}]))]
#[case::tool_reference(json!([{"type":"tool_reference","tool_name":"lookup"}]))]
#[case::untagged(json!([{"text":"found"}]))]
fn mcp_tool_result_content_rejects_non_text_blocks(#[case] content: Value) {
assert!(
serde_json::from_value::<MessagesContentPart>(
json!({"type":"mcp_tool_result","tool_use_id":"mcptoolu_1","content":content})
)
.is_err()
);
}
}

290
litellm-rust/crates/llms-types/src/formats/messages/metadata.rs
View file

@ -338,3 +338,293 @@ pub struct CacheMissedTokens {
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[cfg(test)]
mod tests {
use crate::formats::messages::CacheMissReason;
use crate::formats::messages::ContainerReference;
use crate::formats::messages::ContextManagementResponse;
use crate::formats::messages::McpServer;
use crate::formats::messages::{
MessageRole, MessageType, MessagesCompaction, MessagesContainer,
};
use crate::formats::messages::{
MessagesDiagnostics, MessagesDiagnosticsParam, MessagesMetadata,
};
use crate::formats::messages::OutputFormat;
use crate::formats::messages::Safeguard;
use crate::formats::messages::{SkillType, StopDetails, StopDetailsType, StopReason};
use crate::json_schema::JsonSchema;
use crate::{recognized::Recognized, test_support::*};
use rstest::rstest;
use serde_json::{Value, json};
#[rstest]
fn metadata_contracts_round_trip() {
let metadata = round_trip::<MessagesMetadata>(json!({"user_id":"user_1","future":true}));
assert_eq!(metadata.user_id.as_deref(), Some("user_1"));
assert_eq!(metadata.extra.get("future"), Some(&json!(true)));
let output_format = round_trip::<OutputFormat>(json!({
"type":"json_schema",
"schema":{"type":"object","properties":{"name":{"type":"string"}}},
"strict":true
}));
assert_eq!(output_format.strict, Some(true));
assert!(output_format.extra.is_empty());
let JsonSchema::Object(schema) = &output_format.schema else {
panic!("expected output object schema");
};
assert!(schema.properties.as_ref().unwrap().contains_key("name"));
let compaction =
round_trip::<MessagesCompaction>(json!({"type":"summarize","instructions":"briefly"}));
assert_eq!(compaction.instructions.as_deref(), Some("briefly"));
assert!(compaction.extra.is_empty());
let container = round_trip::<MessagesContainer>(json!({
"id":"container_1",
"expires_at":"2026-01-01T00:00:00Z",
"skills":[{"type":"custom","skill_id":"skill_1","version":"1"}]
}));
assert_eq!(container.id.as_deref(), Some("container_1"));
assert!(container.extra.is_empty());
let [skill] = container.skills.as_ref().unwrap().as_slice() else {
panic!("expected container skill");
};
assert_eq!(skill.skill_type, SkillType::Custom);
assert_eq!(skill.skill_id, "skill_1");
assert_eq!(skill.version.as_deref(), Some("1"));
assert!(skill.extra.is_empty());
let reference = round_trip::<ContainerReference>(json!({"id":"container_1"}));
let ContainerReference::Parameters(parameters) = reference else {
panic!("expected container parameters");
};
assert_eq!(parameters.id.as_deref(), container.id.as_deref());
assert!(parameters.skills.is_none());
round_trip::<ContainerReference>(
json!({"id":"container_1","skills":[{"type":"anthropic","skill_id":"pptx"}]}),
);
let server = round_trip::<McpServer>(json!({
"type":"url",
"url":"https://example.test/mcp",
"name":"search",
"authorization_token":"token",
"tool_configuration":{"allowed_tools":["search"],"enabled":true}
}));
assert_eq!(server.url, "https://example.test/mcp");
assert_eq!(server.name, "search");
assert_eq!(server.authorization_token.as_deref(), Some("token"));
assert!(server.extra.is_empty());
let configuration = server.tool_configuration.as_ref().unwrap();
assert_eq!(configuration.enabled, Some(true));
assert_eq!(
configuration.allowed_tools.as_deref(),
Some([String::from("search")].as_slice())
);
assert!(configuration.extra.is_empty());
let context = round_trip::<ContextManagementResponse>(json!({
"applied_edits":[
{"type":"clear_tool_uses_20250919","cleared_input_tokens":7,"cleared_tool_uses":2},
{"type":"clear_thinking_20251015","cleared_input_tokens":3,"cleared_thinking_turns":1}
]
}));
let [tool_uses, thinking] = context.applied_edits.as_ref().unwrap().as_slice() else {
panic!("expected two applied edits");
};
assert_eq!(
tool_uses.edit_type.as_deref(),
Some("clear_tool_uses_20250919")
);
assert_eq!(
(tool_uses.cleared_input_tokens, tool_uses.cleared_tool_uses),
(Some(7), Some(2))
);
assert!(tool_uses.cleared_thinking_turns.is_none());
assert_eq!(
(
thinking.cleared_input_tokens,
thinking.cleared_thinking_turns
),
(Some(3), Some(1))
);
assert!(tool_uses.extra.is_empty() && thinking.extra.is_empty());
let safeguard = round_trip::<Safeguard>(
json!({"type":"classifier","classifier_context":{"source":"test"}}),
);
assert_eq!(safeguard.safeguard_type, "classifier");
assert_eq!(
safeguard.classifier_context.as_ref().unwrap().get("source"),
Some(&json!("test"))
);
assert!(safeguard.extra.is_empty());
}
#[rstest]
fn stop_details_expose_refusal_and_keep_unknown_types() {
let refusal = round_trip::<StopDetails>(
json!({"type":"refusal","category":"cyber","explanation":"blocked"}),
);
assert_eq!(
refusal.detail_type,
Recognized::Known(StopDetailsType::Refusal)
);
assert_eq!(refusal.category.as_deref(), Some("cyber"));
assert_eq!(refusal.explanation.as_deref(), Some("blocked"));
assert!(refusal.extra.is_empty());
let safeguard =
round_trip::<StopDetails>(json!({"type":"safeguard","safeguard_types":["classifier"]}));
assert_eq!(
safeguard.detail_type,
Recognized::Unrecognized(json!("safeguard"))
);
assert_eq!(safeguard.extra["safeguard_types"], json!(["classifier"]));
}
#[rstest]
fn container_rejects_skill_without_id() {
assert!(
serde_json::from_value::<MessagesContainer>(
json!({"skills":[{"type":"custom","version":"1"}]})
)
.is_err()
);
}
#[rstest]
#[case::without_url(json!({"type":"url","name":"search"}))]
#[case::without_name(json!({"type":"url","url":"https://example.test/mcp"}))]
fn mcp_server_rejects_missing_required_fields(#[case] wire: Value) {
assert!(serde_json::from_value::<McpServer>(wire).is_err());
}
#[rstest]
fn output_format_rejects_missing_schema() {
assert!(serde_json::from_value::<OutputFormat>(json!({"type":"json_schema"})).is_err());
}
#[rstest]
#[case::user(json!("user"))]
#[case::assistant(json!("assistant"))]
#[case::system(json!("system"))]
#[case::future(json!("future_role"))]
fn message_roles_round_trip(#[case] wire: Value) {
round_trip::<MessageRole>(wire);
}
#[rstest]
#[case::message(json!("message"))]
#[case::future(json!("future_type"))]
fn message_types_round_trip(#[case] wire: Value) {
round_trip::<MessageType>(wire);
}
#[rstest]
#[case::end_turn(json!("end_turn"))]
#[case::refusal(json!("refusal"))]
#[case::compaction(json!("compaction"))]
#[case::future(json!("future_reason"))]
fn stop_reasons_round_trip(#[case] wire: Value) {
round_trip::<StopReason>(wire);
}
#[rstest]
#[case::model(json!({"type":"model_changed","cache_missed_input_tokens":12}), Some(12))]
#[case::system(json!({"type":"system_changed","cache_missed_input_tokens":3}), Some(3))]
#[case::tools(json!({"type":"tools_changed","cache_missed_input_tokens":0}), Some(0))]
#[case::messages(json!({"type":"messages_changed","cache_missed_input_tokens":9}), Some(9))]
#[case::not_found(json!({"type":"previous_message_not_found"}), None)]
#[case::unavailable(json!({"type":"unavailable"}), None)]
fn diagnostics_expose_cache_miss_reason(#[case] reason: Value, #[case] missed: Option<u64>) {
let diagnostics = round_trip::<MessagesDiagnostics>(json!({"cache_miss_reason":reason}));
let Some(Recognized::Known(reason)) = &diagnostics.cache_miss_reason else {
panic!("expected known cache miss reason");
};
let actual = match reason {
CacheMissReason::ModelChanged(tokens)
| CacheMissReason::SystemChanged(tokens)
| CacheMissReason::ToolsChanged(tokens)
| CacheMissReason::MessagesChanged(tokens) => {
assert!(tokens.extra.is_empty());
Some(tokens.cache_missed_input_tokens)
}
CacheMissReason::PreviousMessageNotFound { extra }
| CacheMissReason::Unavailable { extra } => {
assert!(extra.is_empty());
None
}
};
assert_eq!(actual, missed);
assert!(diagnostics.extra.is_empty());
}
#[rstest]
#[case::model(json!({"type":"model_changed","cache_missed_input_tokens":12}), "model_changed")]
#[case::system(json!({"type":"system_changed","cache_missed_input_tokens":12}), "system_changed")]
#[case::tools(json!({"type":"tools_changed","cache_missed_input_tokens":12}), "tools_changed")]
#[case::messages(json!({"type":"messages_changed","cache_missed_input_tokens":12}), "messages_changed")]
fn cache_miss_reason_variants_keep_their_tags(#[case] reason: Value, #[case] tag: &str) {
let parsed: CacheMissReason = serde_json::from_value(reason).unwrap();
assert_eq!(serde_json::to_value(parsed).unwrap()["type"], json!(tag));
}
#[rstest]
#[case::pending(json!({"cache_miss_reason":null}))]
#[case::future_reason(json!({"cache_miss_reason":{"type":"future_changed","cache_missed_input_tokens":1}}))]
#[case::missing_tokens(json!({"cache_miss_reason":{"type":"model_changed"}}))]
fn diagnostics_keep_pending_and_unmodeled_reasons(#[case] wire: Value) {
let diagnostics: MessagesDiagnostics = serde_json::from_value(wire.clone()).unwrap();
match &diagnostics.cache_miss_reason {
None => assert_eq!(serde_json::to_value(&diagnostics).unwrap(), json!({})),
Some(Recognized::Unrecognized(kept)) => {
assert_eq!(kept, &wire["cache_miss_reason"]);
assert_eq!(serde_json::to_value(&diagnostics).unwrap(), wire);
}
Some(Recognized::Known(reason)) => panic!("unexpected known reason {reason:?}"),
}
}
#[rstest]
#[case::previous(json!({"previous_message_id":"msg_1"}), Some(Some("msg_1")))]
#[case::first_turn(json!({"previous_message_id":null}), Some(None))]
#[case::absent(json!({}), None)]
fn diagnostics_param_distinguishes_null_from_absent_previous_message(
#[case] wire: Value,
#[case] expected: Option<Option<&str>>,
) {
let param = round_trip::<MessagesDiagnosticsParam>(wire);
assert_eq!(
param.previous_message_id.as_ref().map(Option::as_deref),
expected
);
assert!(param.extra.is_empty());
}
#[rstest]
fn diagnostics_param_rejects_non_string_previous_message() {
assert!(
serde_json::from_value::<MessagesDiagnosticsParam>(json!({"previous_message_id":7}))
.is_err()
);
}
#[rstest]
fn cache_miss_reason_rejects_non_numeric_tokens() {
assert!(
serde_json::from_value::<CacheMissReason>(
json!({"type":"model_changed","cache_missed_input_tokens":"many"})
)
.is_err()
);
}
}

52
litellm-rust/crates/llms-types/src/formats/messages/mod.rs
View file

@ -6,49 +6,9 @@ pub mod streaming;
mod tools;
mod usage;
pub use content::{
AdvisorToolResultContent, BashCodeExecutionOutput, BashCodeExecutionToolResultContent,
BlockContent, BrowserStateBlock, BrowserStateChange, BrowserTab, CharCitation, Citation,
CitationsConfig, CodeExecutionOutput, CodeExecutionResult, CodeExecutionToolResultContent,
CompactionBlock, ContainerUploadBlock, ContentBlockCitation, ContentSource, DocumentBlock,
EncryptedCodeExecutionResult, FallbackBlock, FallbackModel, FallbackTrigger, ImageBlock,
McpListedTool, McpToolListingBlock, McpToolResultBlock, McpToolResultContent,
McpToolResultText, McpToolUseBlock, MessagesContentPart, PageCitation, RedactedThinkingBlock,
SearchResultBlock, SearchResultCitation, ServerToolError, ServerToolResultBlock,
ServerToolUseBlock, TextBlock, TextEditorCodeExecutionToolResultContent,
TextEditorCreateResult, TextEditorFileType, TextEditorStrReplaceResult, TextEditorViewResult,
ThinkingBlock, ToolCaller, ToolChange, ToolChangeBlock, ToolChangeTarget, ToolReferenceBlock,
ToolResultBlock, ToolSearchReference, ToolSearchResult, ToolSearchToolResultContent,
ToolUseBlock, WebFetchDocument, WebFetchResult, WebFetchToolResultContent, WebSearchCitation,
WebSearchResult, WebSearchResultError, WebSearchResultErrorType, WebSearchResultType,
WebSearchToolResultContent,
};
pub use metadata::{
AppliedEdit, CacheMissReason, CacheMissedTokens, CompactionType, ContainerReference,
ContainerSkill, ContextManagementResponse, McpServer, McpServerType, McpToolConfiguration,
MessageRole, MessageType, MessagesCompaction, MessagesContainer, MessagesDiagnostics,
MessagesDiagnosticsParam, MessagesMetadata, OutputFormat, OutputFormatType, Safeguard,
SkillType, StopDetails, StopDetailsType, StopReason,
};
pub use request::{
AdaptiveThinking, CacheControl, ContentBlock, ContentBlockType, ContextEdit, ContextManagement,
ContextTrigger, DisabledThinking, EffortLevel, EnabledThinking, Message, MessageContent,
MessagesOptionalParams, MessagesRequest, MessagesTool, OutputConfig, Speed, SystemPrompt,
ThinkingConfig, ThinkingDisplay,
};
pub use response::MessagesResponse;
pub use tools::{
AdvisorTool, AdvisorToolName, AllowedCaller, BashToolName, BrowserToolsetConfigs,
BuiltinMessagesTool, ClientTool, CodeExecutionToolName, ComputerTool, ComputerTool20251124,
ComputerToolName, ComputerToolsetConfigs, CustomTool, CustomToolType, McpToolset,
MemoryToolName, MessagesToolParam, ResponseInclusion, ServerTool, StrReplaceBasedEditToolName,
StrReplaceEditorName, TextEditorTool20250728, ToolChoice, ToolChoiceType, ToolResultUrlSource,
ToolSearchBm25ToolName, ToolSearchRegexToolName, Toolset, ToolsetToolConfig,
UrlSourceToolReference, UserInputUrlSource, UserLocationType, WebFetchTool,
WebFetchTool20260309, WebFetchTool20260318, WebFetchToolName, WebFetchUrlSources,
WebSearchTool, WebSearchTool20260318, WebSearchToolName, WebSearchUserLocation,
};
pub use usage::{
CacheCreationUsage, MessagesOutputTokensDetails, MessagesUsage, ServerToolUsage,
UsageIteration, UsageIterationType,
};
pub use content::*;
pub use metadata::*;
pub use request::*;
pub use response::*;
pub use tools::*;
pub use usage::*;

131
litellm-rust/crates/llms-types/src/formats/messages/request.rs
View file

@ -387,10 +387,17 @@ impl Message {
#[cfg(test)]
mod tests {
use super::*;
use rstest::rstest;
use serde_json::json;
use super::*;
use crate::formats::messages::ContextTrigger;
use crate::formats::messages::{ContentBlock, ContentBlockType};
use crate::recognized::Recognized;
use serde_json::{Map, Value};
fn round_trip<T: serde::de::DeserializeOwned + serde::Serialize>(value: &Value) -> Value {
let parsed: T = serde_json::from_value(value.clone()).unwrap();
@ -718,4 +725,126 @@ mod tests {
json!(<&'static str>::from(level))
);
}
#[rstest]
#[case::null(json!(null))]
#[case::number(json!(1))]
#[case::boolean(json!(true))]
#[case::array(json!(["tool_use"]))]
#[case::object(json!({"type": "tool_use"}))]
fn content_block_type_rejects_non_string_json(#[case] value: Value) {
assert!(serde_json::from_value::<ContentBlockType>(value).is_err());
}
#[cfg(feature = "schema")]
#[rstest]
fn content_block_type_schema_remains_a_string() {
let schema = schemars::schema_for!(ContentBlockType).to_value();
assert_eq!(schema.get("type"), Some(&json!("string")));
assert_eq!(
schema.get("title"),
Some(&json!(stringify!(ContentBlockType)))
);
}
#[rstest]
#[case::text("text", ContentBlockType::Text)]
#[case::thinking("thinking", ContentBlockType::Thinking)]
#[case::redacted_thinking("redacted_thinking", ContentBlockType::RedactedThinking)]
#[case::tool_use("tool_use", ContentBlockType::ToolUse)]
#[case::server_tool_use("server_tool_use", ContentBlockType::ServerToolUse)]
#[case::tool_result("tool_result", ContentBlockType::ToolResult)]
#[case::compaction("compaction", ContentBlockType::Compaction)]
#[case::advisor_result("advisor_tool_result", ContentBlockType::AdvisorToolResult)]
#[case::web_search_result("web_search_tool_result", ContentBlockType::WebSearchToolResult)]
#[case::image("image", ContentBlockType::Other("image".into()))]
#[case::document("document", ContentBlockType::Other("document".into()))]
#[case::tool_addition("tool_addition", ContentBlockType::Other("tool_addition".into()))]
#[case::tool_removal("tool_removal", ContentBlockType::Other("tool_removal".into()))]
#[case::advisor("advisor_result", ContentBlockType::Other("advisor_result".into()))]
#[case::web_search_error(
"web_search_tool_result_error",
ContentBlockType::Other("web_search_tool_result_error".into())
)]
#[case::future_block("future_block", ContentBlockType::Other("future_block".into()))]
#[case::case_sensitive("Tool_Use", ContentBlockType::Other("Tool_Use".into()))]
#[case::empty("", ContentBlockType::Other(String::new()))]
fn block_type_is_typed_and_round_trips_with_extra_fields(
#[case] wire: &str,
#[case] expected: ContentBlockType,
) {
let input = json!({"type": wire, "future_field": {"nested": [1, null]}});
let block: ContentBlock = serde_json::from_value(input.clone()).unwrap();
assert_eq!(block.block_type.as_ref(), Some(&expected));
assert_eq!(serde_json::to_value(block).unwrap(), input);
}
#[rstest]
#[case::number(json!(1))]
#[case::boolean(json!(true))]
#[case::array(json!(["text"]))]
#[case::enum_object(json!({"text": null}))]
fn block_type_rejects_non_string_values(#[case] value: Value) {
assert!(serde_json::from_value::<ContentBlock>(json!({"type": value})).is_err());
}
#[rstest]
#[case::same_type(json!({"type": "tool_use"}), ContentBlockType::ToolUse, true)]
#[case::other_type(json!({"type": "tool_result"}), ContentBlockType::ToolUse, false)]
#[case::unknown_type(json!({"type": "future_tool"}), ContentBlockType::ToolUse, false)]
#[case::no_type(json!({"text": "x"}), ContentBlockType::Text, false)]
fn is_type_matches_the_exact_block_type(
#[case] block: Value,
#[case] block_type: ContentBlockType,
#[case] expected: bool,
) {
let block: ContentBlock = serde_json::from_value(block).unwrap();
assert_eq!(block.is_type(block_type), expected);
}
#[rstest]
#[case::input_tokens("input_tokens", false)]
#[case::tool_uses("tool_uses", true)]
fn context_trigger_exposes_typed_threshold(#[case] tag: &str, #[case] tool_uses: bool) {
let wire = json!({"type":tag,"value":1024,"extension":true});
let trigger: ContextTrigger = serde_json::from_value(wire.clone()).unwrap();
match &trigger {
ContextTrigger::InputTokens { value, extra } => {
assert!(!tool_uses);
assert_eq!(*value, 1024);
assert_eq!(extra.get("extension"), Some(&json!(true)));
}
ContextTrigger::ToolUses { value, extra } => {
assert!(tool_uses);
assert_eq!(*value, 1024);
assert_eq!(extra.get("extension"), Some(&json!(true)));
}
}
assert_eq!(serde_json::to_value(trigger).unwrap(), wire);
}
#[rstest]
#[case::negative(json!({"type":"input_tokens","value":-1}))]
#[case::wrong_shape(json!({"type":"input_tokens","value":"1024"}))]
#[case::missing_value(json!({"type":"input_tokens"}))]
#[case::null_value(json!({"type":"input_tokens","value":null}))]
#[case::tool_uses_negative(json!({"type":"tool_uses","value":-1}))]
#[case::tool_uses_fractional(json!({"type":"tool_uses","value":1.5}))]
#[case::tool_uses_missing_value(json!({"type":"tool_uses"}))]
#[case::missing_discriminator(json!({"value":1}))]
#[case::unknown_discriminator(json!({"type":"other","value":1}))]
fn token_threshold_requires_unsigned_integer(#[case] wire: Value) {
assert!(serde_json::from_value::<ContextTrigger>(wire).is_err());
}
#[rstest]
fn existing_content_blocks_preserve_opaque_nested_fields() {
let wire =
json!({"type":"tool_result","content":[{"type":"text","text":7}],"source":{"url":7}});
let block: crate::formats::messages::ContentBlock =
serde_json::from_value(wire.clone()).unwrap();
assert_eq!(block.content.as_ref(), wire.get("content"));
assert_eq!(block.extra.get("source"), wire.get("source"));
assert_eq!(serde_json::to_value(block).unwrap(), wire);
}
}

3
litellm-rust/crates/llms-types/src/formats/messages/response.rs
View file

@ -20,11 +20,10 @@ pub struct MessagesResponse {
#[cfg(test)]
mod tests {
use super::*;
use rstest::rstest;
use serde_json::json;
use super::*;
fn response(
stop_reason: Option<&str>,
stop_sequence: Option<&str>,

42
litellm-rust/crates/llms-types/src/formats/messages/streaming.rs
View file

@ -141,3 +141,45 @@ pub enum MessagesStreamEvent {
error: MessagesStreamError,
},
}
#[cfg(test)]
mod tests {
use crate::formats::messages::streaming::MessagesStreamEvent;
use rstest::rstest;
use serde_json::{Value, json};
#[rstest]
#[case::start(json!({
"type": "message_start",
"message": {
"id": "msg", "type": "message", "role": "assistant", "model": "test-model",
"content": [], "stop_reason": null, "stop_sequence": null,
"usage": {"input_tokens": 3, "future_usage": {"count": 9}},
"future_message": [1, 2]
}
}))]
#[case::block(json!({
"type": "content_block_start", "index": 0,
"content_block": {"type": "future", "payload": {"keep": true}}
}))]
#[case::delta(json!({
"type": "message_delta", "delta": {"stop_reason": "end_turn", "future_delta": 42},
"usage": {"output_tokens": 7, "future_usage": true}
}))]
#[case::error(json!({
"type": "error", "error": {"type": "future_error", "message": "failed", "future": 42}
}))]
fn stream_events_preserve_extensible_fields(#[case] wire: Value) {
let event: MessagesStreamEvent = serde_json::from_value(wire.clone()).unwrap();
assert_eq!(serde_json::to_value(event).unwrap(), wire);
}
#[rstest]
#[case::invalid_index(json!({"type": "content_block_stop", "index": "zero"}))]
#[case::missing_delta(json!({"type": "content_block_delta", "index": 0}))]
#[case::unknown_event(json!({"type": "future_event"}))]
fn malformed_or_unrecognized_events_remain_rejected(#[case] wire: Value) {
assert!(serde_json::from_value::<MessagesStreamEvent>(wire).is_err());
}
}

669
litellm-rust/crates/llms-types/src/formats/messages/tools.rs
View file

@ -562,3 +562,672 @@ pub enum MessagesToolParam {
Builtin(Box<BuiltinMessagesTool>),
Custom(Box<CustomTool>),
}
#[cfg(test)]
mod tests {
use crate::formats::messages::{
AdvisorTool, AdvisorToolName, CacheControl, UrlSourceToolReference,
};
use serde_json::Map;
use crate::formats::messages::AllowedCaller;
use crate::formats::messages::BashToolName;
use crate::formats::messages::{BrowserToolsetConfigs, BuiltinMessagesTool};
use crate::formats::messages::{CitationsConfig, ClientTool};
use crate::formats::messages::CodeExecutionToolName;
use crate::formats::messages::{
ComputerTool, ComputerTool20251124, ComputerToolName, ComputerToolsetConfigs,
};
use crate::formats::messages::{CustomTool, CustomToolType};
use crate::formats::messages::McpListedTool;
use crate::formats::messages::{McpToolset, MemoryToolName};
use crate::formats::messages::MessagesToolParam;
use crate::formats::messages::ResponseInclusion;
use crate::formats::messages::ServerTool;
use crate::formats::messages::{StrReplaceBasedEditToolName, StrReplaceEditorName};
use crate::formats::messages::TextEditorTool20250728;
use crate::formats::messages::{
ToolChoice, ToolChoiceType, ToolResultUrlSource, ToolSearchBm25ToolName,
};
use crate::formats::messages::ToolSearchRegexToolName;
use crate::formats::messages::{Toolset, ToolsetToolConfig};
use crate::formats::messages::{UserInputUrlSource, UserLocationType};
use crate::formats::messages::{
WebFetchTool, WebFetchTool20260309, WebFetchTool20260318, WebFetchToolName,
};
use crate::formats::messages::{
WebFetchUrlSources, WebSearchTool, WebSearchTool20260318, WebSearchToolName,
};
use crate::{
formats::messages::WebSearchUserLocation,
json_schema::{JsonSchema, JsonSchemaObject, JsonSchemaType},
};
use crate::test_support::*;
use indexmap::IndexMap;
use rstest::rstest;
use serde_json::{Value, json};
#[rstest]
fn tool_choice_round_trips() {
let choice = round_trip::<ToolChoice>(
json!({"type":"tool","name":"search","disable_parallel_tool_use":true}),
);
assert_eq!(choice.choice_type, ToolChoiceType::Tool);
assert_eq!(choice.name.as_deref(), Some("search"));
assert_eq!(choice.disable_parallel_tool_use, Some(true));
assert!(choice.extra.is_empty());
}
fn client_tool<N>(name: N) -> ClientTool<N> {
ClientTool {
name,
allowed_callers: None,
cache_control: None,
defer_loading: None,
input_examples: None,
strict: None,
extra: Map::new(),
}
}
fn server_tool<N>(name: N) -> ServerTool<N> {
ServerTool {
name,
allowed_callers: None,
cache_control: None,
defer_loading: None,
strict: None,
extra: Map::new(),
}
}
fn computer_tool(display_width_px: u64, display_height_px: u64) -> ComputerTool {
ComputerTool {
name: ComputerToolName::Computer,
display_width_px,
display_height_px,
display_number: None,
allowed_callers: None,
cache_control: None,
defer_loading: None,
input_examples: None,
strict: None,
extra: Map::new(),
}
}
fn web_search_tool() -> WebSearchTool {
WebSearchTool {
name: WebSearchToolName::WebSearch,
allowed_callers: None,
allowed_domains: None,
blocked_domains: None,
cache_control: None,
defer_loading: None,
max_uses: None,
strict: None,
user_location: None,
extra: Map::new(),
}
}
fn web_fetch_tool() -> WebFetchTool {
WebFetchTool {
name: WebFetchToolName::WebFetch,
allowed_callers: None,
allowed_domains: None,
blocked_domains: None,
cache_control: None,
citations: None,
defer_loading: None,
max_content_tokens: None,
max_uses: None,
strict: None,
url_sources: None,
extra: Map::new(),
}
}
fn web_fetch_url_sources() -> WebFetchUrlSources {
WebFetchUrlSources {
client_tool_results: None,
server_tool_results: None,
user_input: None,
extra: Map::new(),
}
}
fn toolset_config(enabled: Option<bool>, defer_loading: Option<bool>) -> ToolsetToolConfig {
ToolsetToolConfig {
defer_loading,
enabled,
extra: Map::new(),
}
}
fn browser_toolset_configs() -> BrowserToolsetConfigs {
BrowserToolsetConfigs {
type_text: None,
close_tab: None,
double_click: None,
file_upload: None,
find: None,
form_input: None,
get_page_text: None,
hold_key: None,
hover: None,
javascript_exec: None,
key: None,
left_click: None,
left_click_drag: None,
left_mouse_down: None,
left_mouse_up: None,
list_tabs: None,
middle_click: None,
mouse_move: None,
navigate: None,
new_tab: None,
read_console: None,
read_network: None,
read_page: None,
right_click: None,
screenshot: None,
scroll: None,
scroll_to: None,
switch_tab: None,
triple_click: None,
wait: None,
zoom: None,
extra: Map::new(),
}
}
fn computer_toolset_configs() -> ComputerToolsetConfigs {
ComputerToolsetConfigs {
type_text: None,
cursor_position: None,
double_click: None,
hold_key: None,
key: None,
left_click: None,
left_click_drag: None,
left_mouse_down: None,
left_mouse_up: None,
middle_click: None,
mouse_move: None,
right_click: None,
screenshot: None,
scroll: None,
triple_click: None,
wait: None,
zoom: None,
extra: Map::new(),
}
}
#[rstest]
#[case::bash_20241022(
json!({"type":"bash_20241022","name":"bash","input_examples":[{"command":"ls"}]}),
BuiltinMessagesTool::Bash20241022(ClientTool {
input_examples: Some(vec![Map::from_iter([(String::from("command"), json!("ls"))])]),
..client_tool(BashToolName::Bash)
})
)]
#[case::bash_20250124(
json!({"type":"bash_20250124","name":"bash","allowed_callers":["direct","code_execution_20260521"]}),
BuiltinMessagesTool::Bash20250124(ClientTool {
allowed_callers: Some(vec![AllowedCaller::Direct, AllowedCaller::CodeExecution20260521]),
..client_tool(BashToolName::Bash)
})
)]
#[case::text_editor_20241022(
json!({"type":"text_editor_20241022","name":"str_replace_editor","defer_loading":true}),
BuiltinMessagesTool::TextEditor20241022(ClientTool {
defer_loading: Some(true),
..client_tool(StrReplaceEditorName::StrReplaceEditor)
})
)]
#[case::text_editor_20250124(
json!({"type":"text_editor_20250124","name":"str_replace_editor","strict":true}),
BuiltinMessagesTool::TextEditor20250124(ClientTool {
strict: Some(true),
..client_tool(StrReplaceEditorName::StrReplaceEditor)
})
)]
#[case::text_editor_20250429(
json!({"type":"text_editor_20250429","name":"str_replace_based_edit_tool","cache_control":{"type":"ephemeral","ttl":"1h"}}),
BuiltinMessagesTool::TextEditor20250429(ClientTool {
cache_control: Some(cache_control("ephemeral", "1h")),
..client_tool(StrReplaceBasedEditToolName::StrReplaceBasedEditTool)
})
)]
#[case::text_editor_20250728(
json!({"type":"text_editor_20250728","name":"str_replace_based_edit_tool","max_characters":10000}),
BuiltinMessagesTool::TextEditor20250728(TextEditorTool20250728 {
name: StrReplaceBasedEditToolName::StrReplaceBasedEditTool,
allowed_callers: None,
cache_control: None,
defer_loading: None,
input_examples: None,
max_characters: Some(10000),
strict: None,
extra: Map::new(),
})
)]
#[case::memory_20250818(
json!({"type":"memory_20250818","name":"memory","allowed_callers":["code_execution_20250825"]}),
BuiltinMessagesTool::Memory20250818(ClientTool {
allowed_callers: Some(vec![AllowedCaller::CodeExecution20250825]),
..client_tool(MemoryToolName::Memory)
})
)]
#[case::computer_20241022(
json!({"type":"computer_20241022","name":"computer","display_width_px":1024,"display_height_px":768,"display_number":1}),
BuiltinMessagesTool::Computer20241022(ComputerTool {
display_number: Some(1),
..computer_tool(1024, 768)
})
)]
#[case::computer_20250124(
json!({"type":"computer_20250124","name":"computer","display_width_px":1280,"display_height_px":800}),
BuiltinMessagesTool::Computer20250124(computer_tool(1280, 800))
)]
#[case::computer_20251124(
json!({"type":"computer_20251124","name":"computer","display_width_px":1920,"display_height_px":1080,"enable_zoom":true}),
BuiltinMessagesTool::Computer20251124(ComputerTool20251124 {
name: ComputerToolName::Computer,
display_width_px: 1920,
display_height_px: 1080,
display_number: None,
enable_zoom: Some(true),
allowed_callers: None,
cache_control: None,
defer_loading: None,
input_examples: None,
strict: None,
extra: Map::new(),
})
)]
#[case::code_execution_20250522(
json!({"type":"code_execution_20250522","name":"code_execution","strict":false}),
BuiltinMessagesTool::CodeExecution20250522(ServerTool {
strict: Some(false),
..server_tool(CodeExecutionToolName::CodeExecution)
})
)]
#[case::code_execution_20250825(
json!({"type":"code_execution_20250825","name":"code_execution","defer_loading":false}),
BuiltinMessagesTool::CodeExecution20250825(ServerTool {
defer_loading: Some(false),
..server_tool(CodeExecutionToolName::CodeExecution)
})
)]
#[case::code_execution_20260120(
json!({"type":"code_execution_20260120","name":"code_execution","allowed_callers":["direct"]}),
BuiltinMessagesTool::CodeExecution20260120(ServerTool {
allowed_callers: Some(vec![AllowedCaller::Direct]),
..server_tool(CodeExecutionToolName::CodeExecution)
})
)]
#[case::code_execution_20260521(
json!({"type":"code_execution_20260521","name":"code_execution","allowed_callers":["code_execution_20260120"]}),
BuiltinMessagesTool::CodeExecution20260521(ServerTool {
allowed_callers: Some(vec![AllowedCaller::CodeExecution20260120]),
..server_tool(CodeExecutionToolName::CodeExecution)
})
)]
#[case::tool_search_regex_20251119(
json!({"type":"tool_search_tool_regex_20251119","name":"tool_search_tool_regex","defer_loading":true}),
BuiltinMessagesTool::ToolSearchRegex20251119(ServerTool {
defer_loading: Some(true),
..server_tool(ToolSearchRegexToolName::ToolSearchToolRegex)
})
)]
#[case::tool_search_regex(
json!({"type":"tool_search_tool_regex","name":"tool_search_tool_regex","strict":true}),
BuiltinMessagesTool::ToolSearchRegex(ServerTool {
strict: Some(true),
..server_tool(ToolSearchRegexToolName::ToolSearchToolRegex)
})
)]
#[case::tool_search_bm25_20251119(
json!({"type":"tool_search_tool_bm25_20251119","name":"tool_search_tool_bm25","defer_loading":false}),
BuiltinMessagesTool::ToolSearchBm2520251119(ServerTool {
defer_loading: Some(false),
..server_tool(ToolSearchBm25ToolName::ToolSearchToolBm25)
})
)]
#[case::tool_search_bm25(
json!({"type":"tool_search_tool_bm25","name":"tool_search_tool_bm25","strict":false}),
BuiltinMessagesTool::ToolSearchBm25(ServerTool {
strict: Some(false),
..server_tool(ToolSearchBm25ToolName::ToolSearchToolBm25)
})
)]
#[case::web_search_20250305(
json!({"type":"web_search_20250305","name":"web_search","max_uses":3,"allowed_domains":["example.test"],
"user_location":{"type":"approximate","city":"San Francisco","country":"US","timezone":"America/Los_Angeles"}}),
BuiltinMessagesTool::WebSearch20250305(WebSearchTool {
max_uses: Some(3),
allowed_domains: Some(vec![String::from("example.test")]),
user_location: Some(WebSearchUserLocation {
location_type: UserLocationType::Approximate,
city: Some(String::from("San Francisco")),
country: Some(String::from("US")),
region: None,
timezone: Some(String::from("America/Los_Angeles")),
extra: Map::new(),
}),
..web_search_tool()
})
)]
#[case::web_search_20260209(
json!({"type":"web_search_20260209","name":"web_search","blocked_domains":["blocked.test"]}),
BuiltinMessagesTool::WebSearch20260209(WebSearchTool {
blocked_domains: Some(vec![String::from("blocked.test")]),
..web_search_tool()
})
)]
#[case::web_search_20260318(
json!({"type":"web_search_20260318","name":"web_search","response_inclusion":"excluded"}),
BuiltinMessagesTool::WebSearch20260318(WebSearchTool20260318 {
name: WebSearchToolName::WebSearch,
allowed_callers: None,
allowed_domains: None,
blocked_domains: None,
cache_control: None,
defer_loading: None,
max_uses: None,
response_inclusion: Some(ResponseInclusion::Excluded),
strict: None,
user_location: None,
extra: Map::new(),
})
)]
#[case::web_fetch_20250910(
json!({"type":"web_fetch_20250910","name":"web_fetch","max_content_tokens":5000,"citations":{"enabled":true},
"url_sources":{"client_tool_results":{"type":"only","tools":[{"type":"tool_reference","name":"lookup"}]},
"server_tool_results":{"type":"all"},"user_input":{"type":"none"}}}),
BuiltinMessagesTool::WebFetch20250910(WebFetchTool {
max_content_tokens: Some(5000),
citations: Some(CitationsConfig {
enabled: Some(true),
extra: Map::new(),
}),
url_sources: Some(WebFetchUrlSources {
client_tool_results: Some(ToolResultUrlSource::Only {
tools: vec![tool_reference("lookup")],
extra: Map::new(),
}),
server_tool_results: Some(ToolResultUrlSource::All { extra: Map::new() }),
user_input: Some(UserInputUrlSource::None { extra: Map::new() }),
extra: Map::new(),
}),
..web_fetch_tool()
})
)]
#[case::web_fetch_20260209(
json!({"type":"web_fetch_20260209","name":"web_fetch","url_sources":{"server_tool_results":{"type":"except","tools":[{"type":"tool_reference","name":"web_search"}]}}}),
BuiltinMessagesTool::WebFetch20260209(WebFetchTool {
url_sources: Some(WebFetchUrlSources {
server_tool_results: Some(ToolResultUrlSource::Except {
tools: vec![tool_reference("web_search")],
extra: Map::new(),
}),
..web_fetch_url_sources()
}),
..web_fetch_tool()
})
)]
#[case::web_fetch_20260309(
json!({"type":"web_fetch_20260309","name":"web_fetch","use_cache":false}),
BuiltinMessagesTool::WebFetch20260309(WebFetchTool20260309 {
name: WebFetchToolName::WebFetch,
allowed_callers: None,
allowed_domains: None,
blocked_domains: None,
cache_control: None,
citations: None,
defer_loading: None,
max_content_tokens: None,
max_uses: None,
strict: None,
url_sources: None,
use_cache: Some(false),
extra: Map::new(),
})
)]
#[case::web_fetch_20260318(
json!({"type":"web_fetch_20260318","name":"web_fetch","use_cache":true,"response_inclusion":"full"}),
BuiltinMessagesTool::WebFetch20260318(WebFetchTool20260318 {
name: WebFetchToolName::WebFetch,
allowed_callers: None,
allowed_domains: None,
blocked_domains: None,
cache_control: None,
citations: None,
defer_loading: None,
max_content_tokens: None,
max_uses: None,
response_inclusion: Some(ResponseInclusion::Full),
strict: None,
url_sources: None,
use_cache: Some(true),
extra: Map::new(),
})
)]
#[case::advisor_20260301(
json!({"type":"advisor_20260301","name":"advisor","model":"claude-opus-5-5","max_tokens":2048,"caching":{"type":"ephemeral","ttl":"5m"}}),
BuiltinMessagesTool::Advisor20260301(AdvisorTool {
name: AdvisorToolName::Advisor,
model: String::from("claude-opus-5-5"),
allowed_callers: None,
cache_control: None,
caching: Some(cache_control("ephemeral", "5m")),
defer_loading: None,
max_tokens: Some(2048),
max_uses: None,
strict: None,
extra: Map::new(),
})
)]
#[case::browser_toolset_20260801(
json!({"type":"browser_toolset_20260801","configs":{"type":{"enabled":false},"javascript_exec":{"defer_loading":true}}}),
BuiltinMessagesTool::BrowserToolset20260801(Toolset {
cache_control: None,
configs: Some(Box::new(BrowserToolsetConfigs {
type_text: Some(toolset_config(Some(false), None)),
javascript_exec: Some(toolset_config(None, Some(true))),
..browser_toolset_configs()
})),
extra: Map::new(),
})
)]
#[case::computer_toolset_20260801(
json!({"type":"computer_toolset_20260801","configs":{"zoom":{"enabled":false},"cursor_position":{"enabled":true}}}),
BuiltinMessagesTool::ComputerToolset20260801(Toolset {
cache_control: None,
configs: Some(Box::new(ComputerToolsetConfigs {
zoom: Some(toolset_config(Some(false), None)),
cursor_position: Some(toolset_config(Some(true), None)),
..computer_toolset_configs()
})),
extra: Map::new(),
})
)]
#[case::mcp_toolset(
json!({"type":"mcp_toolset","mcp_server_name":"kb","default_config":{"enabled":false},
"configs":{"search":{"enabled":true,"defer_loading":true}},
"tools":[{"name":"search","input_schema":{"type":"object"},"description":"Search"}]}),
BuiltinMessagesTool::McpToolset(McpToolset {
mcp_server_name: String::from("kb"),
cache_control: None,
configs: Some(IndexMap::from([(String::from("search"), toolset_config(Some(true), Some(true)))])),
default_config: Some(toolset_config(Some(false), None)),
tools: Some(vec![McpListedTool {
name: String::from("search"),
description: Some(String::from("Search")),
input_schema: JsonSchema::Object(Box::new(JsonSchemaObject {
schema_type: Some(JsonSchemaType::Name(String::from("object"))),
..JsonSchemaObject::default()
})),
extra: Map::new(),
}]),
extra: Map::new(),
})
)]
fn builtin_tools_decode_typed_definitions(
#[case] wire: Value,
#[case] expected: BuiltinMessagesTool,
) {
assert_eq!(round_trip::<BuiltinMessagesTool>(wire), expected);
}
#[rstest]
fn builtin_tools_preserve_unknown_fields() {
let tool = round_trip::<BuiltinMessagesTool>(
json!({"type":"bash_20250124","name":"bash","extension":[1,null]}),
);
let BuiltinMessagesTool::Bash20250124(bash) = tool else {
panic!("expected bash tool");
};
assert_eq!(bash.extra.get("extension"), Some(&json!([1, null])));
}
#[rstest]
#[case::missing_discriminator(json!({"name":"bash"}))]
#[case::unknown_discriminator(json!({"type":"future_tool","name":"bash"}))]
#[case::null_discriminator(json!({"type":null,"name":"bash"}))]
#[case::bash_without_name(json!({"type":"bash_20250124"}))]
#[case::bash_wrong_name(json!({"type":"bash_20241022","name":"shell"}))]
#[case::legacy_editor_with_new_name(json!({"type":"text_editor_20250124","name":"str_replace_based_edit_tool"}))]
#[case::new_editor_with_legacy_name(json!({"type":"text_editor_20250728","name":"str_replace_editor"}))]
#[case::editor_20250429_with_legacy_name(json!({"type":"text_editor_20250429","name":"str_replace_editor"}))]
#[case::memory_without_name(json!({"type":"memory_20250818"}))]
#[case::computer_without_width(json!({"type":"computer_20250124","name":"computer","display_height_px":768}))]
#[case::computer_without_height(json!({"type":"computer_20241022","name":"computer","display_width_px":1024}))]
#[case::zoom_computer_without_display(json!({"type":"computer_20251124","name":"computer"}))]
#[case::code_execution_without_name(json!({"type":"code_execution_20250825"}))]
#[case::regex_search_with_bm25_name(json!({"type":"tool_search_tool_regex","name":"tool_search_tool_bm25"}))]
#[case::bm25_search_with_regex_name(json!({"type":"tool_search_tool_bm25_20251119","name":"tool_search_tool_regex"}))]
#[case::web_search_with_fetch_name(json!({"type":"web_search_20250305","name":"web_fetch"}))]
#[case::web_fetch_without_name(json!({"type":"web_fetch_20260318"}))]
#[case::advisor_without_model(json!({"type":"advisor_20260301","name":"advisor"}))]
#[case::advisor_without_name(json!({"type":"advisor_20260301","model":"claude-opus-5-5"}))]
#[case::mcp_toolset_without_server(json!({"type":"mcp_toolset"}))]
#[case::unknown_allowed_caller(json!({"type":"code_execution_20250825","name":"code_execution","allowed_callers":["code_execution_20250522"]}))]
#[case::unknown_response_inclusion(json!({"type":"web_search_20260318","name":"web_search","response_inclusion":"partial"}))]
#[case::user_input_only_filter(json!({"type":"web_fetch_20250910","name":"web_fetch","url_sources":{"user_input":{"type":"only","tools":[]}}}))]
#[case::negative_limit(json!({"type":"web_search_20250305","name":"web_search","max_uses":-1}))]
#[case::wrong_mcp_config(json!({"type":"mcp_toolset","mcp_server_name":"kb","configs":{"search":{"enabled":"yes"}}}))]
#[case::wrong_browser_config(json!({"type":"browser_toolset_20260801","configs":{"navigate":{"enabled":1}}}))]
fn builtin_tools_reject_malformed_fields(#[case] wire: Value) {
assert!(serde_json::from_value::<BuiltinMessagesTool>(wire.clone()).is_err());
assert!(serde_json::from_value::<MessagesToolParam>(wire).is_err());
}
#[rstest]
fn custom_tool_exposes_schema_and_preserves_extensions() {
let tool = round_trip::<CustomTool>(json!({
"type":"custom",
"name":"lookup",
"input_schema":{"type":"object","properties":{"query":{"type":"string"}}},
"strict":false,
"defer_loading":true,
"allowed_callers":["direct","code_execution_20260120"],
"extension":{"nested":[1,null]}
}));
assert_eq!(
tool.allowed_callers.as_deref(),
Some([AllowedCaller::Direct, AllowedCaller::CodeExecution20260120].as_slice())
);
assert_eq!(tool.tool_type, Some(CustomToolType::Custom));
assert_eq!(tool.name, "lookup");
assert_eq!((tool.strict, tool.defer_loading), (Some(false), Some(true)));
let JsonSchema::Object(schema) = &tool.input_schema else {
panic!("expected an object schema");
};
assert!(schema.properties.as_ref().unwrap().contains_key("query"));
assert_eq!(tool.extra["extension"], json!({"nested":[1,null]}));
}
#[rstest]
fn custom_tool_omits_null_discriminator() {
let tool: CustomTool =
serde_json::from_value(json!({"type":null,"name":"lookup","input_schema":true}))
.unwrap();
assert!(tool.tool_type.is_none());
assert!(tool.description.is_none());
assert_eq!(
serde_json::to_value(tool).unwrap(),
json!({"name":"lookup","input_schema":true})
);
}
#[rstest]
#[case::missing_name(json!({"input_schema":{}}))]
#[case::missing_schema(json!({"name":"lookup"}))]
#[case::wrong_name_shape(json!({"name":7,"input_schema":{}}))]
#[case::builtin_tool(json!({"name":"lookup","input_schema":{},"type":"bash_20250124"}))]
#[case::unknown_discriminator(json!({"name":"lookup","input_schema":{},"type":"future_tool"}))]
#[case::unknown_allowed_caller(json!({"name":"lookup","input_schema":{},"allowed_callers":["anyone"]}))]
fn custom_tool_rejects_invalid_shapes(#[case] wire: Value) {
assert!(serde_json::from_value::<CustomTool>(wire).is_err());
}
#[rstest]
#[case::builtin(json!({"type":"web_search_20250305","name":"web_search","max_uses":2}), true)]
#[case::builtin_toolset(json!({"type":"mcp_toolset","mcp_server_name":"kb"}), true)]
#[case::custom(json!({"name":"lookup","input_schema":{"type":"object"}}), false)]
fn tool_params_dispatch_on_discriminator(#[case] wire: Value, #[case] builtin: bool) {
let tool = round_trip::<MessagesToolParam>(wire);
assert_eq!(matches!(tool, MessagesToolParam::Builtin(_)), builtin);
}
#[rstest]
fn tool_params_reject_unknown_tool_types() {
assert!(
serde_json::from_value::<MessagesToolParam>(
json!({"type":"future_tool","name":"lookup","input_schema":{}})
)
.is_err()
);
}
fn tool_reference(name: &str) -> UrlSourceToolReference {
UrlSourceToolReference::ToolReference {
name: String::from(name),
extra: Map::new(),
}
}
fn cache_control(cache_type: &str, ttl: &str) -> CacheControl {
CacheControl {
cache_type: Some(String::from(cache_type)),
ttl: Some(String::from(ttl)),
..CacheControl::default()
}
}
}

119
litellm-rust/crates/llms-types/src/formats/messages/usage.rs
View file

@ -73,3 +73,122 @@ pub struct MessagesOutputTokensDetails {
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[cfg(test)]
mod tests {
use crate::formats::messages::MessagesUsage;
use crate::formats::messages::UsageIterationType;
use crate::{recognized::Recognized, test_support::*};
use rstest::rstest;
use serde_json::json;
#[rstest]
fn usage_contracts_round_trip() {
let usage = round_trip::<MessagesUsage>(json!({
"input_tokens":10,
"output_tokens":4,
"server_tool_use":{"web_search_requests":2,"web_fetch_requests":1},
"cache_creation":{"ephemeral_1h_input_tokens":3,"ephemeral_5m_input_tokens":1},
"output_tokens_details":{"thinking_tokens":2},
"iterations":[
{"type":"compaction","input_tokens":7,"output_tokens":1},
{"type":"message","input_tokens":3,"output_tokens":3}
],
"service_tier":"priority",
"inference_geo":"global",
"speed":"fast"
}));
assert_eq!(usage.inference_geo.as_deref(), Some("global"));
assert_eq!(usage.input_tokens, Some(10));
assert_eq!(usage.output_tokens, Some(4));
assert!(usage.extra.is_empty());
let server = usage.server_tool_use.as_ref().unwrap();
assert_eq!(server.web_search_requests, Some(2));
assert_eq!(server.web_fetch_requests, Some(1));
let cache = usage.cache_creation.as_ref().unwrap();
assert_eq!(cache.ephemeral_1h_input_tokens, Some(3));
assert_eq!(cache.ephemeral_5m_input_tokens, Some(1));
assert_eq!(
usage
.output_tokens_details
.as_ref()
.unwrap()
.thinking_tokens,
Some(2)
);
let [compaction, message] = usage.iterations.as_ref().unwrap().as_slice() else {
panic!("expected usage iterations");
};
assert_eq!(
compaction.iteration_type,
Recognized::Known(UsageIterationType::Compaction)
);
assert_eq!(
message.iteration_type,
Recognized::Known(UsageIterationType::Message)
);
assert_eq!(
compaction.input_tokens.unwrap() + message.input_tokens.unwrap(),
usage.input_tokens.unwrap()
);
assert_eq!(
compaction.output_tokens.unwrap() + message.output_tokens.unwrap(),
usage.output_tokens.unwrap()
);
}
#[rstest]
fn usage_iterations_expose_advisor_and_fallback_models() {
let usage = round_trip::<MessagesUsage>(json!({
"iterations":[
{"type":"advisor_message","model":"claude-opus-5-5","input_tokens":5,"output_tokens":2,
"cache_creation_input_tokens":4,"cache_read_input_tokens":1,
"cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":4}},
{"type":"fallback_message","model":"claude-sonnet-5-5","input_tokens":6,"output_tokens":3,
"cache_creation_input_tokens":0,"cache_read_input_tokens":2},
{"type":"future_iteration","input_tokens":1}
]
}));
let [advisor, fallback, future] = usage.iterations.as_deref().unwrap() else {
panic!("expected three iterations");
};
assert_eq!(
advisor.iteration_type,
Recognized::Known(UsageIterationType::AdvisorMessage)
);
assert_eq!(advisor.model.as_deref(), Some("claude-opus-5-5"));
assert_eq!(
(
advisor.cache_creation_input_tokens,
advisor.cache_read_input_tokens
),
(Some(4), Some(1))
);
assert_eq!(
advisor
.cache_creation
.as_ref()
.unwrap()
.ephemeral_5m_input_tokens,
advisor.cache_creation_input_tokens
);
assert!(advisor.extra.is_empty());
assert_eq!(
fallback.iteration_type,
Recognized::Known(UsageIterationType::FallbackMessage)
);
assert_eq!(fallback.model.as_deref(), Some("claude-sonnet-5-5"));
assert_eq!(fallback.cache_read_input_tokens, Some(2));
assert!(fallback.extra.is_empty());
assert_eq!(
future.iteration_type,
Recognized::Unrecognized(json!("future_iteration"))
);
assert_eq!(future.input_tokens, Some(1));
}
}

3
litellm-rust/crates/llms-types/src/formats/mod.rs
View file

@ -1,6 +1,3 @@
//! Wire types for each provider API format (OpenAI Chat Completions,
//! Anthropic Messages, Responses, ...), one submodule per format.
pub mod audio_transcription;
pub mod batches;
pub mod chat_completions;

164
litellm-rust/crates/llms-types/src/formats/ocr.rs
View file

@ -1,164 +0,0 @@
use std::collections::BTreeMap;
use serde_json::{Map, Value};
use serde_with::serde_as;
use crate::serde_compat::{FiniteF64, LaxI64};
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type")]
pub enum OcrDocument {
#[serde(rename = "document_url")]
DocumentUrl {
document_url: String,
#[serde(flatten)]
extra_fields: BTreeMap<String, Option<String>>,
},
#[serde(rename = "image_url")]
ImageUrl {
image_url: String,
#[serde(flatten)]
extra_fields: BTreeMap<String, Option<String>>,
},
}
impl OcrDocument {
pub fn source(&self) -> &str {
match self {
Self::DocumentUrl { document_url, .. } => document_url,
Self::ImageUrl { image_url, .. } => image_url,
}
}
pub fn is_remote(&self) -> bool {
let source = self.source();
source.starts_with("http://") || source.starts_with("https://")
}
pub fn with_source(self, source: String) -> Self {
match self {
Self::DocumentUrl { extra_fields, .. } => Self::DocumentUrl {
document_url: source,
extra_fields,
},
Self::ImageUrl { extra_fields, .. } => Self::ImageUrl {
image_url: source,
extra_fields,
},
}
}
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Default, Eq)]
#[serde(rename_all = "lowercase")]
pub enum OcrResponseFormat {
#[default]
Litellm,
Native,
}
#[serde_as]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct OcrPageDimensions {
#[serde_as(deserialize_as = "Option<LaxI64>")]
pub dpi: Option<i64>,
#[serde_as(deserialize_as = "Option<LaxI64>")]
pub height: Option<i64>,
#[serde_as(deserialize_as = "Option<LaxI64>")]
pub width: Option<i64>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct OcrPageImage {
pub image_base64: Option<String>,
pub bbox: Option<Map<String, Value>>,
#[serde(flatten)]
pub extra_fields: Map<String, Value>,
}
#[serde_as]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct OcrPage {
#[serde_as(deserialize_as = "LaxI64")]
pub index: i64,
pub markdown: String,
pub images: Option<Vec<OcrPageImage>>,
pub dimensions: Option<OcrPageDimensions>,
#[serde(flatten)]
pub extra_fields: Map<String, Value>,
}
#[serde_as]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct OcrUsageInfo {
#[serde_as(deserialize_as = "Option<LaxI64>")]
pub pages_processed: Option<i64>,
#[serde_as(deserialize_as = "Option<LaxI64>")]
pub pages_processed_annotation: Option<i64>,
#[serde_as(deserialize_as = "Option<FiniteF64>")]
pub credits: Option<f64>,
#[serde_as(deserialize_as = "Option<LaxI64>")]
pub doc_size_bytes: Option<i64>,
#[serde(flatten)]
pub extra_fields: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct LiteLLMOcrResponse {
pub pages: Vec<OcrPage>,
pub model: String,
pub document_annotation: Option<Value>,
pub usage_info: Option<OcrUsageInfo>,
pub content: Option<String>,
pub tables: Option<Vec<Map<String, Value>>>,
#[serde(rename = "keyValuePairs")]
pub key_value_pairs: Option<Vec<Map<String, Value>>>,
#[serde(default = "ocr_object")]
pub object: String,
#[serde(flatten)]
pub extra_fields: Map<String, Value>,
#[serde(skip_serializing_if = "Option::is_none")]
pub provider_native_response: Option<Map<String, Value>>,
}
impl LiteLLMOcrResponse {
pub fn new(model: impl Into<String>, pages: Vec<OcrPage>) -> Self {
Self {
pages,
model: model.into(),
document_annotation: None,
usage_info: None,
content: None,
tables: None,
key_value_pairs: None,
object: ocr_object(),
extra_fields: Map::new(),
provider_native_response: None,
}
}
pub fn into_json(self) -> Value {
serde_json::to_value(self).expect("OCR response fields are JSON-compatible")
}
}
fn ocr_object() -> String {
"ocr".into()
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct OcrBoundingBox {
pub top_left_x: Option<serde_json::Number>,
pub top_left_y: Option<serde_json::Number>,
pub bottom_right_x: Option<serde_json::Number>,
pub bottom_right_y: Option<serde_json::Number>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}

12
litellm-rust/crates/llms-types/src/formats/ocr/AGENTS.md Normal file
View file

@ -0,0 +1,12 @@
# rules
- LiteLLM's OCR contract follows the Mistral OCR response shape. It is the normalized output every OCR provider returns, not a Mistral type
- Mistral, and Mistral models hosted on Azure AI and Vertex AI, speak it natively
- Cohere, Reducto, AWS Textract, Azure Document Intelligence and DeepSeek OCR on Vertex translate their native output into it in `llms/src/<provider>/ocr/`
- `OcrBoundingBox` is the corner coordinates providers copy into the normalized `bbox`
- Normalized `tables` and `keyValuePairs` stay open because providers pass through different native shapes
- `BaseOcrConfig`, OCR errors and inline-document helpers live in `llms/src/base_llm/ocr/`
# references
- https://docs.mistral.ai/api/endpoint/ocr

5
litellm-rust/crates/llms-types/src/formats/ocr/mod.rs Normal file
View file

@ -0,0 +1,5 @@
mod request;
mod response;
pub use request::*;
pub use response::*;

95
litellm-rust/crates/llms-types/src/formats/ocr/request.rs Normal file
View file

@ -0,0 +1,95 @@
use std::collections::BTreeMap;
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type")]
pub enum OcrDocument {
#[serde(rename = "document_url")]
DocumentUrl {
document_url: String,
#[serde(flatten)]
extra_fields: BTreeMap<String, Option<String>>,
},
#[serde(rename = "image_url")]
ImageUrl {
image_url: String,
#[serde(flatten)]
extra_fields: BTreeMap<String, Option<String>>,
},
}
impl OcrDocument {
pub fn source(&self) -> &str {
match self {
Self::DocumentUrl { document_url, .. } => document_url,
Self::ImageUrl { image_url, .. } => image_url,
}
}
pub fn is_remote(&self) -> bool {
let source = self.source();
source.starts_with("http://") || source.starts_with("https://")
}
pub fn with_source(self, source: String) -> Self {
match self {
Self::DocumentUrl { extra_fields, .. } => Self::DocumentUrl {
document_url: source,
extra_fields,
},
Self::ImageUrl { extra_fields, .. } => Self::ImageUrl {
image_url: source,
extra_fields,
},
}
}
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Default, Eq)]
#[serde(rename_all = "lowercase")]
pub enum OcrResponseFormat {
#[default]
Litellm,
Native,
}
#[cfg(test)]
mod tests {
use crate::formats::ocr::OcrDocument;
use rstest::rstest;
use serde_json::{Value, json};
#[rstest]
fn document_rejects_non_string_provider_fields() {
assert!(
serde_json::from_value::<OcrDocument>(json!({
"type": "image_url", "image_url": "https://example.com/image", "detail": 42
}))
.is_err()
);
}
#[rstest]
#[case::document_url("document_url", "document_name", "application/pdf")]
#[case::image_url("image_url", "detail", "image/png")]
fn document_variants_preserve_provider_fields_when_rewriting_sources(
#[case] kind: &str,
#[case] field: &str,
#[case] mime_type: &str,
#[values(json!("kept"), Value::Null)] extra: Value,
) {
let original = "https://example.com/input";
let replacement = format!("data:{mime_type};base64,AA==");
let document: OcrDocument =
serde_json::from_value(json!({"type": kind, kind: original, field: extra})).unwrap();
assert_eq!(document.source(), original);
assert!(document.is_remote());
let rewritten = document.with_source(replacement.clone());
assert!(!rewritten.is_remote());
assert_eq!(
serde_json::to_value(rewritten).unwrap(),
json!({"type": kind, kind: replacement, field: extra})
);
}
}

224
litellm-rust/crates/llms-types/src/formats/ocr/response.rs Normal file
View file

@ -0,0 +1,224 @@
use serde_json::{Map, Value};
use serde_with::serde_as;
use crate::serde_compat::{FiniteF64, LaxI64};
#[serde_as]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct OcrPageDimensions {
#[serde_as(deserialize_as = "Option<LaxI64>")]
pub dpi: Option<i64>,
#[serde_as(deserialize_as = "Option<LaxI64>")]
pub height: Option<i64>,
#[serde_as(deserialize_as = "Option<LaxI64>")]
pub width: Option<i64>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct OcrPageImage {
pub image_base64: Option<String>,
pub bbox: Option<Map<String, Value>>,
#[serde(flatten)]
pub extra_fields: Map<String, Value>,
}
#[serde_as]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct OcrPage {
#[serde_as(deserialize_as = "LaxI64")]
pub index: i64,
pub markdown: String,
pub images: Option<Vec<OcrPageImage>>,
pub dimensions: Option<OcrPageDimensions>,
#[serde(flatten)]
pub extra_fields: Map<String, Value>,
}
#[serde_as]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct OcrUsageInfo {
#[serde_as(deserialize_as = "Option<LaxI64>")]
pub pages_processed: Option<i64>,
#[serde_as(deserialize_as = "Option<LaxI64>")]
pub pages_processed_annotation: Option<i64>,
#[serde_as(deserialize_as = "Option<FiniteF64>")]
pub credits: Option<f64>,
#[serde_as(deserialize_as = "Option<LaxI64>")]
pub doc_size_bytes: Option<i64>,
#[serde(flatten)]
pub extra_fields: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct LiteLLMOcrResponse {
pub pages: Vec<OcrPage>,
pub model: String,
pub document_annotation: Option<Value>,
pub usage_info: Option<OcrUsageInfo>,
pub content: Option<String>,
pub tables: Option<Vec<Map<String, Value>>>,
#[serde(rename = "keyValuePairs")]
pub key_value_pairs: Option<Vec<Map<String, Value>>>,
#[serde(default = "ocr_object")]
pub object: String,
#[serde(flatten)]
pub extra_fields: Map<String, Value>,
#[serde(skip_serializing_if = "Option::is_none")]
pub provider_native_response: Option<Map<String, Value>>,
}
impl LiteLLMOcrResponse {
pub fn new(model: impl Into<String>, pages: Vec<OcrPage>) -> Self {
Self {
pages,
model: model.into(),
document_annotation: None,
usage_info: None,
content: None,
tables: None,
key_value_pairs: None,
object: ocr_object(),
extra_fields: Map::new(),
provider_native_response: None,
}
}
pub fn into_json(self) -> Value {
serde_json::to_value(self).expect("OCR response fields are JSON-compatible")
}
}
fn ocr_object() -> String {
"ocr".into()
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct OcrBoundingBox {
pub top_left_x: Option<serde_json::Number>,
pub top_left_y: Option<serde_json::Number>,
pub bottom_right_x: Option<serde_json::Number>,
pub bottom_right_y: Option<serde_json::Number>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[cfg(test)]
mod tests {
use crate::formats::ocr::{LiteLLMOcrResponse, OcrBoundingBox, OcrPage};
use rstest::rstest;
use serde_json::{Map, Value, json};
#[rstest]
#[case::missing_page_fields(json!({"pages": [{}]}))]
#[case::invalid_markdown(json!({"pages": [{"index": 0, "markdown": false}]}))]
#[case::invalid_image_bounds(json!({"pages": [{"index": 0, "markdown": "", "images": [{"bbox": []}]}]}))]
#[case::fractional_page_count(json!({"usage_info": {"pages_processed": 1.5}}))]
#[case::invalid_table(json!({"tables": [false]}))]
#[case::invalid_key_value_pair(json!({"keyValuePairs": [[]]}))]
#[case::invalid_native_response(json!({"provider_native_response": []}))]
fn normalized_response_rejects_invalid_shared_fields(#[case] fields: Value) {
let payload: Map<String, Value> = json!({"model": "model", "pages": []})
.as_object()
.unwrap()
.iter()
.chain(fields.as_object().unwrap())
.map(|(key, value)| (key.clone(), value.clone()))
.collect();
assert!(serde_json::from_value::<LiteLLMOcrResponse>(Value::Object(payload)).is_err());
}
#[rstest]
#[case::large_integer(json!("9007199254740993.0"), 9_007_199_254_740_993)]
#[case::signed_decimal(json!("+2.000"), 2)]
#[case::separator(json!("1_000"), 1000)]
#[case::boolean(json!(true), 1)]
#[case::integral_float(json!(2.0), 2)]
fn numeric_coercion_preserves_integer_precision(#[case] value: Value, #[case] expected: i64) {
let page: OcrPage =
serde_json::from_value(json!({"index": value, "markdown": ""})).unwrap();
assert_eq!(page.index, expected);
assert_eq!(
serde_json::to_value(page).unwrap()["index"],
json!(expected)
);
}
#[rstest]
#[case::exponent(json!("1e2"))]
#[case::missing_integer(json!(".0"))]
#[case::missing_fraction(json!("2."))]
#[case::leading_separator(json!("_2"))]
#[case::repeated_separator(json!("2__0"))]
#[case::fractional_float(json!(2.5))]
#[case::null(json!(null))]
fn page_index_rejects_invalid_integers(#[case] value: Value) {
assert!(
serde_json::from_value::<OcrPage>(json!({"index": value, "markdown": ""})).is_err()
);
}
#[rstest]
#[case::absent_native(None)]
#[case::present_native(Some(Map::from_iter([("native".into(), json!({"nested": [null, 1]}))])))]
fn response_serialization_preserves_extensions_and_native_presence(
#[case] native: Option<Map<String, Value>>,
) {
let response = LiteLLMOcrResponse {
extra_fields: Map::from_iter([("provider_field".into(), json!("kept"))]),
provider_native_response: native.clone(),
..LiteLLMOcrResponse::new("model", vec![])
};
let serialized = response.into_json();
assert_eq!(serialized["provider_field"], "kept");
assert_eq!(
serialized.get("provider_native_response").cloned(),
native.clone().map(Value::Object)
);
let decoded: LiteLLMOcrResponse = serde_json::from_value(serialized.clone()).unwrap();
assert_eq!(decoded.provider_native_response, native);
assert_eq!(decoded.into_json(), serialized);
}
#[rstest]
fn bounding_box_exposes_corner_coordinates_and_keeps_extensions() {
let wire = json!({"top_left_x":1,"top_left_y":2.5,"bottom_right_x":30,"bottom_right_y":40,"future":true});
let bounds: OcrBoundingBox = serde_json::from_value(wire.clone()).unwrap();
assert_eq!(bounds.top_left_x, Some(1.into()));
assert_eq!(
bounds
.top_left_y
.as_ref()
.and_then(serde_json::Number::as_f64),
Some(2.5)
);
assert_eq!(bounds.bottom_right_x, Some(30.into()));
assert_eq!(bounds.bottom_right_y, Some(40.into()));
assert_eq!(Value::Object(bounds.extra.clone()), json!({"future":true}));
assert_eq!(serde_json::to_value(bounds).unwrap(), wire);
}
#[rstest]
fn partial_bounding_box_omits_null_corners() {
let bounds: OcrBoundingBox =
serde_json::from_value(json!({"top_left_x":null,"bottom_right_y":4})).unwrap();
assert!(bounds.top_left_x.is_none());
assert_eq!(
serde_json::to_value(bounds).unwrap(),
json!({"bottom_right_y":4})
);
}
#[rstest]
#[case::string_corner(json!({"top_left_x":"1"}))]
#[case::array_corner(json!({"bottom_right_y":[4]}))]
fn bounding_box_rejects_non_numeric_corners(#[case] wire: Value) {
assert!(serde_json::from_value::<OcrBoundingBox>(wire).is_err());
}
}

15
litellm-rust/crates/llms-types/src/formats/responses/AGENTS.md Normal file
View file

@ -0,0 +1,15 @@
# rules
- Responses originated at OpenAI and is now offered by other hosts and gateways. Types here describe the format, not OpenAI's deployment of it
- Served over HTTP and over the WebSocket mode. `ResponsesWsEvent` is a wire event and lives here
- `ResponsesWsTransformResult` wraps a provider transformation's output, so it lives in `llms/src/base_llm/responses/`
- Hosts that accept only part of the format, or add output item types, are handled in `llms/src/<provider>/responses/`. Keep unknown output items passing through instead of rejecting them
- Responses-to-Chat emulation and model-specific parameter rewriting are transformations, not format rules
# references
- https://developers.openai.com/api/reference/resources/responses
- https://developers.openai.com/api/reference/resources/responses/streaming-events
- https://github.com/openai/openai-python/blob/main/src/openai/types/responses/response.py
- https://github.com/openai/openai-python/blob/main/src/openai/types/responses/response_output_item.py
- https://github.com/openai/openai-python/blob/main/src/openai/types/responses/mcp_tool_call_error.py

2
litellm-rust/crates/llms-types/src/formats/responses/mod.rs
View file

@ -3,4 +3,4 @@ mod response;
pub mod streaming_websocket;
pub use output::*;
pub use response::ResponsesApiResponse;
pub use response::*;

581
litellm-rust/crates/llms-types/src/formats/responses/output.rs
View file

@ -829,3 +829,584 @@ pub struct ResponsesMcpApprovalResponse {
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[cfg(test)]
mod tests {
use crate::formats::{
chat_completions::{PromptCacheBreakpoint, PromptCacheMode},
responses::{
ResponsesAdditionalTools, ResponsesApplyPatchCall, ResponsesApplyPatchCallOutput,
ResponsesApplyPatchOperation, ResponsesCodeOutput, ResponsesCompaction,
ResponsesComputerAction, ResponsesComputerCall, ResponsesComputerCallOutput,
ResponsesComputerOutput, ResponsesContentPart, ResponsesCoordinate,
ResponsesCustomToolCall, ResponsesCustomToolCallOutput, ResponsesFunctionCall,
ResponsesFunctionCallOutput, ResponsesImageGenerationCall, ResponsesInputContent,
ResponsesLocalShellAction, ResponsesLocalShellCall, ResponsesLocalShellCallOutput,
ResponsesMcpApprovalRequest, ResponsesMcpApprovalResponse, ResponsesMcpError,
ResponsesMcpErrorDetail, ResponsesOutputItem, ResponsesProgram, ResponsesProgramOutput,
ResponsesSafetyCheck, ResponsesShellAction, ResponsesShellCall,
ResponsesShellCallOutput, ResponsesShellEnvironment, ResponsesShellOutcome,
ResponsesShellOutputChunk, ResponsesToolCaller, ResponsesToolOutput,
ResponsesToolSearchCall, ResponsesToolSearchOutput, ResponsesWebSearchAction,
},
};
use rstest::rstest;
use serde_json::{Value, json};
#[rstest]
#[case::message(json!({"type":"message","role":"assistant","content":[{"type":"output_text","text":"answer","annotations":[{"type":"url_citation","url":"https://example.test","start_index":0,"end_index":6}]}]}))]
#[case::function_call(json!({"type":"function_call","call_id":"call_1","name":"lookup","arguments":"{\"query\":\"q\"}"}))]
#[case::custom_tool(json!({"type":"custom_tool_call","name":"lookup","input":"q"}))]
#[case::reasoning(json!({"type":"reasoning","summary":[{"type":"summary_text","text":"summary"}],"encrypted_content":"opaque"}))]
#[case::web_search(json!({"type":"web_search_call","action":{"type":"search","queries":["q"],"sources":[{"type":"url","url":"https://example.test"}]}}))]
#[case::file_search(json!({"type":"file_search_call","queries":["q"],"results":[{"file_id":"file_1","score":1,"attributes":{"custom":[1,null]}}]}))]
#[case::code(json!({"type":"code_interpreter_call","outputs":[{"type":"logs","logs":"done"},{"type":"image","url":"https://example.test"}]}))]
#[case::image(json!({"type":"image_generation_call","result":"generated"}))]
#[case::mcp(json!({"type":"mcp_call","server_label":"server","name":"lookup","arguments":"{}","output":"done"}))]
fn output_items_round_trip(#[case] wire: Value) {
let item: ResponsesOutputItem = serde_json::from_value(wire.clone()).unwrap();
match &item {
ResponsesOutputItem::Message(message) => {
assert_eq!(message.role.as_deref(), Some("assistant"));
let Some(content) = &message.content else {
panic!("expected content")
};
let [
ResponsesContentPart::OutputText {
text,
annotations: Some(annotations),
..
},
] = content.as_slice()
else {
panic!("expected output text and annotations")
};
assert_eq!(text, "answer");
let [crate::formats::responses::ResponsesAnnotation::UrlCitation(citation)] =
annotations.as_slice()
else {
panic!("expected URL citation")
};
assert_eq!(citation.url.as_deref(), Some("https://example.test"));
assert_eq!(citation.start_index, Some(0));
}
ResponsesOutputItem::FunctionCall(call) => {
assert_eq!(call.call_id.as_deref(), Some("call_1"));
assert_eq!(call.name.as_deref(), Some("lookup"));
assert_eq!(call.arguments.as_deref(), Some("{\"query\":\"q\"}"));
}
ResponsesOutputItem::CustomToolCall(call) => {
assert_eq!(call.name.as_deref(), Some("lookup"));
assert_eq!(call.input.as_deref(), Some("q"));
}
ResponsesOutputItem::Reasoning(reasoning) => {
assert_eq!(reasoning.encrypted_content.as_deref(), Some("opaque"));
let Some(summary) = &reasoning.summary else {
panic!("expected summary")
};
let [ResponsesContentPart::SummaryText { text, .. }] = summary.as_slice() else {
panic!("expected summary text")
};
assert_eq!(text, "summary");
}
ResponsesOutputItem::WebSearchCall(call) => {
let Some(ResponsesWebSearchAction::Search {
queries: Some(queries),
sources: Some(sources),
..
}) = &call.action
else {
panic!("expected search action")
};
assert_eq!(queries, &["q"]);
assert_eq!(sources[0].url.as_deref(), Some("https://example.test"));
}
ResponsesOutputItem::FileSearchCall(call) => {
assert_eq!(call.queries.as_deref(), Some(["q".to_owned()].as_slice()));
let Some(results) = &call.results else {
panic!("expected search results")
};
assert_eq!(results[0].file_id.as_deref(), Some("file_1"));
assert_eq!(results[0].score, Some(1.into()));
assert_eq!(
results[0].attributes.as_ref().unwrap()["custom"],
json!([1, null])
);
}
ResponsesOutputItem::CodeInterpreterCall(call) => {
let Some(outputs) = &call.outputs else {
panic!("expected code outputs")
};
let [
ResponsesCodeOutput::Logs { logs, .. },
ResponsesCodeOutput::Image { url, .. },
] = outputs.as_slice()
else {
panic!("expected logs and image")
};
assert_eq!(logs, "done");
assert_eq!(url, "https://example.test");
}
ResponsesOutputItem::ImageGenerationCall(call) => {
assert_eq!(call.result.as_deref(), Some("generated"))
}
ResponsesOutputItem::McpCall(call) => {
assert_eq!(call.server_label.as_deref(), Some("server"));
assert_eq!(call.name.as_deref(), Some("lookup"));
assert_eq!(call.arguments.as_deref(), Some("{}"));
assert_eq!(call.output.as_deref(), Some("done"));
}
other => panic!("unexpected variant {other:?}"),
}
assert_eq!(serde_json::to_value(item).unwrap(), wire);
}
#[rstest]
#[case::find_in_page(
json!({"type":"find_in_page","url":"https://example.test","pattern":"needle"}),
json!({"type":"find_in_page","url":"https://example.test","pattern":"needle"})
)]
#[case::legacy_find(
json!({"type":"find","url":"https://example.test","pattern":"needle"}),
json!({"type":"find_in_page","url":"https://example.test","pattern":"needle"})
)]
fn web_search_find_action_exposes_url_and_pattern(
#[case] wire: Value,
#[case] serialized: Value,
) {
let action: ResponsesWebSearchAction = serde_json::from_value(wire).unwrap();
let ResponsesWebSearchAction::FindInPage {
url,
pattern,
extra,
} = &action
else {
panic!("expected find_in_page action");
};
assert_eq!(url, "https://example.test");
assert_eq!(pattern, "needle");
assert!(extra.is_empty());
assert_eq!(serde_json::to_value(action).unwrap(), serialized);
}
#[rstest]
#[case::with_url(json!({"type":"open_page","url":"https://example.test"}), Some("https://example.test"))]
#[case::without_url(json!({"type":"open_page"}), None)]
fn web_search_open_page_url_is_optional(#[case] wire: Value, #[case] expected: Option<&str>) {
let action: ResponsesWebSearchAction = serde_json::from_value(wire.clone()).unwrap();
let ResponsesWebSearchAction::OpenPage { url, .. } = &action else {
panic!("expected open_page action");
};
assert_eq!(url.as_deref(), expected);
assert_eq!(serde_json::to_value(action).unwrap(), wire);
}
#[rstest]
#[case::find_pattern(json!({"type":"find_in_page","url":"u","pattern":false}))]
#[case::find_missing_url(json!({"type":"find_in_page","pattern":"p"}))]
#[case::open_page_url(json!({"type":"open_page","url":7}))]
fn web_search_action_rejects_malformed_fields(#[case] wire: Value) {
assert!(serde_json::from_value::<ResponsesWebSearchAction>(wire).is_err());
}
#[rstest]
#[case::refusal(json!({"type":"refusal","refusal":"refused","future":null}), ResponsesContentPart::Refusal { refusal:"refused".into(), extra:serde_json::Map::from_iter([("future".into(), Value::Null)]) })]
#[case::reasoning(json!({"type":"reasoning_text","text":"reason"}), ResponsesContentPart::ReasoningText { text:"reason".into(), extra:Default::default() })]
fn content_parts_expose_typed_variants(
#[case] wire: Value,
#[case] expected: ResponsesContentPart,
) {
let content: ResponsesContentPart = serde_json::from_value(wire.clone()).unwrap();
assert_eq!(content, expected);
assert_eq!(serde_json::to_value(content).unwrap(), wire);
}
#[rstest]
fn mcp_discovery_exposes_tools_and_nested_schemas() {
let wire = json!({
"type":"mcp_list_tools","id":"item_1","server_label":"tools",
"tools":[{"name":"lookup","description":"Look up a value","input_schema":{"type":"object","properties":{"query":{"type":"string"}}},"annotations":{"read_only":false},"future":null}],
"extension":[1,null]
});
let item: ResponsesOutputItem = serde_json::from_value(wire.clone()).unwrap();
let ResponsesOutputItem::McpListTools(discovery) = &item else {
panic!("expected discovery")
};
assert_eq!(discovery.server_label.as_deref(), Some("tools"));
let tool = &discovery.tools.as_ref().unwrap()[0];
assert_eq!(tool.name.as_deref(), Some("lookup"));
let Some(crate::json_schema::JsonSchema::Object(schema)) = &tool.input_schema else {
panic!("expected tool schema")
};
assert!(schema.properties.as_ref().unwrap().contains_key("query"));
assert_eq!(tool.annotations, Some(json!({"read_only":false})));
assert_eq!(tool.extra["future"], Value::Null);
assert_eq!(serde_json::to_value(item).unwrap(), wire);
}
#[rstest]
#[case::partial(json!({"server_label":"tools","tools":[]}))]
#[case::failure(json!({"server_label":"tools","error":"unavailable"}))]
fn mcp_discovery_accepts_partial_payloads(#[case] wire: Value) {
let discovery: crate::formats::responses::ResponsesMcpListTools =
serde_json::from_value(wire.clone()).unwrap();
assert_eq!(discovery.server_label.as_deref(), Some("tools"));
assert!(discovery.id.is_none());
assert_eq!(serde_json::to_value(discovery).unwrap(), wire);
}
#[rstest]
#[case::message(json!("unavailable"), ResponsesMcpError::Message("unavailable".into()))]
#[case::protocol(json!({"type":"mcp_protocol_error","code":-32600,"message":"invalid","extension":null}), ResponsesMcpError::Detail(ResponsesMcpErrorDetail::McpProtocolError {code:-32600,message:"invalid".into(),extra:serde_json::Map::from_iter([("extension".into(), Value::Null)])}))]
#[case::http(json!({"type":"http_error","code":503,"message":"unavailable"}), ResponsesMcpError::Detail(ResponsesMcpErrorDetail::HttpError {code:503,message:"unavailable".into(),extra:Default::default()}))]
#[case::tool(json!({"type":"mcp_tool_execution_error","content":{"arbitrary":[1,null]}}), ResponsesMcpError::Detail(ResponsesMcpErrorDetail::McpToolExecutionError {content:json!({"arbitrary":[1,null]}),extra:Default::default()}))]
fn mcp_call_errors_expose_typed_variants(
#[case] wire: Value,
#[case] expected: ResponsesMcpError,
) {
let error: ResponsesMcpError = serde_json::from_value(wire.clone()).unwrap();
assert_eq!(error, expected);
assert_eq!(serde_json::to_value(&error).unwrap(), wire);
let call: ResponsesOutputItem =
serde_json::from_value(json!({"type":"mcp_call","error":wire})).unwrap();
let ResponsesOutputItem::McpCall(call) = call else {
panic!("expected call")
};
assert_eq!(call.error, Some(error));
}
#[rstest]
#[case::tools_not_array(json!({"type":"mcp_list_tools","tools":{}}))]
#[case::invalid_name(json!({"type":"mcp_list_tools","tools":[{"name":7}]}))]
#[case::invalid_schema(json!({"type":"mcp_list_tools","tools":[{"input_schema":{"properties":{"query":7}}}]}))]
#[case::invalid_error_code(json!({"type":"mcp_call","error":{"type":"http_error","code":"503","message":"unavailable"}}))]
#[case::missing_error_content(json!({"type":"mcp_call","error":{"type":"mcp_tool_execution_error"}}))]
#[case::unknown_error_tag(json!({"type":"mcp_call","error":{"type":"future"}}))]
fn mcp_output_rejects_malformed_known_fields(#[case] wire: Value) {
assert!(serde_json::from_value::<ResponsesOutputItem>(wire).is_err());
}
fn program_caller() -> Option<ResponsesToolCaller> {
Some(ResponsesToolCaller::Program {
caller_id: "prog_1".into(),
extra: Default::default(),
})
}
fn direct_caller() -> Option<ResponsesToolCaller> {
Some(ResponsesToolCaller::Direct {
extra: Default::default(),
})
}
fn safety_check() -> ResponsesSafetyCheck {
ResponsesSafetyCheck {
id: text("sc_1"),
code: text("malicious_instructions"),
message: text("check"),
..Default::default()
}
}
#[rstest]
#[case::function_call(
json!({"type":"function_call","id":"fc_1","status":"completed","call_id":"call_1","name":"lookup","arguments":"{}","namespace":"ns","async":true,"caller":{"type":"program","caller_id":"prog_1"}}),
ResponsesOutputItem::FunctionCall(ResponsesFunctionCall {
id: text("fc_1"), status: text("completed"), call_id: text("call_1"), name: text("lookup"),
arguments: text("{}"), namespace: text("ns"), r#async: Some(true), caller: program_caller(), ..Default::default()
})
)]
#[case::custom_tool_call(
json!({"type":"custom_tool_call","call_id":"call_1","name":"lookup","input":"q","namespace":"ns","async":false,"caller":{"type":"direct"}}),
ResponsesOutputItem::CustomToolCall(ResponsesCustomToolCall {
call_id: text("call_1"), name: text("lookup"), input: text("q"), namespace: text("ns"),
r#async: Some(false), caller: direct_caller(), ..Default::default()
})
)]
#[case::image_generation_call(
json!({"type":"image_generation_call","id":"ig_1","status":"completed","result":"b64","action":"edit","background":"opaque","output_format":"webp","quality":"high","revised_prompt":"a cat","size":"1536x864"}),
ResponsesOutputItem::ImageGenerationCall(ResponsesImageGenerationCall {
id: text("ig_1"), status: text("completed"), result: text("b64"), action: text("edit"), background: text("opaque"),
output_format: text("webp"), quality: text("high"), revised_prompt: text("a cat"), size: text("1536x864"), ..Default::default()
})
)]
#[case::function_call_output_text(
json!({"type":"function_call_output","id":"fco_1","status":"completed","call_id":"call_1","output":"done","caller":{"type":"direct"},"created_by":"user_1","name":"lookup","namespace":"ns"}),
ResponsesOutputItem::FunctionCallOutput(ResponsesFunctionCallOutput {
id: text("fco_1"), status: text("completed"), call_id: text("call_1"), output: Some(ResponsesToolOutput::Text("done".into())),
caller: direct_caller(), created_by: text("user_1"), name: text("lookup"), namespace: text("ns"), ..Default::default()
})
)]
#[case::function_call_output_content(
json!({"type":"function_call_output","call_id":"call_1","output":[
{"type":"input_text","text":"t","prompt_cache_breakpoint":{"mode":"explicit"}},
{"type":"input_image","detail":"low","file_id":"file_1","image_url":"https://example.test/i.png"},
{"type":"input_file","detail":"high","file_data":"data","file_id":"file_2","file_url":"https://example.test/f","filename":"f.pdf"}
]}),
ResponsesOutputItem::FunctionCallOutput(ResponsesFunctionCallOutput {
call_id: text("call_1"),
output: Some(ResponsesToolOutput::Content(vec![
ResponsesInputContent::InputText {
text: "t".into(),
prompt_cache_breakpoint: Some(PromptCacheBreakpoint { mode: PromptCacheMode::Explicit, extra: Default::default() }),
extra: Default::default(),
},
ResponsesInputContent::InputImage {
detail: "low".into(), file_id: text("file_1"), image_url: text("https://example.test/i.png"),
prompt_cache_breakpoint: None, extra: Default::default(),
},
ResponsesInputContent::InputFile {
detail: text("high"), file_data: text("data"), file_id: text("file_2"), file_url: text("https://example.test/f"),
filename: text("f.pdf"), prompt_cache_breakpoint: None, extra: Default::default(),
},
])),
..Default::default()
})
)]
#[case::custom_tool_call_output(
json!({"type":"custom_tool_call_output","id":"cto_1","status":"completed","call_id":"call_1","output":[{"type":"input_text","text":"t"}],"caller":{"type":"program","caller_id":"prog_1"},"created_by":"user_1"}),
ResponsesOutputItem::CustomToolCallOutput(ResponsesCustomToolCallOutput {
id: text("cto_1"), status: text("completed"), call_id: text("call_1"),
output: Some(ResponsesToolOutput::Content(vec![ResponsesInputContent::InputText {
text: "t".into(), prompt_cache_breakpoint: None, extra: Default::default(),
}])),
caller: program_caller(), created_by: text("user_1"), ..Default::default()
})
)]
#[case::computer_call(
json!({"type":"computer_call","id":"cu_1","status":"completed","call_id":"call_1",
"pending_safety_checks":[{"id":"sc_1","code":"malicious_instructions","message":"check"}],
"action":{"type":"click","button":"left","x":1,"y":2,"keys":["shift"]},
"actions":[
{"type":"double_click","x":3,"y":4},
{"type":"drag","path":[{"x":5,"y":6},{"x":7,"y":8}],"keys":["ctrl"]},
{"type":"keypress","keys":["enter"]},
{"type":"move","x":-1,"y":9},
{"type":"screenshot"},
{"type":"scroll","scroll_x":0,"scroll_y":-10,"x":11,"y":12},
{"type":"type","text":"hello"},
{"type":"wait"}
]}),
ResponsesOutputItem::ComputerCall(ResponsesComputerCall {
id: text("cu_1"), status: text("completed"), call_id: text("call_1"), pending_safety_checks: Some(vec![safety_check()]),
action: Some(ResponsesComputerAction::Click { button: "left".into(), x: 1, y: 2, keys: Some(vec!["shift".into()]), extra: Default::default() }),
actions: Some(vec![
ResponsesComputerAction::DoubleClick { x: 3, y: 4, keys: None, extra: Default::default() },
ResponsesComputerAction::Drag {
path: vec![
ResponsesCoordinate { x: 5, y: 6, extra: Default::default() },
ResponsesCoordinate { x: 7, y: 8, extra: Default::default() },
],
keys: Some(vec!["ctrl".into()]),
extra: Default::default(),
},
ResponsesComputerAction::Keypress { keys: vec!["enter".into()], extra: Default::default() },
ResponsesComputerAction::Move { x: -1, y: 9, keys: None, extra: Default::default() },
ResponsesComputerAction::Screenshot { extra: Default::default() },
ResponsesComputerAction::Scroll { scroll_x: 0, scroll_y: -10, x: 11, y: 12, keys: None, extra: Default::default() },
ResponsesComputerAction::Type { text: "hello".into(), extra: Default::default() },
ResponsesComputerAction::Wait { extra: Default::default() },
]),
..Default::default()
})
)]
#[case::computer_call_output(
json!({"type":"computer_call_output","id":"cuo_1","status":"completed","call_id":"call_1","output":{"type":"computer_screenshot","file_id":"file_1","image_url":"https://example.test/s.png"},"acknowledged_safety_checks":[{"id":"sc_1","code":"malicious_instructions","message":"check"}],"created_by":"user_1"}),
ResponsesOutputItem::ComputerCallOutput(ResponsesComputerCallOutput {
id: text("cuo_1"), status: text("completed"), call_id: text("call_1"),
output: Some(ResponsesComputerOutput::ComputerScreenshot { file_id: text("file_1"), image_url: text("https://example.test/s.png"), extra: Default::default() }),
acknowledged_safety_checks: Some(vec![safety_check()]), created_by: text("user_1"), ..Default::default()
})
)]
#[case::program(
json!({"type":"program","id":"prog_item","call_id":"prog_1","code":"run()","fingerprint":"fp"}),
ResponsesOutputItem::Program(ResponsesProgram {
id: text("prog_item"), call_id: text("prog_1"), code: text("run()"), fingerprint: text("fp"), ..Default::default()
})
)]
#[case::program_output(
json!({"type":"program_output","id":"po_1","status":"completed","call_id":"prog_1","result":"42"}),
ResponsesOutputItem::ProgramOutput(ResponsesProgramOutput {
id: text("po_1"), status: text("completed"), call_id: text("prog_1"), result: text("42"), ..Default::default()
})
)]
#[case::tool_search_call(
json!({"type":"tool_search_call","id":"ts_1","status":"completed","call_id":"call_1","arguments":{"query":["weather",null]},"execution":"server","created_by":"user_1"}),
ResponsesOutputItem::ToolSearchCall(ResponsesToolSearchCall {
id: text("ts_1"), status: text("completed"), call_id: text("call_1"), arguments: Some(json!({"query":["weather",null]})),
execution: text("server"), created_by: text("user_1"), ..Default::default()
})
)]
#[case::tool_search_output(
json!({"type":"tool_search_output","id":"tso_1","status":"completed","call_id":"call_1","execution":"client","tools":[{"type":"function","name":"lookup","parameters":null,"strict":true}],"created_by":"user_1"}),
ResponsesOutputItem::ToolSearchOutput(ResponsesToolSearchOutput {
id: text("tso_1"), status: text("completed"), call_id: text("call_1"), execution: text("client"),
tools: Some(vec![serde_json::Map::from_iter([
("type".into(), json!("function")), ("name".into(), json!("lookup")),
("parameters".into(), Value::Null), ("strict".into(), json!(true)),
])]),
created_by: text("user_1"), ..Default::default()
})
)]
#[case::additional_tools(
json!({"type":"additional_tools","id":"at_1","role":"developer","tools":[{"type":"local_shell"}]}),
ResponsesOutputItem::AdditionalTools(ResponsesAdditionalTools {
id: text("at_1"), role: text("developer"),
tools: Some(vec![serde_json::Map::from_iter([("type".into(), json!("local_shell"))])]), ..Default::default()
})
)]
#[case::compaction(
json!({"type":"compaction","id":"cmp_1","encrypted_content":"opaque","created_by":"user_1"}),
ResponsesOutputItem::Compaction(ResponsesCompaction {
id: text("cmp_1"), encrypted_content: text("opaque"), created_by: text("user_1"), ..Default::default()
})
)]
#[case::local_shell_call(
json!({"type":"local_shell_call","id":"ls_1","status":"completed","call_id":"call_1","action":{"type":"exec","command":["ls","-a"],"env":{"HOME":"/home/u"},"timeout_ms":1000,"user":"u","working_directory":"/tmp"}}),
ResponsesOutputItem::LocalShellCall(ResponsesLocalShellCall {
id: text("ls_1"), status: text("completed"), call_id: text("call_1"),
action: Some(ResponsesLocalShellAction::Exec {
command: vec!["ls".into(), "-a".into()], env: [("HOME".into(), "/home/u".into())].into(),
timeout_ms: Some(1000), user: text("u"), working_directory: text("/tmp"), extra: Default::default(),
}),
..Default::default()
})
)]
#[case::local_shell_call_output(
json!({"type":"local_shell_call_output","id":"lso_1","status":"completed","output":"{\"stdout\":\"x\"}"}),
ResponsesOutputItem::LocalShellCallOutput(ResponsesLocalShellCallOutput {
id: text("lso_1"), status: text("completed"), output: text("{\"stdout\":\"x\"}"), ..Default::default()
})
)]
#[case::shell_call_local(
json!({"type":"shell_call","id":"sh_1","status":"in_progress","call_id":"call_1","action":{"commands":["ls"],"max_output_length":100,"timeout_ms":500},"environment":{"type":"local"},"caller":{"type":"direct"},"created_by":"user_1"}),
ResponsesOutputItem::ShellCall(ResponsesShellCall {
id: text("sh_1"), status: text("in_progress"), call_id: text("call_1"),
action: Some(ResponsesShellAction { commands: Some(vec!["ls".into()]), max_output_length: Some(100), timeout_ms: Some(500), ..Default::default() }),
environment: Some(ResponsesShellEnvironment::Local { extra: Default::default() }),
caller: direct_caller(), created_by: text("user_1"), ..Default::default()
})
)]
#[case::shell_call_container(
json!({"type":"shell_call","call_id":"call_1","environment":{"type":"container_reference","container_id":"cntr_1"}}),
ResponsesOutputItem::ShellCall(ResponsesShellCall {
call_id: text("call_1"),
environment: Some(ResponsesShellEnvironment::ContainerReference { container_id: "cntr_1".into(), extra: Default::default() }),
..Default::default()
})
)]
#[case::shell_call_output(
json!({"type":"shell_call_output","id":"sho_1","status":"completed","call_id":"call_1","max_output_length":100,"output":[
{"outcome":{"type":"exit","exit_code":2},"stdout":"out","stderr":"err","created_by":"user_1"},
{"outcome":{"type":"timeout"},"stdout":"","stderr":""}
],"caller":{"type":"program","caller_id":"prog_1"},"created_by":"user_1"}),
ResponsesOutputItem::ShellCallOutput(ResponsesShellCallOutput {
id: text("sho_1"), status: text("completed"), call_id: text("call_1"), max_output_length: Some(100),
output: Some(vec![
ResponsesShellOutputChunk {
outcome: Some(ResponsesShellOutcome::Exit { exit_code: 2, extra: Default::default() }),
stdout: text("out"), stderr: text("err"), created_by: text("user_1"), ..Default::default()
},
ResponsesShellOutputChunk {
outcome: Some(ResponsesShellOutcome::Timeout { extra: Default::default() }),
stdout: text(""), stderr: text(""), ..Default::default()
},
]),
caller: program_caller(), created_by: text("user_1"), ..Default::default()
})
)]
#[case::apply_patch_create(
json!({"type":"apply_patch_call","id":"ap_1","status":"completed","call_id":"call_1","operation":{"type":"create_file","path":"a.txt","diff":"+a"},"caller":{"type":"direct"},"created_by":"user_1"}),
ResponsesOutputItem::ApplyPatchCall(ResponsesApplyPatchCall {
id: text("ap_1"), status: text("completed"), call_id: text("call_1"),
operation: Some(ResponsesApplyPatchOperation::CreateFile { path: "a.txt".into(), diff: "+a".into(), extra: Default::default() }),
caller: direct_caller(), created_by: text("user_1"), ..Default::default()
})
)]
#[case::apply_patch_delete(
json!({"type":"apply_patch_call","call_id":"call_1","operation":{"type":"delete_file","path":"a.txt"}}),
ResponsesOutputItem::ApplyPatchCall(ResponsesApplyPatchCall {
call_id: text("call_1"),
operation: Some(ResponsesApplyPatchOperation::DeleteFile { path: "a.txt".into(), extra: Default::default() }),
..Default::default()
})
)]
#[case::apply_patch_update(
json!({"type":"apply_patch_call","call_id":"call_1","operation":{"type":"update_file","path":"a.txt","diff":"-a\n+b"}}),
ResponsesOutputItem::ApplyPatchCall(ResponsesApplyPatchCall {
call_id: text("call_1"),
operation: Some(ResponsesApplyPatchOperation::UpdateFile { path: "a.txt".into(), diff: "-a\n+b".into(), extra: Default::default() }),
..Default::default()
})
)]
#[case::apply_patch_call_output(
json!({"type":"apply_patch_call_output","id":"apo_1","status":"failed","call_id":"call_1","output":"conflict","caller":{"type":"direct"},"created_by":"user_1"}),
ResponsesOutputItem::ApplyPatchCallOutput(ResponsesApplyPatchCallOutput {
id: text("apo_1"), status: text("failed"), call_id: text("call_1"), output: text("conflict"),
caller: direct_caller(), created_by: text("user_1"), ..Default::default()
})
)]
#[case::mcp_approval_request(
json!({"type":"mcp_approval_request","id":"apr_1","server_label":"s","name":"n","arguments":"{}"}),
ResponsesOutputItem::McpApprovalRequest(ResponsesMcpApprovalRequest {
id: text("apr_1"), server_label: text("s"), name: text("n"), arguments: text("{}"), ..Default::default()
})
)]
#[case::mcp_approval_response(
json!({"type":"mcp_approval_response","id":"aprr_1","approval_request_id":"apr_1","approve":false,"reason":"denied"}),
ResponsesOutputItem::McpApprovalResponse(ResponsesMcpApprovalResponse {
id: text("aprr_1"), approval_request_id: text("apr_1"), approve: Some(false), reason: text("denied"), ..Default::default()
})
)]
fn documented_output_items_parse_into_typed_variants(
#[case] wire: Value,
#[case] expected: ResponsesOutputItem,
) {
let item: ResponsesOutputItem = serde_json::from_value(wire.clone()).unwrap();
assert_eq!(item, expected);
assert_eq!(serde_json::to_value(item).unwrap(), wire);
}
#[rstest]
#[case::unknown_caller(json!({"type":"function_call","caller":{"type":"future"}}))]
#[case::program_caller_missing_id(json!({"type":"shell_call","caller":{"type":"program"}}))]
#[case::untagged_caller(json!({"type":"custom_tool_call","caller":{}}))]
#[case::async_not_bool(json!({"type":"function_call","async":"yes"}))]
#[case::output_not_text_or_list(json!({"type":"function_call_output","output":{"text":"t"}}))]
#[case::unknown_input_content(json!({"type":"function_call_output","output":[{"type":"input_audio"}]}))]
#[case::input_text_missing_text(json!({"type":"custom_tool_call_output","output":[{"type":"input_text"}]}))]
#[case::input_image_missing_detail(json!({"type":"custom_tool_call_output","output":[{"type":"input_image","file_id":"f"}]}))]
#[case::bad_cache_mode(json!({"type":"custom_tool_call_output","output":[{"type":"input_text","text":"t","prompt_cache_breakpoint":{"mode":"implicit"}}]}))]
#[case::unknown_computer_action(json!({"type":"computer_call","action":{"type":"teleport"}}))]
#[case::click_missing_button(json!({"type":"computer_call","action":{"type":"click","x":1,"y":2}}))]
#[case::click_fractional_x(json!({"type":"computer_call","action":{"type":"click","button":"left","x":1.5,"y":2}}))]
#[case::drag_point_missing_y(json!({"type":"computer_call","actions":[{"type":"drag","path":[{"x":1}]}]}))]
#[case::keypress_keys_not_list(json!({"type":"computer_call","actions":[{"type":"keypress","keys":"enter"}]}))]
#[case::safety_check_not_object(json!({"type":"computer_call","pending_safety_checks":["sc_1"]}))]
#[case::unknown_screenshot_tag(json!({"type":"computer_call_output","output":{"type":"screenshot"}}))]
#[case::untagged_screenshot(json!({"type":"computer_call_output","output":{"file_id":"f"}}))]
#[case::tool_not_object(json!({"type":"tool_search_output","tools":["lookup"]}))]
#[case::unknown_local_shell_action(json!({"type":"local_shell_call","action":{"type":"spawn","command":[],"env":{}}}))]
#[case::local_shell_missing_env(json!({"type":"local_shell_call","action":{"type":"exec","command":["ls"]}}))]
#[case::local_shell_env_value(json!({"type":"local_shell_call","action":{"type":"exec","command":["ls"],"env":{"A":1}}}))]
#[case::shell_commands_not_list(json!({"type":"shell_call","action":{"commands":"ls"}}))]
#[case::unknown_shell_environment(json!({"type":"shell_call","environment":{"type":"container_auto"}}))]
#[case::container_missing_id(json!({"type":"shell_call","environment":{"type":"container_reference"}}))]
#[case::unknown_shell_outcome(json!({"type":"shell_call_output","output":[{"outcome":{"type":"killed"}}]}))]
#[case::exit_missing_code(json!({"type":"shell_call_output","output":[{"outcome":{"type":"exit"}}]}))]
#[case::negative_max_output(json!({"type":"shell_call_output","max_output_length":-1}))]
#[case::unknown_patch_operation(json!({"type":"apply_patch_call","operation":{"type":"rename_file","path":"a"}}))]
#[case::update_missing_diff(json!({"type":"apply_patch_call","operation":{"type":"update_file","path":"a"}}))]
#[case::approve_not_bool(json!({"type":"mcp_approval_response","approve":"true"}))]
#[case::compaction_content_not_string(json!({"type":"compaction","encrypted_content":7}))]
#[case::program_code_not_string(json!({"type":"program","code":["run()"]}))]
fn output_items_reject_malformed_typed_fields(#[case] wire: Value) {
assert!(serde_json::from_value::<ResponsesOutputItem>(wire).is_err());
}
fn text(value: &str) -> Option<String> {
Some(value.to_owned())
}
}

152
litellm-rust/crates/llms-types/src/formats/responses/streaming_websocket.rs
View file

@ -106,9 +106,18 @@ pub struct ResponsesEventResponse {
#[cfg(test)]
mod tests {
use super::*;
use rstest::rstest;
use super::*;
use crate::formats::responses::ResponsesOutputItem;
use crate::{
formats::responses::streaming_websocket::{ResponsesEventResponse, ResponsesWsEventType},
recognized::Recognized,
test_support::*,
};
use serde_json::{Value, json};
#[test]
fn error_frame_matches_proxy_shape() {
@ -135,4 +144,145 @@ mod tests {
let event: ResponsesWsEvent = serde_json::from_value(payload).expect("valid event");
assert_eq!(event.model(), expected);
}
#[rstest]
#[case::create("response.create", ResponsesWsEventType::ResponseCreate)]
#[case::created("response.created", ResponsesWsEventType::ResponseCreated)]
#[case::completed("response.completed", ResponsesWsEventType::ResponseCompleted)]
#[case::failed("response.failed", ResponsesWsEventType::ResponseFailed)]
#[case::incomplete("response.incomplete", ResponsesWsEventType::ResponseIncomplete)]
#[case::error("error", ResponsesWsEventType::Error)]
#[case::unknown(
"response.output_text.delta",
ResponsesWsEventType::Other("response.output_text.delta".to_string())
)]
#[case::empty("", ResponsesWsEventType::Other(String::new()))]
#[case::case_sensitive(
"Response.Completed",
ResponsesWsEventType::Other("Response.Completed".into())
)]
#[case::escaped("future\"\\\n", ResponsesWsEventType::Other("future\"\\\n".into()))]
fn websocket_event_type_round_trips(
#[case] wire: &str,
#[case] expected: ResponsesWsEventType,
) {
let serialized = serde_json::to_string(&expected).unwrap();
assert_eq!(serialized, serde_json::to_string(wire).unwrap());
assert_eq!(
serde_json::from_str::<ResponsesWsEventType>(&serialized).unwrap(),
expected
);
}
#[rstest]
#[case::number("17")]
#[case::boolean("true")]
#[case::null("null")]
#[case::array("[]")]
#[case::object("{}")]
fn websocket_event_type_rejects_non_strings(#[case] wire: &str) {
assert!(serde_json::from_str::<ResponsesWsEventType>(wire).is_err());
}
#[cfg(feature = "schema")]
#[rstest]
fn websocket_event_type_schema_is_open_string() {
let schema = schemars::schema_for!(ResponsesWsEventType);
assert_eq!(
schema.to_value().get("type"),
Some(&serde_json::json!("string"))
);
}
#[rstest]
fn nested_event_response_exposes_typed_output_and_preserves_extensions() {
let wire = json!({
"id":"response_1",
"model":"example-model",
"status":"completed",
"output":[{"type":"function_call","call_id":"call_1","name":"lookup","arguments":"{}","extension":true}],
"extension":{"nested":[1,null]}
});
let response: ResponsesEventResponse = serde_json::from_value(wire.clone()).unwrap();
assert_eq!(response.id.as_deref(), Some("response_1"));
assert_eq!(response.model.as_deref(), Some("example-model"));
let Some(output) = &response.output else {
panic!("expected typed output");
};
let [Recognized::Known(ResponsesOutputItem::FunctionCall(call))] = output.as_slice() else {
panic!("expected function call");
};
assert_eq!(call.name.as_deref(), Some("lookup"));
assert_eq!(call.arguments.as_deref(), Some("{}"));
assert_eq!(serde_json::to_value(response).unwrap(), wire);
}
#[rstest]
#[case::empty(json!({}))]
#[case::partial(json!({"id":"response_1","output":[]}))]
fn nested_event_response_accepts_partial_metadata(#[case] wire: Value) {
round_trip::<ResponsesEventResponse>(wire);
}
#[rstest]
#[case::wrong_model(json!({"model":7}))]
#[case::wrong_status(json!({"status":false}))]
#[case::wrong_output(json!({"output":{}}))]
fn event_response_rejects_malformed_typed_fields(#[case] wire: Value) {
assert!(serde_json::from_value::<ResponsesEventResponse>(wire).is_err());
}
#[rstest]
#[case::unknown_type(json!({"type":"future_item","id":"item_1","payload":[1,null]}))]
#[case::missing_tag(json!({"id":"item_1"}))]
#[case::malformed_known(json!({"type":"message","content":[{"type":"output_text","text":7}]}))]
fn event_response_keeps_unmodeled_output_items_beside_typed_ones(#[case] item: Value) {
let wire = json!({"output":[{"type":"function_call","name":"lookup"}, item.clone()]});
let response: ResponsesEventResponse = serde_json::from_value(wire.clone()).unwrap();
let Some(
[
Recognized::Known(ResponsesOutputItem::FunctionCall(call)),
Recognized::Unrecognized(kept),
],
) = response.output.as_deref()
else {
panic!("expected one typed item and one preserved item");
};
assert_eq!(call.name.as_deref(), Some("lookup"));
assert_eq!(kept, &item);
assert_eq!(serde_json::to_value(response).unwrap(), wire);
}
#[rstest]
fn event_response_optional_fields_omit_missing_and_null() {
let response: ResponsesEventResponse = serde_json::from_value(json!({
"id":null,"model":null,"status":null,"output":null,"future":null
}))
.unwrap();
assert!(response.id.is_none());
assert!(response.model.is_none());
assert!(response.status.is_none());
assert!(response.output.is_none());
assert_eq!(
serde_json::to_value(response).unwrap(),
json!({"future":null})
);
}
#[rstest]
#[case::compaction(json!({"type":"compaction","id":"cmp_1","encrypted_content":"opaque"}))]
#[case::approval(json!({"type":"mcp_approval_request","id":"apr_1","server_label":"s","name":"n","arguments":"{}"}))]
#[case::shell(json!({"type":"shell_call","call_id":"call_1","action":{"commands":["ls"]}}))]
fn event_response_types_documented_output_items(#[case] item: Value) {
let wire = json!({"output":[item.clone()]});
let response: ResponsesEventResponse = serde_json::from_value(wire.clone()).unwrap();
let Some([Recognized::Known(known)]) = response.output.as_deref() else {
panic!("expected one typed item");
};
assert_eq!(
known,
&serde_json::from_value::<ResponsesOutputItem>(item).unwrap()
);
assert_eq!(serde_json::to_value(response).unwrap(), wire);
}
}

144
litellm-rust/crates/llms-types/src/json_schema.rs
View file

@ -49,3 +49,147 @@ pub struct JsonSchemaObject {
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[cfg(test)]
mod tests {
use crate::{
json_schema::{JsonSchema, JsonSchemaItems, JsonSchemaObject, JsonSchemaType},
test_support::*,
};
use rstest::rstest;
use serde_json::{Value, json};
#[rstest]
#[case::boolean(json!(false))]
#[case::nested(json!({
"type":["object","null"],
"properties":{"nested":{"type":"array","items":{"$ref":"#/$defs/item"}},"free":true},
"additionalProperties":{"type":"string"},
"$defs":{"item":{"anyOf":[{"type":"string"},false]}},
"required":["nested"],
"enum":[{"custom":[1,null]}],
"future":null
}))]
fn recursive_schema_round_trips(#[case] wire: Value) {
let schema = round_trip::<JsonSchema>(wire);
match schema {
JsonSchema::Boolean(allowed) => assert!(!allowed),
JsonSchema::Object(schema) => {
assert_eq!(
schema.schema_type,
Some(JsonSchemaType::Names(vec!["object".into(), "null".into()]))
);
let properties = schema.properties.as_ref().unwrap();
let JsonSchema::Object(nested) = &properties["nested"] else {
panic!("expected nested schema")
};
let Some(JsonSchemaItems::Schema(items)) = &nested.items else {
panic!("expected single items schema")
};
let JsonSchema::Object(items) = items.as_ref() else {
panic!("expected items schema")
};
assert_eq!(items.reference.as_deref(), Some("#/$defs/item"));
assert_eq!(properties["free"], JsonSchema::Boolean(true));
let JsonSchema::Object(definition) = &schema.defs.as_ref().unwrap()["item"] else {
panic!("expected schema definition")
};
assert_eq!(
definition.any_of.as_ref().unwrap()[1],
JsonSchema::Boolean(false)
);
assert_eq!(schema.extra["enum"], json!([{"custom":[1,null]}]));
}
}
}
#[rstest]
fn schema_maps_keep_wire_key_order() {
let wire = r#"{"type":"object","properties":{"zeta":{"type":"string"},"alpha":{"type":"integer"}},"$defs":{"y":true,"b":false}}"#;
let schema: JsonSchemaObject = serde_json::from_str(wire).unwrap();
let names: Vec<&str> = schema
.properties
.as_ref()
.unwrap()
.keys()
.map(String::as_str)
.collect();
assert_eq!(names, ["zeta", "alpha"]);
assert_eq!(serde_json::to_string(&schema).unwrap(), wire);
}
#[rstest]
fn schema_object_round_trips_supported_keywords() {
let schema = round_trip::<JsonSchemaObject>(json!({
"type":"object",
"properties":{"name":{"type":"string"}},
"required":["name"],
"additionalProperties":false,
"$ref":"#/$defs/value",
"strict":true,
"extension":{"nested":[1,null]}
}));
assert_eq!(
schema.schema_type,
Some(JsonSchemaType::Name("object".into()))
);
assert_eq!(schema.required.as_deref(), Some(["name".into()].as_slice()));
assert_eq!(
schema.additional_properties.as_deref(),
Some(&JsonSchema::Boolean(false))
);
assert_eq!(schema.reference.as_deref(), Some("#/$defs/value"));
assert_eq!(schema.strict, Some(true));
}
#[rstest]
#[case::scalar(json!(7))]
#[case::properties_shape(json!({"properties":[]}))]
#[case::nested_schema(json!({"properties":{"name":7}}))]
#[case::schema_type(json!({"type":["string",7]}))]
#[case::items_shape(json!({"items":7}))]
#[case::tuple_member(json!({"items":[{"type":"string"},7]}))]
#[case::prefix_items_shape(json!({"prefixItems":{"type":"string"}}))]
#[case::composite_shape(json!({"anyOf":["string"]}))]
fn schemas_reject_malformed_known_keywords(#[case] wire: Value) {
assert!(serde_json::from_value::<JsonSchema>(wire).is_err());
}
#[rstest]
#[case::draft_07_tuple(json!({"type":"array","items":[{"type":"string"},true]}))]
#[case::draft_2020_12_tuple(json!({"type":"array","prefixItems":[{"type":"string"},true],"items":false}))]
fn tuple_schemas_expose_positional_members(#[case] wire: Value) {
let schema = round_trip::<JsonSchemaObject>(wire);
let members = match (&schema.items, &schema.prefix_items) {
(Some(JsonSchemaItems::Tuple(members)), None) => members,
(Some(JsonSchemaItems::Schema(rest)), Some(members)) => {
assert_eq!(rest.as_ref(), &JsonSchema::Boolean(false));
members
}
_ => panic!("expected positional members"),
};
let [JsonSchema::Object(first), JsonSchema::Boolean(true)] = members.as_slice() else {
panic!("expected string schema then true");
};
assert_eq!(
first.schema_type,
Some(JsonSchemaType::Name("string".into()))
);
assert!(schema.extra.is_empty());
}
#[rstest]
fn empty_schema_omits_null_optionals_and_preserves_arbitrary_keywords() {
let schema: JsonSchemaObject = serde_json::from_value(
json!({"type":null,"properties":null,"const":{"arbitrary":[1,null]},"future":null}),
)
.unwrap();
assert!(schema.schema_type.is_none());
assert!(schema.properties.is_none());
assert_eq!(
serde_json::to_value(schema).unwrap(),
json!({"const":{"arbitrary":[1,null]},"future":null})
);
}
}

40
litellm-rust/crates/llms-types/src/lib.rs
View file

@ -10,3 +10,43 @@ pub mod json_schema;
pub mod providers;
pub mod recognized;
pub mod serde_compat;
#[cfg(test)]
mod test_support;
#[cfg(test)]
mod tests {
use crate::formats::chat_completions::ChatMessage;
use rstest::rstest;
use serde_json::json;
#[rstest]
fn wire_type_preserves_serialization() {
let message = ChatMessage {
role: "user".to_owned(),
content: None,
name: None,
extra: Default::default(),
};
assert_eq!(
serde_json::to_value(message).unwrap(),
json!({"role": "user"})
);
}
#[cfg(feature = "schema")]
#[rstest]
fn wire_type_supports_schema_generation() {
let schema = schemars::schema_for!(ChatMessage);
assert!(
schema
.to_value()
.get("properties")
.and_then(serde_json::Value::as_object)
.is_some_and(|properties| properties.contains_key("role"))
);
}
}

18
litellm-rust/crates/llms-types/src/providers/AGENTS.md
View file

@ -1,18 +0,0 @@
# references
These links describe the provider-specific wire types in this directory. Shared Messages payload references live in `../formats/messages/AGENTS.md`
## anthropic.rs
`AnthropicBeta`, `BetaSet` and `BetaProvider` represent the `anthropic-beta` header values, their
wire spelling and per-host support
- https://platform.claude.com/docs/en/api/beta-headers.md
## minimax.rs
MiniMax's Messages reference documents its image, video, and mid-conversation system content extensions. Its cache reference documents `cache_control` on those blocks
- https://platform.minimax.io/docs/api-reference/text-chat-anthropic
- https://platform.minimax.io/docs/api-reference/text-chat-anthropic.md
- https://platform.minimax.io/docs/api-reference/anthropic-api-compatible-cache.md

9
litellm-rust/crates/llms-types/src/providers/anthropic/AGENTS.md Normal file
View file

@ -0,0 +1,9 @@
# rules
- Shared across Anthropic's Messages, chat, count-tokens and batches adapters: API base, endpoint paths, header names, API version and default headers
- `beta.rs` holds `AnthropicBeta`, `BetaSet` and `BetaProvider`, the `anthropic-beta` values, their wire spelling and which hosts accept each one
- Choosing which betas a request needs is policy in `llms/src/anthropic/`
# references
- https://platform.claude.com/docs/en/api/beta-headers.md

8
litellm-rust/crates/llms-types/src/providers/anthropic.rs → litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs
View file

@ -380,16 +380,14 @@ impl fmt::Display for BetaSet {
#[cfg(test)]
mod tests {
use std::collections::BTreeSet;
use super::*;
use indexmap::IndexMap;
use rstest::rstest;
use super::*;
use std::collections::BTreeSet;
fn beta_headers_config() -> IndexMap<String, serde_json::Value> {
serde_json::from_str(include_str!(
"../../../../../litellm/anthropic_beta_headers_config.json"
"../../../../../../litellm/anthropic_beta_headers_config.json"
))
.unwrap()
}

15
litellm-rust/crates/llms-types/src/providers/anthropic/constants.rs Normal file
View file

@ -0,0 +1,15 @@
pub const API_BASE: &str = "https://api.anthropic.com";
pub const MESSAGES_PATH: &str = "/v1/messages";
pub const BATCHES_PATH: &str = "/v1/messages/batches";
pub const COUNT_TOKENS_PATH: &str = "/v1/messages/count_tokens";
pub const API_KEY_HEADER: &str = "x-api-key";
pub const BETA_HEADER: &str = "anthropic-beta";
pub const VERSION_HEADER: &str = "anthropic-version";
pub const DIRECT_BROWSER_ACCESS_HEADER: &str = "anthropic-dangerous-direct-browser-access";
pub const API_VERSION: &str = "2023-06-01";
pub const DEFAULT_HEADERS: &[(&str, &str)] = &[
(VERSION_HEADER, API_VERSION),
("content-type", "application/json"),
];

5
litellm-rust/crates/llms-types/src/providers/anthropic/mod.rs Normal file
View file

@ -0,0 +1,5 @@
mod beta;
mod constants;
pub use beta::*;
pub use constants::*;

10
litellm-rust/crates/llms-types/src/providers/bedrock/AGENTS.md Normal file
View file

@ -0,0 +1,10 @@
# rules
- Shared across Bedrock's chat, Messages and audio-transcription adapters: Converse and InvokeModel paths, the invocation-metrics key and the Converse response body
- The Converse response is a partial projection. Unread blocks decode as `ConverseContentBlock::Other` so adapters can decline them
# references
- https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html
- https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_InvokeModel.html
- https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_InvokeModelWithResponseStream.html

4
litellm-rust/crates/llms-types/src/providers/bedrock/constants.rs Normal file
View file

@ -0,0 +1,4 @@
pub const CONVERSE_PATH: &str = "/converse";
pub const INVOKE_PATH: &str = "invoke";
pub const INVOKE_STREAM_PATH: &str = "invoke-with-response-stream";
pub const INVOCATION_METRICS_KEY: &str = "amazon-bedrock-invocationMetrics";

76
litellm-rust/crates/llms-types/src/providers/bedrock/converse.rs Normal file
View file

@ -0,0 +1,76 @@
use serde::Deserialize;
use serde_json::Value;
#[derive(Debug, Deserialize)]
pub struct ConverseResponse {
pub output: ConverseOutput,
#[serde(default)]
pub usage: ConverseUsage,
#[serde(rename = "stopReason")]
pub stop_reason: Option<String>,
}
#[derive(Debug, Deserialize)]
pub struct ConverseOutput {
pub message: ConverseMessage,
}
#[derive(Debug, Deserialize)]
pub struct ConverseMessage {
pub content: Vec<ConverseContentBlock>,
}
#[derive(Debug)]
pub enum ConverseContentBlock {
Text { text: String },
Other,
}
impl<'de> Deserialize<'de> for ConverseContentBlock {
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
let value = Value::deserialize(deserializer)?;
let Some(text) = value.get("text") else {
return Ok(Self::Other);
};
let text = text.as_str().ok_or_else(|| {
serde::de::Error::custom("invalid type for `text`, expected a string")
})?;
Ok(Self::Text {
text: text.to_owned(),
})
}
}
#[derive(Debug, Default, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ConverseUsage {
pub input_tokens: u64,
pub output_tokens: u64,
#[serde(default)]
pub cache_read_input_tokens: u64,
#[serde(default)]
pub cache_write_input_tokens: u64,
pub total_tokens: Option<u64>,
}
impl ConverseResponse {
pub fn content_text(&self) -> String {
self.output
.message
.content
.iter()
.filter_map(|block| match block {
ConverseContentBlock::Text { text } => Some(text.as_str()),
ConverseContentBlock::Other => None,
})
.collect()
}
pub fn message_content_is_non_text(&self) -> bool {
self.output
.message
.content
.iter()
.any(|block| matches!(block, ConverseContentBlock::Other))
}
}

5
litellm-rust/crates/llms-types/src/providers/bedrock/mod.rs Normal file
View file

@ -0,0 +1,5 @@
mod constants;
mod converse;
pub use constants::*;
pub use converse::*;

59
litellm-rust/crates/llms-types/src/providers/minimax.rs
View file

@ -1,59 +0,0 @@
use serde_json::{Map, Value};
use crate::formats::messages::CacheControl;
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum MinimaxMessagesContentBlock {
Image(MinimaxMediaBlock),
Video(MinimaxMediaBlock),
MidConvSystem {
text: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct MinimaxMediaBlock {
pub source: MinimaxMediaSource,
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum MinimaxMediaSource {
Base64 {
media_type: String,
data: String,
#[serde(flatten)]
options: MinimaxMediaOptions,
},
Url {
url: String,
#[serde(flatten)]
options: MinimaxMediaOptions,
},
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct MinimaxMediaOptions {
pub detail: Option<MinimaxMediaDetail>,
pub fps: Option<serde_json::Number>,
pub max_long_side_pixel: Option<u64>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(rename_all = "snake_case")]
pub enum MinimaxMediaDetail {
Low,
Default,
High,
}

9
litellm-rust/crates/llms-types/src/providers/minimax/AGENTS.md Normal file
View file

@ -0,0 +1,9 @@
# rules
- Typed MiniMax extensions of the Messages format: image, video and mid-conversation system content blocks, with `cache_control` on them
# references
- https://platform.minimax.io/docs/api-reference/text-chat-anthropic
- https://platform.minimax.io/docs/api-reference/text-chat-anthropic.md
- https://platform.minimax.io/docs/api-reference/anthropic-api-compatible-cache.md

146
litellm-rust/crates/llms-types/src/providers/minimax/messages.rs Normal file
View file

@ -0,0 +1,146 @@
use serde_json::{Map, Value};
use crate::formats::messages::CacheControl;
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum MinimaxMessagesContentBlock {
Image(MinimaxMediaBlock),
Video(MinimaxMediaBlock),
MidConvSystem {
text: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct MinimaxMediaBlock {
pub source: MinimaxMediaSource,
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum MinimaxMediaSource {
Base64 {
media_type: String,
data: String,
#[serde(flatten)]
options: MinimaxMediaOptions,
},
Url {
url: String,
#[serde(flatten)]
options: MinimaxMediaOptions,
},
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct MinimaxMediaOptions {
pub detail: Option<MinimaxMediaDetail>,
pub fps: Option<serde_json::Number>,
pub max_long_side_pixel: Option<u64>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(rename_all = "snake_case")]
pub enum MinimaxMediaDetail {
Low,
Default,
High,
}
#[cfg(test)]
mod tests {
use crate::providers::minimax::{
MinimaxMediaDetail, MinimaxMediaSource, MinimaxMessagesContentBlock,
};
use rstest::rstest;
use serde_json::{Value, json};
#[rstest]
#[case::image(json!({"type":"image","source":{"type":"base64","media_type":"image/png","data":"AA==","detail":"low"}}))]
#[case::video(json!({"type":"video","source":{"type":"url","url":"https://example.test/video","detail":"high","fps":1,"max_long_side_pixel":1024,"future":null},"cache_control":{"type":"ephemeral"}}))]
#[case::mid_conversation_system(json!({"type":"mid_conv_system","text":"instruction","future":null}))]
fn provider_content_blocks_round_trip(#[case] wire: Value) {
let block: MinimaxMessagesContentBlock = serde_json::from_value(wire.clone()).unwrap();
match &block {
MinimaxMessagesContentBlock::Image(image) => {
let MinimaxMediaSource::Base64 {
media_type,
data,
options,
} = &image.source
else {
panic!("expected base64 image source");
};
assert_eq!((media_type.as_str(), data.as_str()), ("image/png", "AA=="));
assert_eq!(options.detail, Some(MinimaxMediaDetail::Low));
assert!(options.extra.is_empty());
}
MinimaxMessagesContentBlock::Video(video) => {
let MinimaxMediaSource::Url { url, options } = &video.source else {
panic!("expected URL video source");
};
assert_eq!(url, "https://example.test/video");
assert_eq!(options.detail, Some(MinimaxMediaDetail::High));
assert_eq!(options.fps, Some(1.into()));
assert_eq!(options.max_long_side_pixel, Some(1024));
assert_eq!(options.extra["future"], Value::Null);
assert_eq!(
video.cache_control.as_ref().unwrap().cache_type.as_deref(),
Some("ephemeral")
);
}
MinimaxMessagesContentBlock::MidConvSystem { text, extra } => {
assert_eq!(text, "instruction");
assert_eq!(extra["future"], Value::Null);
}
}
assert_eq!(serde_json::to_value(block).unwrap(), wire);
}
#[rstest]
#[case::missing_source(json!({"type":"video"}))]
#[case::bad_source_tag(json!({"type":"image","source":{"type":"future"}}))]
#[case::url_without_url(json!({"type":"video","source":{"type":"url"}}))]
#[case::base64_without_data(json!({"type":"image","source":{"type":"base64","media_type":"image/png"}}))]
#[case::bad_detail(json!({"type":"image","source":{"type":"url","url":"u","detail":7}}))]
#[case::bad_fps(json!({"type":"video","source":{"type":"url","url":"u","fps":"fast"}}))]
#[case::missing_text(json!({"type":"mid_conv_system"}))]
fn provider_content_rejects_malformed_fields(#[case] wire: Value) {
assert!(serde_json::from_value::<MinimaxMessagesContentBlock>(wire).is_err());
}
#[rstest]
fn partial_media_options_omit_null_optionals() {
let block: MinimaxMessagesContentBlock = serde_json::from_value(json!({
"type":"video",
"source":{"type":"url","url":"u","detail":null,"fps":null,"future":null},
"cache_control":null
}))
.unwrap();
let MinimaxMessagesContentBlock::Video(video) = &block else {
panic!("expected video")
};
let MinimaxMediaSource::Url { options, .. } = &video.source else {
panic!("expected URL source");
};
assert!(options.detail.is_none());
assert!(options.fps.is_none());
assert!(video.cache_control.is_none());
assert_eq!(
serde_json::to_value(block).unwrap(),
json!({"type":"video","source":{"type":"url","url":"u","future":null}})
);
}
}

3
litellm-rust/crates/llms-types/src/providers/minimax/mod.rs Normal file
View file

@ -0,0 +1,3 @@
mod messages;
pub use messages::*;

2
litellm-rust/crates/llms-types/src/providers/mod.rs
View file

@ -1,2 +1,4 @@
pub mod anthropic;
pub mod bedrock;
pub mod minimax;
pub mod vertex_ai;

8
litellm-rust/crates/llms-types/src/providers/vertex_ai/AGENTS.md Normal file
View file

@ -0,0 +1,8 @@
# rules
- Shared across Vertex AI's Messages and OCR adapters: the global host, the global and default locations, the `rawPredict` methods and the Anthropic API version Vertex expects in the body
- Location validation and host selection are in `llms/src/vertex_ai/common_utils.rs`
# references
- https://cloud.google.com/vertex-ai/docs/general/locations

7
litellm-rust/crates/llms-types/src/providers/vertex_ai/constants.rs Normal file
View file

@ -0,0 +1,7 @@
pub const GLOBAL_API_BASE: &str = "https://aiplatform.googleapis.com";
pub const GLOBAL_LOCATION: &str = "global";
pub const DEFAULT_LOCATION: &str = "us-central1";
pub const RAW_PREDICT: &str = "rawPredict";
pub const STREAM_RAW_PREDICT: &str = "streamRawPredict";
pub const ANTHROPIC_VERSION: &str = "vertex-2023-10-16";

3
litellm-rust/crates/llms-types/src/providers/vertex_ai/mod.rs Normal file
View file

@ -0,0 +1,3 @@
mod constants;
pub use constants::*;

3
litellm-rust/crates/llms-types/src/recognized.rs
View file

@ -18,11 +18,10 @@ impl<T> Recognized<T> {
#[cfg(test)]
mod tests {
use super::*;
use rstest::rstest;
use serde_json::json;
use super::*;
#[rstest]
#[case::known(json!(7), Recognized::Known(7))]
#[case::wrong_type(json!("7"), Recognized::Unrecognized(json!("7")))]

91
litellm-rust/crates/llms-types/src/serde_compat.rs
View file

@ -111,3 +111,94 @@ fn integral_float(value: f64) -> Option<i64> {
&& value < -(i64::MIN as f64))
.then_some(value as i64)
}
#[cfg(test)]
mod tests {
use crate::serde_compat::{FiniteF64, LaxI64};
use rstest::rstest;
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use serde_with::serde_as;
#[serde_as]
#[derive(Debug, Deserialize, Serialize, PartialEq)]
struct Numbers {
#[serde_as(deserialize_as = "Option<Vec<LaxI64>>")]
integers: Option<Vec<i64>>,
#[serde_as(deserialize_as = "Option<FiniteF64>")]
float: Option<f64>,
}
#[rstest]
fn adapters_compose_and_serialize_as_numbers() {
let numbers: Numbers = serde_json::from_value(json!({
"integers": ["9007199254740993.0", "1_000", " +2.000 ", 3.0, true],
"float": " 1.5 "
}))
.unwrap();
assert_eq!(
serde_json::to_value(numbers).unwrap(),
json!({"integers": [9_007_199_254_740_993_i64, 1000, 2, 3, 1], "float": 1.5})
);
}
#[rstest]
#[case::missing(json!({}))]
#[case::null(json!({"integers": null, "float": null}))]
fn optional_adapters_accept_missing_and_null_fields(#[case] input: Value) {
assert_eq!(
serde_json::from_value::<Numbers>(input).unwrap(),
Numbers {
integers: None,
float: None
}
);
}
#[rstest]
#[case::minimum(json!(i64::MIN), i64::MIN)]
#[case::maximum(json!(i64::MAX), i64::MAX)]
#[case::maximum_string(json!(i64::MAX.to_string()), i64::MAX)]
fn integers_preserve_bounds(#[case] input: Value, #[case] expected: i64) {
let numbers: Numbers = serde_json::from_value(json!({"integers": [input]})).unwrap();
assert_eq!(numbers.integers, Some(vec![expected]));
}
#[rstest]
#[case::unsigned_maximum(json!(u64::MAX))]
#[case::above_maximum(json!(9_223_372_036_854_775_808_u64))]
#[case::float_above_maximum(json!(9_223_372_036_854_775_808.0))]
#[case::below_minimum(json!("-9223372036854775809"))]
#[case::precise_fraction(json!("1.0000000000000001"))]
#[case::exponent(json!("1e3"))]
#[case::missing_fraction(json!("2."))]
#[case::missing_integer(json!(".0"))]
#[case::leading_separator(json!("_2"))]
#[case::repeated_separator(json!("2__0"))]
#[case::fraction(json!(2.5))]
#[case::null(json!(null))]
#[case::object(json!({}))]
fn integers_reject_invalid_values(#[case] input: Value) {
assert!(serde_json::from_value::<Numbers>(json!({"integers": [input]})).is_err());
}
#[rstest]
#[case::nan(json!("NaN"))]
#[case::positive_infinity(json!("inf"))]
#[case::negative_infinity(json!("-inf"))]
#[case::overflow(json!("1e999"))]
#[case::array(json!([]))]
fn floats_reject_nonfinite_and_invalid_values(#[case] input: Value) {
assert!(serde_json::from_value::<Numbers>(json!({"float": input})).is_err());
}
#[rstest]
#[case::integer(json!(2), 2.0)]
#[case::float(json!(2.5), 2.5)]
#[case::boolean(json!(true), 1.0)]
fn floats_accept_finite_numbers(#[case] input: Value, #[case] expected: f64) {
let numbers: Numbers = serde_json::from_value(json!({"float": input})).unwrap();
assert_eq!(numbers.float, Some(expected));
}
}

11
litellm-rust/crates/llms-types/src/test_support.rs Normal file
View file

@ -0,0 +1,11 @@
use serde::{Serialize, de::DeserializeOwned};
use serde_json::Value;
pub(crate) fn round_trip<T>(wire: Value) -> T
where
T: DeserializeOwned + Serialize,
{
let parsed: T = serde_json::from_value(wire.clone()).unwrap();
assert_eq!(serde_json::to_value(&parsed).unwrap(), wire);
parsed
}

163
litellm-rust/crates/llms-types/tests/chat.rs
View file

@ -1,163 +0,0 @@
use litellm_llms_types::formats::chat_completions::{
ChatContentPart, ChatLogprobs, ChatMediaUrl, PromptCacheMode,
};
use litellm_llms_types::formats::messages::ContentSource;
use rstest::rstest;
use serde_json::{Value, json};
#[rstest]
#[case::parts(json!([
{"type":"text","text":"hello","cache_control":{"type":"ephemeral"},"prompt_cache_breakpoint":{"mode":"explicit"}},
{"type":"image_url","image_url":{"url":"https://example.test/image","detail":"high","format":"image/png"}},
{"type":"video_url","video_url":"https://example.test/video"},
{"type":"input_audio","input_audio":{"data":"AA==","format":"wav"}},
{"type":"file","file":{"file_id":"file_1","file_data":"JVBE","filename":"a.mp4","format":"video/mp4","detail":"low","video_metadata":{"fps":1,"start_offset":"1s","end_offset":"2s"}}},
{"type":"document","source":{"type":"text","media_type":"text/plain","data":"doc"},"title":"T","context":"C","citations":{"enabled":true}},
{"type":"refusal","refusal":"refused"}
]))]
fn content_parts_round_trip(#[case] wire: Value) {
let parts: Vec<ChatContentPart> = serde_json::from_value(wire.clone()).unwrap();
let [
ChatContentPart::Text {
text,
cache_control,
prompt_cache_breakpoint: Some(breakpoint),
extra: text_extra,
},
ChatContentPart::ImageUrl {
image_url: ChatMediaUrl::Parameters(image),
prompt_cache_breakpoint: None,
..
},
ChatContentPart::VideoUrl {
video_url: ChatMediaUrl::Url(video),
..
},
ChatContentPart::InputAudio { input_audio, .. },
ChatContentPart::File { file, .. },
ChatContentPart::Document {
source,
title: Some(title),
context: Some(context),
citations: Some(citations),
..
},
ChatContentPart::Refusal { refusal, .. },
] = parts.as_slice()
else {
panic!("expected typed content parts");
};
assert_eq!(text, "hello");
assert_eq!(
cache_control.as_ref().unwrap().cache_type.as_deref(),
Some("ephemeral")
);
assert_eq!(breakpoint.mode, PromptCacheMode::Explicit);
assert!(breakpoint.extra.is_empty());
assert!(text_extra.is_empty());
assert_eq!(image.url, "https://example.test/image");
assert_eq!(image.detail.as_deref(), Some("high"));
assert_eq!(image.format.as_deref(), Some("image/png"));
assert!(image.extra.is_empty());
assert_eq!(video, "https://example.test/video");
assert_eq!(input_audio.data, "AA==");
assert_eq!(input_audio.format, "wav");
assert_eq!(file.file_id.as_deref(), Some("file_1"));
assert_eq!(file.file_data.as_deref(), Some("JVBE"));
assert_eq!(file.filename.as_deref(), Some("a.mp4"));
assert_eq!(file.format.as_deref(), Some("video/mp4"));
assert_eq!(file.detail.as_deref(), Some("low"));
assert!(file.extra.is_empty());
let metadata = file.video_metadata.as_ref().unwrap();
assert_eq!(metadata.fps, Some(1.into()));
assert_eq!(metadata.start_offset.as_deref(), Some("1s"));
assert_eq!(metadata.end_offset.as_deref(), Some("2s"));
assert!(metadata.extra.is_empty());
let ContentSource::Text {
data, media_type, ..
} = source.as_ref()
else {
panic!("expected document text source");
};
assert_eq!(data, "doc");
assert_eq!(media_type, "text/plain");
assert_eq!(title, "T");
assert_eq!(context, "C");
assert_eq!(citations.enabled, Some(true));
assert_eq!(refusal, "refused");
assert_eq!(serde_json::to_value(parts).unwrap(), wire);
}
#[rstest]
fn logprobs_round_trip_with_tokens_and_alternatives() {
let wire = json!({
"content":[{"token":"hi","logprob":-1,"bytes":[104,105],"top_logprobs":[{"token":"hey","logprob":-2.5,"bytes":[104]}]}],
"refusal":[{"token":"refused","logprob":-3}]
});
let logprobs: ChatLogprobs = serde_json::from_value(wire.clone()).unwrap();
let token = &logprobs.content.as_ref().unwrap()[0];
assert_eq!(token.token, "hi");
assert_eq!(token.logprob, (-1).into());
assert_eq!(token.bytes.as_deref(), Some([104, 105].as_slice()));
let alternative = &token.top_logprobs.as_ref().unwrap()[0];
assert_eq!(alternative.token, "hey");
assert_eq!(alternative.bytes.as_deref(), Some([104].as_slice()));
assert_eq!(logprobs.refusal.as_ref().unwrap()[0].token, "refused");
assert!(logprobs.refusal.as_ref().unwrap()[0].top_logprobs.is_none());
assert_eq!(serde_json::to_value(logprobs).unwrap(), wire);
}
#[rstest]
#[case::text(json!({"type":"text","text":false}))]
#[case::missing_image(json!({"type":"image_url"}))]
#[case::audio_shape(json!({"type":"input_audio","input_audio":{"data":7,"format":"wav"}}))]
#[case::audio_missing_format(json!({"type":"input_audio","input_audio":{"data":"AA=="}}))]
#[case::image_missing_url(json!({"type":"image_url","image_url":{"detail":"high"}}))]
#[case::file_metadata(json!({"type":"file","file":{"video_metadata":{"fps":"fast"}}}))]
#[case::document_source(json!({"type":"document","source":{"type":"url","url":false}}))]
#[case::refusal_missing_text(json!({"type":"refusal"}))]
#[case::file_missing_file(json!({"type":"file"}))]
#[case::document_missing_source(json!({"type":"document"}))]
#[case::video_missing_url(json!({"type":"video_url"}))]
#[case::cache_control_shape(json!({"type":"text","text":"t","cache_control":"ephemeral"}))]
#[case::breakpoint_missing_mode(json!({"type":"text","text":"t","prompt_cache_breakpoint":{}}))]
#[case::breakpoint_unknown_mode(json!({"type":"text","text":"t","prompt_cache_breakpoint":{"mode":"auto"}}))]
#[case::unknown_tag(json!({"type":"future"}))]
fn content_parts_reject_malformed_typed_fields(#[case] wire: Value) {
assert!(serde_json::from_value::<ChatContentPart>(wire).is_err());
}
#[rstest]
fn partial_file_preserves_extensions_and_omits_null_optionals() {
let wire = json!({"type":"file","file":{"file_id":null,"filename":"a.pdf","extension":[1,null]},"future":true});
let part: ChatContentPart = serde_json::from_value(wire).unwrap();
let ChatContentPart::File {
file,
prompt_cache_breakpoint: None,
extra,
} = &part
else {
panic!("expected file")
};
assert!(file.file_id.is_none());
assert!(file.file_data.is_none());
assert_eq!(file.filename.as_deref(), Some("a.pdf"));
assert_eq!(extra["future"], json!(true));
assert_eq!(
serde_json::to_value(part).unwrap(),
json!({"type":"file","file":{"filename":"a.pdf","extension":[1,null]},"future":true})
);
}
#[rstest]
#[case::missing_token(json!({"logprob":-1}))]
#[case::missing_logprob(json!({"token":"hi"}))]
#[case::wrong_bytes(json!({"token":"hi","logprob":-1,"bytes":[256]}))]
fn token_logprobs_reject_malformed_fields(#[case] wire: Value) {
assert!(
serde_json::from_value::<litellm_llms_types::formats::chat_completions::ChatTokenLogprob>(
wire
)
.is_err()
);
}

148
litellm-rust/crates/llms-types/tests/json_schema.rs
View file

@ -1,148 +0,0 @@
use litellm_llms_types::json_schema::{
JsonSchema, JsonSchemaItems, JsonSchemaObject, JsonSchemaType,
};
use rstest::rstest;
use serde::{Serialize, de::DeserializeOwned};
use serde_json::{Value, json};
fn round_trip<T>(wire: Value) -> T
where
T: DeserializeOwned + Serialize,
{
let parsed: T = serde_json::from_value(wire.clone()).unwrap();
assert_eq!(serde_json::to_value(&parsed).unwrap(), wire);
parsed
}
#[rstest]
#[case::boolean(json!(false))]
#[case::nested(json!({
"type":["object","null"],
"properties":{"nested":{"type":"array","items":{"$ref":"#/$defs/item"}},"free":true},
"additionalProperties":{"type":"string"},
"$defs":{"item":{"anyOf":[{"type":"string"},false]}},
"required":["nested"],
"enum":[{"custom":[1,null]}],
"future":null
}))]
fn recursive_schema_round_trips(#[case] wire: Value) {
let schema = round_trip::<JsonSchema>(wire);
match schema {
JsonSchema::Boolean(allowed) => assert!(!allowed),
JsonSchema::Object(schema) => {
assert_eq!(
schema.schema_type,
Some(JsonSchemaType::Names(vec!["object".into(), "null".into()]))
);
let properties = schema.properties.as_ref().unwrap();
let JsonSchema::Object(nested) = &properties["nested"] else {
panic!("expected nested schema")
};
let Some(JsonSchemaItems::Schema(items)) = &nested.items else {
panic!("expected single items schema")
};
let JsonSchema::Object(items) = items.as_ref() else {
panic!("expected items schema")
};
assert_eq!(items.reference.as_deref(), Some("#/$defs/item"));
assert_eq!(properties["free"], JsonSchema::Boolean(true));
let JsonSchema::Object(definition) = &schema.defs.as_ref().unwrap()["item"] else {
panic!("expected schema definition")
};
assert_eq!(
definition.any_of.as_ref().unwrap()[1],
JsonSchema::Boolean(false)
);
assert_eq!(schema.extra["enum"], json!([{"custom":[1,null]}]));
}
}
}
#[rstest]
fn schema_maps_keep_wire_key_order() {
let wire = r#"{"type":"object","properties":{"zeta":{"type":"string"},"alpha":{"type":"integer"}},"$defs":{"y":true,"b":false}}"#;
let schema: JsonSchemaObject = serde_json::from_str(wire).unwrap();
let names: Vec<&str> = schema
.properties
.as_ref()
.unwrap()
.keys()
.map(String::as_str)
.collect();
assert_eq!(names, ["zeta", "alpha"]);
assert_eq!(serde_json::to_string(&schema).unwrap(), wire);
}
#[rstest]
fn schema_object_round_trips_supported_keywords() {
let schema = round_trip::<JsonSchemaObject>(json!({
"type":"object",
"properties":{"name":{"type":"string"}},
"required":["name"],
"additionalProperties":false,
"$ref":"#/$defs/value",
"strict":true,
"extension":{"nested":[1,null]}
}));
assert_eq!(
schema.schema_type,
Some(JsonSchemaType::Name("object".into()))
);
assert_eq!(schema.required.as_deref(), Some(["name".into()].as_slice()));
assert_eq!(
schema.additional_properties.as_deref(),
Some(&JsonSchema::Boolean(false))
);
assert_eq!(schema.reference.as_deref(), Some("#/$defs/value"));
assert_eq!(schema.strict, Some(true));
}
#[rstest]
#[case::scalar(json!(7))]
#[case::properties_shape(json!({"properties":[]}))]
#[case::nested_schema(json!({"properties":{"name":7}}))]
#[case::schema_type(json!({"type":["string",7]}))]
#[case::items_shape(json!({"items":7}))]
#[case::tuple_member(json!({"items":[{"type":"string"},7]}))]
#[case::prefix_items_shape(json!({"prefixItems":{"type":"string"}}))]
#[case::composite_shape(json!({"anyOf":["string"]}))]
fn schemas_reject_malformed_known_keywords(#[case] wire: Value) {
assert!(serde_json::from_value::<JsonSchema>(wire).is_err());
}
#[rstest]
#[case::draft_07_tuple(json!({"type":"array","items":[{"type":"string"},true]}))]
#[case::draft_2020_12_tuple(json!({"type":"array","prefixItems":[{"type":"string"},true],"items":false}))]
fn tuple_schemas_expose_positional_members(#[case] wire: Value) {
let schema = round_trip::<JsonSchemaObject>(wire);
let members = match (&schema.items, &schema.prefix_items) {
(Some(JsonSchemaItems::Tuple(members)), None) => members,
(Some(JsonSchemaItems::Schema(rest)), Some(members)) => {
assert_eq!(rest.as_ref(), &JsonSchema::Boolean(false));
members
}
_ => panic!("expected positional members"),
};
let [JsonSchema::Object(first), JsonSchema::Boolean(true)] = members.as_slice() else {
panic!("expected string schema then true");
};
assert_eq!(
first.schema_type,
Some(JsonSchemaType::Name("string".into()))
);
assert!(schema.extra.is_empty());
}
#[rstest]
fn empty_schema_omits_null_optionals_and_preserves_arbitrary_keywords() {
let schema: JsonSchemaObject = serde_json::from_value(
json!({"type":null,"properties":null,"const":{"arbitrary":[1,null]},"future":null}),
)
.unwrap();
assert!(schema.schema_type.is_none());
assert!(schema.properties.is_none());
assert_eq!(
serde_json::to_value(schema).unwrap(),
json!({"const":{"arbitrary":[1,null]},"future":null})
);
}

79
litellm-rust/crates/llms-types/tests/messages_request.rs
View file

@ -1,79 +0,0 @@
use litellm_llms_types::formats::messages::{ContentBlock, ContentBlockType};
use rstest::rstest;
use serde_json::{Value, json};
#[rstest]
#[case::null(json!(null))]
#[case::number(json!(1))]
#[case::boolean(json!(true))]
#[case::array(json!(["tool_use"]))]
#[case::object(json!({"type": "tool_use"}))]
fn content_block_type_rejects_non_string_json(#[case] value: Value) {
assert!(serde_json::from_value::<ContentBlockType>(value).is_err());
}
#[cfg(feature = "schema")]
#[rstest]
fn content_block_type_schema_remains_a_string() {
let schema = schemars::schema_for!(ContentBlockType).to_value();
assert_eq!(schema.get("type"), Some(&json!("string")));
assert_eq!(
schema.get("title"),
Some(&json!(stringify!(ContentBlockType)))
);
}
#[rstest]
#[case::text("text", ContentBlockType::Text)]
#[case::thinking("thinking", ContentBlockType::Thinking)]
#[case::redacted_thinking("redacted_thinking", ContentBlockType::RedactedThinking)]
#[case::tool_use("tool_use", ContentBlockType::ToolUse)]
#[case::server_tool_use("server_tool_use", ContentBlockType::ServerToolUse)]
#[case::tool_result("tool_result", ContentBlockType::ToolResult)]
#[case::compaction("compaction", ContentBlockType::Compaction)]
#[case::advisor_result("advisor_tool_result", ContentBlockType::AdvisorToolResult)]
#[case::web_search_result("web_search_tool_result", ContentBlockType::WebSearchToolResult)]
#[case::image("image", ContentBlockType::Other("image".into()))]
#[case::document("document", ContentBlockType::Other("document".into()))]
#[case::tool_addition("tool_addition", ContentBlockType::Other("tool_addition".into()))]
#[case::tool_removal("tool_removal", ContentBlockType::Other("tool_removal".into()))]
#[case::advisor("advisor_result", ContentBlockType::Other("advisor_result".into()))]
#[case::web_search_error(
"web_search_tool_result_error",
ContentBlockType::Other("web_search_tool_result_error".into())
)]
#[case::future_block("future_block", ContentBlockType::Other("future_block".into()))]
#[case::case_sensitive("Tool_Use", ContentBlockType::Other("Tool_Use".into()))]
#[case::empty("", ContentBlockType::Other(String::new()))]
fn block_type_is_typed_and_round_trips_with_extra_fields(
#[case] wire: &str,
#[case] expected: ContentBlockType,
) {
let input = json!({"type": wire, "future_field": {"nested": [1, null]}});
let block: ContentBlock = serde_json::from_value(input.clone()).unwrap();
assert_eq!(block.block_type.as_ref(), Some(&expected));
assert_eq!(serde_json::to_value(block).unwrap(), input);
}
#[rstest]
#[case::number(json!(1))]
#[case::boolean(json!(true))]
#[case::array(json!(["text"]))]
#[case::enum_object(json!({"text": null}))]
fn block_type_rejects_non_string_values(#[case] value: Value) {
assert!(serde_json::from_value::<ContentBlock>(json!({"type": value})).is_err());
}
#[rstest]
#[case::same_type(json!({"type": "tool_use"}), ContentBlockType::ToolUse, true)]
#[case::other_type(json!({"type": "tool_result"}), ContentBlockType::ToolUse, false)]
#[case::unknown_type(json!({"type": "future_tool"}), ContentBlockType::ToolUse, false)]
#[case::no_type(json!({"text": "x"}), ContentBlockType::Text, false)]
fn is_type_matches_the_exact_block_type(
#[case] block: Value,
#[case] block_type: ContentBlockType,
#[case] expected: bool,
) {
let block: ContentBlock = serde_json::from_value(block).unwrap();
assert_eq!(block.is_type(block_type), expected);
}

37
litellm-rust/crates/llms-types/tests/messages_streaming.rs
View file

@ -1,37 +0,0 @@
use litellm_llms_types::formats::messages::streaming::MessagesStreamEvent;
use rstest::rstest;
use serde_json::{Value, json};
#[rstest]
#[case::start(json!({
"type": "message_start",
"message": {
"id": "msg", "type": "message", "role": "assistant", "model": "test-model",
"content": [], "stop_reason": null, "stop_sequence": null,
"usage": {"input_tokens": 3, "future_usage": {"count": 9}},
"future_message": [1, 2]
}
}))]
#[case::block(json!({
"type": "content_block_start", "index": 0,
"content_block": {"type": "future", "payload": {"keep": true}}
}))]
#[case::delta(json!({
"type": "message_delta", "delta": {"stop_reason": "end_turn", "future_delta": 42},
"usage": {"output_tokens": 7, "future_usage": true}
}))]
#[case::error(json!({
"type": "error", "error": {"type": "future_error", "message": "failed", "future": 42}
}))]
fn stream_events_preserve_extensible_fields(#[case] wire: Value) {
let event: MessagesStreamEvent = serde_json::from_value(wire.clone()).unwrap();
assert_eq!(serde_json::to_value(event).unwrap(), wire);
}
#[rstest]
#[case::invalid_index(json!({"type": "content_block_stop", "index": "zero"}))]
#[case::missing_delta(json!({"type": "content_block_delta", "index": 0}))]
#[case::unknown_event(json!({"type": "future_event"}))]
fn malformed_or_unrecognized_events_remain_rejected(#[case] wire: Value) {
assert!(serde_json::from_value::<MessagesStreamEvent>(wire).is_err());
}

1512
litellm-rust/crates/llms-types/tests/messages_types.rs
View file

File diff suppressed because it is too large Load diff

82
litellm-rust/crates/llms-types/tests/minimax.rs
View file

@ -1,82 +0,0 @@
use litellm_llms_types::providers::minimax::{
MinimaxMediaDetail, MinimaxMediaSource, MinimaxMessagesContentBlock,
};
use rstest::rstest;
use serde_json::{Value, json};
#[rstest]
#[case::image(json!({"type":"image","source":{"type":"base64","media_type":"image/png","data":"AA==","detail":"low"}}))]
#[case::video(json!({"type":"video","source":{"type":"url","url":"https://example.test/video","detail":"high","fps":1,"max_long_side_pixel":1024,"future":null},"cache_control":{"type":"ephemeral"}}))]
#[case::mid_conversation_system(json!({"type":"mid_conv_system","text":"instruction","future":null}))]
fn provider_content_blocks_round_trip(#[case] wire: Value) {
let block: MinimaxMessagesContentBlock = serde_json::from_value(wire.clone()).unwrap();
match &block {
MinimaxMessagesContentBlock::Image(image) => {
let MinimaxMediaSource::Base64 {
media_type,
data,
options,
} = &image.source
else {
panic!("expected base64 image source");
};
assert_eq!((media_type.as_str(), data.as_str()), ("image/png", "AA=="));
assert_eq!(options.detail, Some(MinimaxMediaDetail::Low));
assert!(options.extra.is_empty());
}
MinimaxMessagesContentBlock::Video(video) => {
let MinimaxMediaSource::Url { url, options } = &video.source else {
panic!("expected URL video source");
};
assert_eq!(url, "https://example.test/video");
assert_eq!(options.detail, Some(MinimaxMediaDetail::High));
assert_eq!(options.fps, Some(1.into()));
assert_eq!(options.max_long_side_pixel, Some(1024));
assert_eq!(options.extra["future"], Value::Null);
assert_eq!(
video.cache_control.as_ref().unwrap().cache_type.as_deref(),
Some("ephemeral")
);
}
MinimaxMessagesContentBlock::MidConvSystem { text, extra } => {
assert_eq!(text, "instruction");
assert_eq!(extra["future"], Value::Null);
}
}
assert_eq!(serde_json::to_value(block).unwrap(), wire);
}
#[rstest]
#[case::missing_source(json!({"type":"video"}))]
#[case::bad_source_tag(json!({"type":"image","source":{"type":"future"}}))]
#[case::url_without_url(json!({"type":"video","source":{"type":"url"}}))]
#[case::base64_without_data(json!({"type":"image","source":{"type":"base64","media_type":"image/png"}}))]
#[case::bad_detail(json!({"type":"image","source":{"type":"url","url":"u","detail":7}}))]
#[case::bad_fps(json!({"type":"video","source":{"type":"url","url":"u","fps":"fast"}}))]
#[case::missing_text(json!({"type":"mid_conv_system"}))]
fn provider_content_rejects_malformed_fields(#[case] wire: Value) {
assert!(serde_json::from_value::<MinimaxMessagesContentBlock>(wire).is_err());
}
#[rstest]
fn partial_media_options_omit_null_optionals() {
let block: MinimaxMessagesContentBlock = serde_json::from_value(json!({
"type":"video",
"source":{"type":"url","url":"u","detail":null,"fps":null,"future":null},
"cache_control":null
}))
.unwrap();
let MinimaxMessagesContentBlock::Video(video) = &block else {
panic!("expected video")
};
let MinimaxMediaSource::Url { options, .. } = &video.source else {
panic!("expected URL source");
};
assert!(options.detail.is_none());
assert!(options.fps.is_none());
assert!(video.cache_control.is_none());
assert_eq!(
serde_json::to_value(block).unwrap(),
json!({"type":"video","source":{"type":"url","url":"u","future":null}})
);
}

140
litellm-rust/crates/llms-types/tests/ocr.rs
View file

@ -1,140 +0,0 @@
use litellm_llms_types::formats::ocr::{LiteLLMOcrResponse, OcrBoundingBox, OcrDocument, OcrPage};
use rstest::rstest;
use serde_json::{Map, Value, json};
#[rstest]
#[case::missing_page_fields(json!({"pages": [{}]}))]
#[case::invalid_markdown(json!({"pages": [{"index": 0, "markdown": false}]}))]
#[case::invalid_image_bounds(json!({"pages": [{"index": 0, "markdown": "", "images": [{"bbox": []}]}]}))]
#[case::fractional_page_count(json!({"usage_info": {"pages_processed": 1.5}}))]
#[case::invalid_table(json!({"tables": [false]}))]
#[case::invalid_key_value_pair(json!({"keyValuePairs": [[]]}))]
#[case::invalid_native_response(json!({"provider_native_response": []}))]
fn normalized_response_rejects_invalid_shared_fields(#[case] fields: Value) {
let payload: Map<String, Value> = json!({"model": "model", "pages": []})
.as_object()
.unwrap()
.iter()
.chain(fields.as_object().unwrap())
.map(|(key, value)| (key.clone(), value.clone()))
.collect();
assert!(serde_json::from_value::<LiteLLMOcrResponse>(Value::Object(payload)).is_err());
}
#[rstest]
fn document_rejects_non_string_provider_fields() {
assert!(
serde_json::from_value::<OcrDocument>(json!({
"type": "image_url", "image_url": "https://example.com/image", "detail": 42
}))
.is_err()
);
}
#[rstest]
#[case::large_integer(json!("9007199254740993.0"), 9_007_199_254_740_993)]
#[case::signed_decimal(json!("+2.000"), 2)]
#[case::separator(json!("1_000"), 1000)]
#[case::boolean(json!(true), 1)]
#[case::integral_float(json!(2.0), 2)]
fn numeric_coercion_preserves_integer_precision(#[case] value: Value, #[case] expected: i64) {
let page: OcrPage = serde_json::from_value(json!({"index": value, "markdown": ""})).unwrap();
assert_eq!(page.index, expected);
assert_eq!(
serde_json::to_value(page).unwrap()["index"],
json!(expected)
);
}
#[rstest]
#[case::exponent(json!("1e2"))]
#[case::missing_integer(json!(".0"))]
#[case::missing_fraction(json!("2."))]
#[case::leading_separator(json!("_2"))]
#[case::repeated_separator(json!("2__0"))]
#[case::fractional_float(json!(2.5))]
#[case::null(json!(null))]
fn page_index_rejects_invalid_integers(#[case] value: Value) {
assert!(serde_json::from_value::<OcrPage>(json!({"index": value, "markdown": ""})).is_err());
}
#[rstest]
#[case::document_url("document_url", "document_name", "application/pdf")]
#[case::image_url("image_url", "detail", "image/png")]
fn document_variants_preserve_provider_fields_when_rewriting_sources(
#[case] kind: &str,
#[case] field: &str,
#[case] mime_type: &str,
#[values(json!("kept"), Value::Null)] extra: Value,
) {
let original = "https://example.com/input";
let replacement = format!("data:{mime_type};base64,AA==");
let document: OcrDocument =
serde_json::from_value(json!({"type": kind, kind: original, field: extra})).unwrap();
assert_eq!(document.source(), original);
assert!(document.is_remote());
let rewritten = document.with_source(replacement.clone());
assert!(!rewritten.is_remote());
assert_eq!(
serde_json::to_value(rewritten).unwrap(),
json!({"type": kind, kind: replacement, field: extra})
);
}
#[rstest]
#[case::absent_native(None)]
#[case::present_native(Some(Map::from_iter([("native".into(), json!({"nested": [null, 1]}))])))]
fn response_serialization_preserves_extensions_and_native_presence(
#[case] native: Option<Map<String, Value>>,
) {
let response = LiteLLMOcrResponse {
extra_fields: Map::from_iter([("provider_field".into(), json!("kept"))]),
provider_native_response: native.clone(),
..LiteLLMOcrResponse::new("model", vec![])
};
let serialized = response.into_json();
assert_eq!(serialized["provider_field"], "kept");
assert_eq!(
serialized.get("provider_native_response").cloned(),
native.clone().map(Value::Object)
);
let decoded: LiteLLMOcrResponse = serde_json::from_value(serialized.clone()).unwrap();
assert_eq!(decoded.provider_native_response, native);
assert_eq!(decoded.into_json(), serialized);
}
#[rstest]
fn bounding_box_exposes_corner_coordinates_and_keeps_extensions() {
let wire = json!({"top_left_x":1,"top_left_y":2.5,"bottom_right_x":30,"bottom_right_y":40,"future":true});
let bounds: OcrBoundingBox = serde_json::from_value(wire.clone()).unwrap();
assert_eq!(bounds.top_left_x, Some(1.into()));
assert_eq!(
bounds
.top_left_y
.as_ref()
.and_then(serde_json::Number::as_f64),
Some(2.5)
);
assert_eq!(bounds.bottom_right_x, Some(30.into()));
assert_eq!(bounds.bottom_right_y, Some(40.into()));
assert_eq!(Value::Object(bounds.extra.clone()), json!({"future":true}));
assert_eq!(serde_json::to_value(bounds).unwrap(), wire);
}
#[rstest]
fn partial_bounding_box_omits_null_corners() {
let bounds: OcrBoundingBox =
serde_json::from_value(json!({"top_left_x":null,"bottom_right_y":4})).unwrap();
assert!(bounds.top_left_x.is_none());
assert_eq!(
serde_json::to_value(bounds).unwrap(),
json!({"bottom_right_y":4})
);
}
#[rstest]
#[case::string_corner(json!({"top_left_x":"1"}))]
#[case::array_corner(json!({"bottom_right_y":[4]}))]
fn bounding_box_rejects_non_numeric_corners(#[case] wire: Value) {
assert!(serde_json::from_value::<OcrBoundingBox>(wire).is_err());
}

717
litellm-rust/crates/llms-types/tests/responses.rs
View file

@ -1,717 +0,0 @@
use litellm_llms_types::formats::chat_completions::{PromptCacheBreakpoint, PromptCacheMode};
use litellm_llms_types::formats::responses::{
ResponsesAdditionalTools, ResponsesApplyPatchCall, ResponsesApplyPatchCallOutput,
ResponsesApplyPatchOperation, ResponsesCodeOutput, ResponsesCompaction,
ResponsesComputerAction, ResponsesComputerCall, ResponsesComputerCallOutput,
ResponsesComputerOutput, ResponsesContentPart, ResponsesCoordinate, ResponsesCustomToolCall,
ResponsesCustomToolCallOutput, ResponsesFunctionCall, ResponsesFunctionCallOutput,
ResponsesImageGenerationCall, ResponsesInputContent, ResponsesLocalShellAction,
ResponsesLocalShellCall, ResponsesLocalShellCallOutput, ResponsesMcpApprovalRequest,
ResponsesMcpApprovalResponse, ResponsesMcpError, ResponsesMcpErrorDetail, ResponsesOutputItem,
ResponsesProgram, ResponsesProgramOutput, ResponsesSafetyCheck, ResponsesShellAction,
ResponsesShellCall, ResponsesShellCallOutput, ResponsesShellEnvironment, ResponsesShellOutcome,
ResponsesShellOutputChunk, ResponsesToolCaller, ResponsesToolOutput, ResponsesToolSearchCall,
ResponsesToolSearchOutput, ResponsesWebSearchAction,
streaming_websocket::{ResponsesEventResponse, ResponsesWsEventType},
};
use litellm_llms_types::recognized::Recognized;
use rstest::rstest;
use serde::{Serialize, de::DeserializeOwned};
use serde_json::{Value, json};
fn round_trip<T>(wire: Value)
where
T: DeserializeOwned + Serialize,
{
let parsed: T = serde_json::from_value(wire.clone()).unwrap();
assert_eq!(serde_json::to_value(parsed).unwrap(), wire);
}
#[rstest]
#[case::create("response.create", ResponsesWsEventType::ResponseCreate)]
#[case::created("response.created", ResponsesWsEventType::ResponseCreated)]
#[case::completed("response.completed", ResponsesWsEventType::ResponseCompleted)]
#[case::failed("response.failed", ResponsesWsEventType::ResponseFailed)]
#[case::incomplete("response.incomplete", ResponsesWsEventType::ResponseIncomplete)]
#[case::error("error", ResponsesWsEventType::Error)]
#[case::unknown(
"response.output_text.delta",
ResponsesWsEventType::Other("response.output_text.delta".to_string())
)]
#[case::empty("", ResponsesWsEventType::Other(String::new()))]
#[case::case_sensitive(
"Response.Completed",
ResponsesWsEventType::Other("Response.Completed".into())
)]
#[case::escaped("future\"\\\n", ResponsesWsEventType::Other("future\"\\\n".into()))]
fn websocket_event_type_round_trips(#[case] wire: &str, #[case] expected: ResponsesWsEventType) {
let serialized = serde_json::to_string(&expected).unwrap();
assert_eq!(serialized, serde_json::to_string(wire).unwrap());
assert_eq!(
serde_json::from_str::<ResponsesWsEventType>(&serialized).unwrap(),
expected
);
}
#[rstest]
#[case::number("17")]
#[case::boolean("true")]
#[case::null("null")]
#[case::array("[]")]
#[case::object("{}")]
fn websocket_event_type_rejects_non_strings(#[case] wire: &str) {
assert!(serde_json::from_str::<ResponsesWsEventType>(wire).is_err());
}
#[cfg(feature = "schema")]
#[rstest]
fn websocket_event_type_schema_is_open_string() {
let schema = schemars::schema_for!(ResponsesWsEventType);
assert_eq!(
schema.to_value().get("type"),
Some(&serde_json::json!("string"))
);
}
#[rstest]
#[case::message(json!({"type":"message","role":"assistant","content":[{"type":"output_text","text":"answer","annotations":[{"type":"url_citation","url":"https://example.test","start_index":0,"end_index":6}]}]}))]
#[case::function_call(json!({"type":"function_call","call_id":"call_1","name":"lookup","arguments":"{\"query\":\"q\"}"}))]
#[case::custom_tool(json!({"type":"custom_tool_call","name":"lookup","input":"q"}))]
#[case::reasoning(json!({"type":"reasoning","summary":[{"type":"summary_text","text":"summary"}],"encrypted_content":"opaque"}))]
#[case::web_search(json!({"type":"web_search_call","action":{"type":"search","queries":["q"],"sources":[{"type":"url","url":"https://example.test"}]}}))]
#[case::file_search(json!({"type":"file_search_call","queries":["q"],"results":[{"file_id":"file_1","score":1,"attributes":{"custom":[1,null]}}]}))]
#[case::code(json!({"type":"code_interpreter_call","outputs":[{"type":"logs","logs":"done"},{"type":"image","url":"https://example.test"}]}))]
#[case::image(json!({"type":"image_generation_call","result":"generated"}))]
#[case::mcp(json!({"type":"mcp_call","server_label":"server","name":"lookup","arguments":"{}","output":"done"}))]
fn output_items_round_trip(#[case] wire: Value) {
let item: ResponsesOutputItem = serde_json::from_value(wire.clone()).unwrap();
match &item {
ResponsesOutputItem::Message(message) => {
assert_eq!(message.role.as_deref(), Some("assistant"));
let Some(content) = &message.content else {
panic!("expected content")
};
let [
ResponsesContentPart::OutputText {
text,
annotations: Some(annotations),
..
},
] = content.as_slice()
else {
panic!("expected output text and annotations")
};
assert_eq!(text, "answer");
let [
litellm_llms_types::formats::responses::ResponsesAnnotation::UrlCitation(citation),
] = annotations.as_slice()
else {
panic!("expected URL citation")
};
assert_eq!(citation.url.as_deref(), Some("https://example.test"));
assert_eq!(citation.start_index, Some(0));
}
ResponsesOutputItem::FunctionCall(call) => {
assert_eq!(call.call_id.as_deref(), Some("call_1"));
assert_eq!(call.name.as_deref(), Some("lookup"));
assert_eq!(call.arguments.as_deref(), Some("{\"query\":\"q\"}"));
}
ResponsesOutputItem::CustomToolCall(call) => {
assert_eq!(call.name.as_deref(), Some("lookup"));
assert_eq!(call.input.as_deref(), Some("q"));
}
ResponsesOutputItem::Reasoning(reasoning) => {
assert_eq!(reasoning.encrypted_content.as_deref(), Some("opaque"));
let Some(summary) = &reasoning.summary else {
panic!("expected summary")
};
let [ResponsesContentPart::SummaryText { text, .. }] = summary.as_slice() else {
panic!("expected summary text")
};
assert_eq!(text, "summary");
}
ResponsesOutputItem::WebSearchCall(call) => {
let Some(ResponsesWebSearchAction::Search {
queries: Some(queries),
sources: Some(sources),
..
}) = &call.action
else {
panic!("expected search action")
};
assert_eq!(queries, &["q"]);
assert_eq!(sources[0].url.as_deref(), Some("https://example.test"));
}
ResponsesOutputItem::FileSearchCall(call) => {
assert_eq!(call.queries.as_deref(), Some(["q".to_owned()].as_slice()));
let Some(results) = &call.results else {
panic!("expected search results")
};
assert_eq!(results[0].file_id.as_deref(), Some("file_1"));
assert_eq!(results[0].score, Some(1.into()));
assert_eq!(
results[0].attributes.as_ref().unwrap()["custom"],
json!([1, null])
);
}
ResponsesOutputItem::CodeInterpreterCall(call) => {
let Some(outputs) = &call.outputs else {
panic!("expected code outputs")
};
let [
ResponsesCodeOutput::Logs { logs, .. },
ResponsesCodeOutput::Image { url, .. },
] = outputs.as_slice()
else {
panic!("expected logs and image")
};
assert_eq!(logs, "done");
assert_eq!(url, "https://example.test");
}
ResponsesOutputItem::ImageGenerationCall(call) => {
assert_eq!(call.result.as_deref(), Some("generated"))
}
ResponsesOutputItem::McpCall(call) => {
assert_eq!(call.server_label.as_deref(), Some("server"));
assert_eq!(call.name.as_deref(), Some("lookup"));
assert_eq!(call.arguments.as_deref(), Some("{}"));
assert_eq!(call.output.as_deref(), Some("done"));
}
other => panic!("unexpected variant {other:?}"),
}
assert_eq!(serde_json::to_value(item).unwrap(), wire);
}
#[rstest]
fn nested_event_response_exposes_typed_output_and_preserves_extensions() {
let wire = json!({
"id":"response_1",
"model":"example-model",
"status":"completed",
"output":[{"type":"function_call","call_id":"call_1","name":"lookup","arguments":"{}","extension":true}],
"extension":{"nested":[1,null]}
});
let response: ResponsesEventResponse = serde_json::from_value(wire.clone()).unwrap();
assert_eq!(response.id.as_deref(), Some("response_1"));
assert_eq!(response.model.as_deref(), Some("example-model"));
let Some(output) = &response.output else {
panic!("expected typed output");
};
let [Recognized::Known(ResponsesOutputItem::FunctionCall(call))] = output.as_slice() else {
panic!("expected function call");
};
assert_eq!(call.name.as_deref(), Some("lookup"));
assert_eq!(call.arguments.as_deref(), Some("{}"));
assert_eq!(serde_json::to_value(response).unwrap(), wire);
}
#[rstest]
#[case::empty(json!({}))]
#[case::partial(json!({"id":"response_1","output":[]}))]
fn nested_event_response_accepts_partial_metadata(#[case] wire: Value) {
round_trip::<ResponsesEventResponse>(wire);
}
#[rstest]
#[case::wrong_model(json!({"model":7}))]
#[case::wrong_status(json!({"status":false}))]
#[case::wrong_output(json!({"output":{}}))]
fn event_response_rejects_malformed_typed_fields(#[case] wire: Value) {
assert!(serde_json::from_value::<ResponsesEventResponse>(wire).is_err());
}
#[rstest]
#[case::unknown_type(json!({"type":"future_item","id":"item_1","payload":[1,null]}))]
#[case::missing_tag(json!({"id":"item_1"}))]
#[case::malformed_known(json!({"type":"message","content":[{"type":"output_text","text":7}]}))]
fn event_response_keeps_unmodeled_output_items_beside_typed_ones(#[case] item: Value) {
let wire = json!({"output":[{"type":"function_call","name":"lookup"}, item.clone()]});
let response: ResponsesEventResponse = serde_json::from_value(wire.clone()).unwrap();
let Some(
[
Recognized::Known(ResponsesOutputItem::FunctionCall(call)),
Recognized::Unrecognized(kept),
],
) = response.output.as_deref()
else {
panic!("expected one typed item and one preserved item");
};
assert_eq!(call.name.as_deref(), Some("lookup"));
assert_eq!(kept, &item);
assert_eq!(serde_json::to_value(response).unwrap(), wire);
}
#[rstest]
#[case::find_in_page(
json!({"type":"find_in_page","url":"https://example.test","pattern":"needle"}),
json!({"type":"find_in_page","url":"https://example.test","pattern":"needle"})
)]
#[case::legacy_find(
json!({"type":"find","url":"https://example.test","pattern":"needle"}),
json!({"type":"find_in_page","url":"https://example.test","pattern":"needle"})
)]
fn web_search_find_action_exposes_url_and_pattern(#[case] wire: Value, #[case] serialized: Value) {
let action: ResponsesWebSearchAction = serde_json::from_value(wire).unwrap();
let ResponsesWebSearchAction::FindInPage {
url,
pattern,
extra,
} = &action
else {
panic!("expected find_in_page action");
};
assert_eq!(url, "https://example.test");
assert_eq!(pattern, "needle");
assert!(extra.is_empty());
assert_eq!(serde_json::to_value(action).unwrap(), serialized);
}
#[rstest]
#[case::with_url(json!({"type":"open_page","url":"https://example.test"}), Some("https://example.test"))]
#[case::without_url(json!({"type":"open_page"}), None)]
fn web_search_open_page_url_is_optional(#[case] wire: Value, #[case] expected: Option<&str>) {
let action: ResponsesWebSearchAction = serde_json::from_value(wire.clone()).unwrap();
let ResponsesWebSearchAction::OpenPage { url, .. } = &action else {
panic!("expected open_page action");
};
assert_eq!(url.as_deref(), expected);
assert_eq!(serde_json::to_value(action).unwrap(), wire);
}
#[rstest]
#[case::find_pattern(json!({"type":"find_in_page","url":"u","pattern":false}))]
#[case::find_missing_url(json!({"type":"find_in_page","pattern":"p"}))]
#[case::open_page_url(json!({"type":"open_page","url":7}))]
fn web_search_action_rejects_malformed_fields(#[case] wire: Value) {
assert!(serde_json::from_value::<ResponsesWebSearchAction>(wire).is_err());
}
#[rstest]
fn event_response_optional_fields_omit_missing_and_null() {
let response: ResponsesEventResponse = serde_json::from_value(json!({
"id":null,"model":null,"status":null,"output":null,"future":null
}))
.unwrap();
assert!(response.id.is_none());
assert!(response.model.is_none());
assert!(response.status.is_none());
assert!(response.output.is_none());
assert_eq!(
serde_json::to_value(response).unwrap(),
json!({"future":null})
);
}
#[rstest]
#[case::refusal(json!({"type":"refusal","refusal":"refused","future":null}), ResponsesContentPart::Refusal { refusal:"refused".into(), extra:serde_json::Map::from_iter([("future".into(), Value::Null)]) })]
#[case::reasoning(json!({"type":"reasoning_text","text":"reason"}), ResponsesContentPart::ReasoningText { text:"reason".into(), extra:Default::default() })]
fn content_parts_expose_typed_variants(
#[case] wire: Value,
#[case] expected: ResponsesContentPart,
) {
let content: ResponsesContentPart = serde_json::from_value(wire.clone()).unwrap();
assert_eq!(content, expected);
assert_eq!(serde_json::to_value(content).unwrap(), wire);
}
#[rstest]
fn mcp_discovery_exposes_tools_and_nested_schemas() {
let wire = json!({
"type":"mcp_list_tools","id":"item_1","server_label":"tools",
"tools":[{"name":"lookup","description":"Look up a value","input_schema":{"type":"object","properties":{"query":{"type":"string"}}},"annotations":{"read_only":false},"future":null}],
"extension":[1,null]
});
let item: ResponsesOutputItem = serde_json::from_value(wire.clone()).unwrap();
let ResponsesOutputItem::McpListTools(discovery) = &item else {
panic!("expected discovery")
};
assert_eq!(discovery.server_label.as_deref(), Some("tools"));
let tool = &discovery.tools.as_ref().unwrap()[0];
assert_eq!(tool.name.as_deref(), Some("lookup"));
let Some(litellm_llms_types::json_schema::JsonSchema::Object(schema)) = &tool.input_schema
else {
panic!("expected tool schema")
};
assert!(schema.properties.as_ref().unwrap().contains_key("query"));
assert_eq!(tool.annotations, Some(json!({"read_only":false})));
assert_eq!(tool.extra["future"], Value::Null);
assert_eq!(serde_json::to_value(item).unwrap(), wire);
}
#[rstest]
#[case::partial(json!({"server_label":"tools","tools":[]}))]
#[case::failure(json!({"server_label":"tools","error":"unavailable"}))]
fn mcp_discovery_accepts_partial_payloads(#[case] wire: Value) {
let discovery: litellm_llms_types::formats::responses::ResponsesMcpListTools =
serde_json::from_value(wire.clone()).unwrap();
assert_eq!(discovery.server_label.as_deref(), Some("tools"));
assert!(discovery.id.is_none());
assert_eq!(serde_json::to_value(discovery).unwrap(), wire);
}
#[rstest]
#[case::message(json!("unavailable"), ResponsesMcpError::Message("unavailable".into()))]
#[case::protocol(json!({"type":"mcp_protocol_error","code":-32600,"message":"invalid","extension":null}), ResponsesMcpError::Detail(ResponsesMcpErrorDetail::McpProtocolError {code:-32600,message:"invalid".into(),extra:serde_json::Map::from_iter([("extension".into(), Value::Null)])}))]
#[case::http(json!({"type":"http_error","code":503,"message":"unavailable"}), ResponsesMcpError::Detail(ResponsesMcpErrorDetail::HttpError {code:503,message:"unavailable".into(),extra:Default::default()}))]
#[case::tool(json!({"type":"mcp_tool_execution_error","content":{"arbitrary":[1,null]}}), ResponsesMcpError::Detail(ResponsesMcpErrorDetail::McpToolExecutionError {content:json!({"arbitrary":[1,null]}),extra:Default::default()}))]
fn mcp_call_errors_expose_typed_variants(#[case] wire: Value, #[case] expected: ResponsesMcpError) {
let error: ResponsesMcpError = serde_json::from_value(wire.clone()).unwrap();
assert_eq!(error, expected);
assert_eq!(serde_json::to_value(&error).unwrap(), wire);
let call: ResponsesOutputItem =
serde_json::from_value(json!({"type":"mcp_call","error":wire})).unwrap();
let ResponsesOutputItem::McpCall(call) = call else {
panic!("expected call")
};
assert_eq!(call.error, Some(error));
}
#[rstest]
#[case::tools_not_array(json!({"type":"mcp_list_tools","tools":{}}))]
#[case::invalid_name(json!({"type":"mcp_list_tools","tools":[{"name":7}]}))]
#[case::invalid_schema(json!({"type":"mcp_list_tools","tools":[{"input_schema":{"properties":{"query":7}}}]}))]
#[case::invalid_error_code(json!({"type":"mcp_call","error":{"type":"http_error","code":"503","message":"unavailable"}}))]
#[case::missing_error_content(json!({"type":"mcp_call","error":{"type":"mcp_tool_execution_error"}}))]
#[case::unknown_error_tag(json!({"type":"mcp_call","error":{"type":"future"}}))]
fn mcp_output_rejects_malformed_known_fields(#[case] wire: Value) {
assert!(serde_json::from_value::<ResponsesOutputItem>(wire).is_err());
}
fn text(value: &str) -> Option<String> {
Some(value.to_owned())
}
fn program_caller() -> Option<ResponsesToolCaller> {
Some(ResponsesToolCaller::Program {
caller_id: "prog_1".into(),
extra: Default::default(),
})
}
fn direct_caller() -> Option<ResponsesToolCaller> {
Some(ResponsesToolCaller::Direct {
extra: Default::default(),
})
}
fn safety_check() -> ResponsesSafetyCheck {
ResponsesSafetyCheck {
id: text("sc_1"),
code: text("malicious_instructions"),
message: text("check"),
..Default::default()
}
}
#[rstest]
#[case::function_call(
json!({"type":"function_call","id":"fc_1","status":"completed","call_id":"call_1","name":"lookup","arguments":"{}","namespace":"ns","async":true,"caller":{"type":"program","caller_id":"prog_1"}}),
ResponsesOutputItem::FunctionCall(ResponsesFunctionCall {
id: text("fc_1"), status: text("completed"), call_id: text("call_1"), name: text("lookup"),
arguments: text("{}"), namespace: text("ns"), r#async: Some(true), caller: program_caller(), ..Default::default()
})
)]
#[case::custom_tool_call(
json!({"type":"custom_tool_call","call_id":"call_1","name":"lookup","input":"q","namespace":"ns","async":false,"caller":{"type":"direct"}}),
ResponsesOutputItem::CustomToolCall(ResponsesCustomToolCall {
call_id: text("call_1"), name: text("lookup"), input: text("q"), namespace: text("ns"),
r#async: Some(false), caller: direct_caller(), ..Default::default()
})
)]
#[case::image_generation_call(
json!({"type":"image_generation_call","id":"ig_1","status":"completed","result":"b64","action":"edit","background":"opaque","output_format":"webp","quality":"high","revised_prompt":"a cat","size":"1536x864"}),
ResponsesOutputItem::ImageGenerationCall(ResponsesImageGenerationCall {
id: text("ig_1"), status: text("completed"), result: text("b64"), action: text("edit"), background: text("opaque"),
output_format: text("webp"), quality: text("high"), revised_prompt: text("a cat"), size: text("1536x864"), ..Default::default()
})
)]
#[case::function_call_output_text(
json!({"type":"function_call_output","id":"fco_1","status":"completed","call_id":"call_1","output":"done","caller":{"type":"direct"},"created_by":"user_1","name":"lookup","namespace":"ns"}),
ResponsesOutputItem::FunctionCallOutput(ResponsesFunctionCallOutput {
id: text("fco_1"), status: text("completed"), call_id: text("call_1"), output: Some(ResponsesToolOutput::Text("done".into())),
caller: direct_caller(), created_by: text("user_1"), name: text("lookup"), namespace: text("ns"), ..Default::default()
})
)]
#[case::function_call_output_content(
json!({"type":"function_call_output","call_id":"call_1","output":[
{"type":"input_text","text":"t","prompt_cache_breakpoint":{"mode":"explicit"}},
{"type":"input_image","detail":"low","file_id":"file_1","image_url":"https://example.test/i.png"},
{"type":"input_file","detail":"high","file_data":"data","file_id":"file_2","file_url":"https://example.test/f","filename":"f.pdf"}
]}),
ResponsesOutputItem::FunctionCallOutput(ResponsesFunctionCallOutput {
call_id: text("call_1"),
output: Some(ResponsesToolOutput::Content(vec![
ResponsesInputContent::InputText {
text: "t".into(),
prompt_cache_breakpoint: Some(PromptCacheBreakpoint { mode: PromptCacheMode::Explicit, extra: Default::default() }),
extra: Default::default(),
},
ResponsesInputContent::InputImage {
detail: "low".into(), file_id: text("file_1"), image_url: text("https://example.test/i.png"),
prompt_cache_breakpoint: None, extra: Default::default(),
},
ResponsesInputContent::InputFile {
detail: text("high"), file_data: text("data"), file_id: text("file_2"), file_url: text("https://example.test/f"),
filename: text("f.pdf"), prompt_cache_breakpoint: None, extra: Default::default(),
},
])),
..Default::default()
})
)]
#[case::custom_tool_call_output(
json!({"type":"custom_tool_call_output","id":"cto_1","status":"completed","call_id":"call_1","output":[{"type":"input_text","text":"t"}],"caller":{"type":"program","caller_id":"prog_1"},"created_by":"user_1"}),
ResponsesOutputItem::CustomToolCallOutput(ResponsesCustomToolCallOutput {
id: text("cto_1"), status: text("completed"), call_id: text("call_1"),
output: Some(ResponsesToolOutput::Content(vec![ResponsesInputContent::InputText {
text: "t".into(), prompt_cache_breakpoint: None, extra: Default::default(),
}])),
caller: program_caller(), created_by: text("user_1"), ..Default::default()
})
)]
#[case::computer_call(
json!({"type":"computer_call","id":"cu_1","status":"completed","call_id":"call_1",
"pending_safety_checks":[{"id":"sc_1","code":"malicious_instructions","message":"check"}],
"action":{"type":"click","button":"left","x":1,"y":2,"keys":["shift"]},
"actions":[
{"type":"double_click","x":3,"y":4},
{"type":"drag","path":[{"x":5,"y":6},{"x":7,"y":8}],"keys":["ctrl"]},
{"type":"keypress","keys":["enter"]},
{"type":"move","x":-1,"y":9},
{"type":"screenshot"},
{"type":"scroll","scroll_x":0,"scroll_y":-10,"x":11,"y":12},
{"type":"type","text":"hello"},
{"type":"wait"}
]}),
ResponsesOutputItem::ComputerCall(ResponsesComputerCall {
id: text("cu_1"), status: text("completed"), call_id: text("call_1"), pending_safety_checks: Some(vec![safety_check()]),
action: Some(ResponsesComputerAction::Click { button: "left".into(), x: 1, y: 2, keys: Some(vec!["shift".into()]), extra: Default::default() }),
actions: Some(vec![
ResponsesComputerAction::DoubleClick { x: 3, y: 4, keys: None, extra: Default::default() },
ResponsesComputerAction::Drag {
path: vec![
ResponsesCoordinate { x: 5, y: 6, extra: Default::default() },
ResponsesCoordinate { x: 7, y: 8, extra: Default::default() },
],
keys: Some(vec!["ctrl".into()]),
extra: Default::default(),
},
ResponsesComputerAction::Keypress { keys: vec!["enter".into()], extra: Default::default() },
ResponsesComputerAction::Move { x: -1, y: 9, keys: None, extra: Default::default() },
ResponsesComputerAction::Screenshot { extra: Default::default() },
ResponsesComputerAction::Scroll { scroll_x: 0, scroll_y: -10, x: 11, y: 12, keys: None, extra: Default::default() },
ResponsesComputerAction::Type { text: "hello".into(), extra: Default::default() },
ResponsesComputerAction::Wait { extra: Default::default() },
]),
..Default::default()
})
)]
#[case::computer_call_output(
json!({"type":"computer_call_output","id":"cuo_1","status":"completed","call_id":"call_1","output":{"type":"computer_screenshot","file_id":"file_1","image_url":"https://example.test/s.png"},"acknowledged_safety_checks":[{"id":"sc_1","code":"malicious_instructions","message":"check"}],"created_by":"user_1"}),
ResponsesOutputItem::ComputerCallOutput(ResponsesComputerCallOutput {
id: text("cuo_1"), status: text("completed"), call_id: text("call_1"),
output: Some(ResponsesComputerOutput::ComputerScreenshot { file_id: text("file_1"), image_url: text("https://example.test/s.png"), extra: Default::default() }),
acknowledged_safety_checks: Some(vec![safety_check()]), created_by: text("user_1"), ..Default::default()
})
)]
#[case::program(
json!({"type":"program","id":"prog_item","call_id":"prog_1","code":"run()","fingerprint":"fp"}),
ResponsesOutputItem::Program(ResponsesProgram {
id: text("prog_item"), call_id: text("prog_1"), code: text("run()"), fingerprint: text("fp"), ..Default::default()
})
)]
#[case::program_output(
json!({"type":"program_output","id":"po_1","status":"completed","call_id":"prog_1","result":"42"}),
ResponsesOutputItem::ProgramOutput(ResponsesProgramOutput {
id: text("po_1"), status: text("completed"), call_id: text("prog_1"), result: text("42"), ..Default::default()
})
)]
#[case::tool_search_call(
json!({"type":"tool_search_call","id":"ts_1","status":"completed","call_id":"call_1","arguments":{"query":["weather",null]},"execution":"server","created_by":"user_1"}),
ResponsesOutputItem::ToolSearchCall(ResponsesToolSearchCall {
id: text("ts_1"), status: text("completed"), call_id: text("call_1"), arguments: Some(json!({"query":["weather",null]})),
execution: text("server"), created_by: text("user_1"), ..Default::default()
})
)]
#[case::tool_search_output(
json!({"type":"tool_search_output","id":"tso_1","status":"completed","call_id":"call_1","execution":"client","tools":[{"type":"function","name":"lookup","parameters":null,"strict":true}],"created_by":"user_1"}),
ResponsesOutputItem::ToolSearchOutput(ResponsesToolSearchOutput {
id: text("tso_1"), status: text("completed"), call_id: text("call_1"), execution: text("client"),
tools: Some(vec![serde_json::Map::from_iter([
("type".into(), json!("function")), ("name".into(), json!("lookup")),
("parameters".into(), Value::Null), ("strict".into(), json!(true)),
])]),
created_by: text("user_1"), ..Default::default()
})
)]
#[case::additional_tools(
json!({"type":"additional_tools","id":"at_1","role":"developer","tools":[{"type":"local_shell"}]}),
ResponsesOutputItem::AdditionalTools(ResponsesAdditionalTools {
id: text("at_1"), role: text("developer"),
tools: Some(vec![serde_json::Map::from_iter([("type".into(), json!("local_shell"))])]), ..Default::default()
})
)]
#[case::compaction(
json!({"type":"compaction","id":"cmp_1","encrypted_content":"opaque","created_by":"user_1"}),
ResponsesOutputItem::Compaction(ResponsesCompaction {
id: text("cmp_1"), encrypted_content: text("opaque"), created_by: text("user_1"), ..Default::default()
})
)]
#[case::local_shell_call(
json!({"type":"local_shell_call","id":"ls_1","status":"completed","call_id":"call_1","action":{"type":"exec","command":["ls","-a"],"env":{"HOME":"/home/u"},"timeout_ms":1000,"user":"u","working_directory":"/tmp"}}),
ResponsesOutputItem::LocalShellCall(ResponsesLocalShellCall {
id: text("ls_1"), status: text("completed"), call_id: text("call_1"),
action: Some(ResponsesLocalShellAction::Exec {
command: vec!["ls".into(), "-a".into()], env: [("HOME".into(), "/home/u".into())].into(),
timeout_ms: Some(1000), user: text("u"), working_directory: text("/tmp"), extra: Default::default(),
}),
..Default::default()
})
)]
#[case::local_shell_call_output(
json!({"type":"local_shell_call_output","id":"lso_1","status":"completed","output":"{\"stdout\":\"x\"}"}),
ResponsesOutputItem::LocalShellCallOutput(ResponsesLocalShellCallOutput {
id: text("lso_1"), status: text("completed"), output: text("{\"stdout\":\"x\"}"), ..Default::default()
})
)]
#[case::shell_call_local(
json!({"type":"shell_call","id":"sh_1","status":"in_progress","call_id":"call_1","action":{"commands":["ls"],"max_output_length":100,"timeout_ms":500},"environment":{"type":"local"},"caller":{"type":"direct"},"created_by":"user_1"}),
ResponsesOutputItem::ShellCall(ResponsesShellCall {
id: text("sh_1"), status: text("in_progress"), call_id: text("call_1"),
action: Some(ResponsesShellAction { commands: Some(vec!["ls".into()]), max_output_length: Some(100), timeout_ms: Some(500), ..Default::default() }),
environment: Some(ResponsesShellEnvironment::Local { extra: Default::default() }),
caller: direct_caller(), created_by: text("user_1"), ..Default::default()
})
)]
#[case::shell_call_container(
json!({"type":"shell_call","call_id":"call_1","environment":{"type":"container_reference","container_id":"cntr_1"}}),
ResponsesOutputItem::ShellCall(ResponsesShellCall {
call_id: text("call_1"),
environment: Some(ResponsesShellEnvironment::ContainerReference { container_id: "cntr_1".into(), extra: Default::default() }),
..Default::default()
})
)]
#[case::shell_call_output(
json!({"type":"shell_call_output","id":"sho_1","status":"completed","call_id":"call_1","max_output_length":100,"output":[
{"outcome":{"type":"exit","exit_code":2},"stdout":"out","stderr":"err","created_by":"user_1"},
{"outcome":{"type":"timeout"},"stdout":"","stderr":""}
],"caller":{"type":"program","caller_id":"prog_1"},"created_by":"user_1"}),
ResponsesOutputItem::ShellCallOutput(ResponsesShellCallOutput {
id: text("sho_1"), status: text("completed"), call_id: text("call_1"), max_output_length: Some(100),
output: Some(vec![
ResponsesShellOutputChunk {
outcome: Some(ResponsesShellOutcome::Exit { exit_code: 2, extra: Default::default() }),
stdout: text("out"), stderr: text("err"), created_by: text("user_1"), ..Default::default()
},
ResponsesShellOutputChunk {
outcome: Some(ResponsesShellOutcome::Timeout { extra: Default::default() }),
stdout: text(""), stderr: text(""), ..Default::default()
},
]),
caller: program_caller(), created_by: text("user_1"), ..Default::default()
})
)]
#[case::apply_patch_create(
json!({"type":"apply_patch_call","id":"ap_1","status":"completed","call_id":"call_1","operation":{"type":"create_file","path":"a.txt","diff":"+a"},"caller":{"type":"direct"},"created_by":"user_1"}),
ResponsesOutputItem::ApplyPatchCall(ResponsesApplyPatchCall {
id: text("ap_1"), status: text("completed"), call_id: text("call_1"),
operation: Some(ResponsesApplyPatchOperation::CreateFile { path: "a.txt".into(), diff: "+a".into(), extra: Default::default() }),
caller: direct_caller(), created_by: text("user_1"), ..Default::default()
})
)]
#[case::apply_patch_delete(
json!({"type":"apply_patch_call","call_id":"call_1","operation":{"type":"delete_file","path":"a.txt"}}),
ResponsesOutputItem::ApplyPatchCall(ResponsesApplyPatchCall {
call_id: text("call_1"),
operation: Some(ResponsesApplyPatchOperation::DeleteFile { path: "a.txt".into(), extra: Default::default() }),
..Default::default()
})
)]
#[case::apply_patch_update(
json!({"type":"apply_patch_call","call_id":"call_1","operation":{"type":"update_file","path":"a.txt","diff":"-a\n+b"}}),
ResponsesOutputItem::ApplyPatchCall(ResponsesApplyPatchCall {
call_id: text("call_1"),
operation: Some(ResponsesApplyPatchOperation::UpdateFile { path: "a.txt".into(), diff: "-a\n+b".into(), extra: Default::default() }),
..Default::default()
})
)]
#[case::apply_patch_call_output(
json!({"type":"apply_patch_call_output","id":"apo_1","status":"failed","call_id":"call_1","output":"conflict","caller":{"type":"direct"},"created_by":"user_1"}),
ResponsesOutputItem::ApplyPatchCallOutput(ResponsesApplyPatchCallOutput {
id: text("apo_1"), status: text("failed"), call_id: text("call_1"), output: text("conflict"),
caller: direct_caller(), created_by: text("user_1"), ..Default::default()
})
)]
#[case::mcp_approval_request(
json!({"type":"mcp_approval_request","id":"apr_1","server_label":"s","name":"n","arguments":"{}"}),
ResponsesOutputItem::McpApprovalRequest(ResponsesMcpApprovalRequest {
id: text("apr_1"), server_label: text("s"), name: text("n"), arguments: text("{}"), ..Default::default()
})
)]
#[case::mcp_approval_response(
json!({"type":"mcp_approval_response","id":"aprr_1","approval_request_id":"apr_1","approve":false,"reason":"denied"}),
ResponsesOutputItem::McpApprovalResponse(ResponsesMcpApprovalResponse {
id: text("aprr_1"), approval_request_id: text("apr_1"), approve: Some(false), reason: text("denied"), ..Default::default()
})
)]
fn documented_output_items_parse_into_typed_variants(
#[case] wire: Value,
#[case] expected: ResponsesOutputItem,
) {
let item: ResponsesOutputItem = serde_json::from_value(wire.clone()).unwrap();
assert_eq!(item, expected);
assert_eq!(serde_json::to_value(item).unwrap(), wire);
}
#[rstest]
#[case::compaction(json!({"type":"compaction","id":"cmp_1","encrypted_content":"opaque"}))]
#[case::approval(json!({"type":"mcp_approval_request","id":"apr_1","server_label":"s","name":"n","arguments":"{}"}))]
#[case::shell(json!({"type":"shell_call","call_id":"call_1","action":{"commands":["ls"]}}))]
fn event_response_types_documented_output_items(#[case] item: Value) {
let wire = json!({"output":[item.clone()]});
let response: ResponsesEventResponse = serde_json::from_value(wire.clone()).unwrap();
let Some([Recognized::Known(known)]) = response.output.as_deref() else {
panic!("expected one typed item");
};
assert_eq!(
known,
&serde_json::from_value::<ResponsesOutputItem>(item).unwrap()
);
assert_eq!(serde_json::to_value(response).unwrap(), wire);
}
#[rstest]
#[case::unknown_caller(json!({"type":"function_call","caller":{"type":"future"}}))]
#[case::program_caller_missing_id(json!({"type":"shell_call","caller":{"type":"program"}}))]
#[case::untagged_caller(json!({"type":"custom_tool_call","caller":{}}))]
#[case::async_not_bool(json!({"type":"function_call","async":"yes"}))]
#[case::output_not_text_or_list(json!({"type":"function_call_output","output":{"text":"t"}}))]
#[case::unknown_input_content(json!({"type":"function_call_output","output":[{"type":"input_audio"}]}))]
#[case::input_text_missing_text(json!({"type":"custom_tool_call_output","output":[{"type":"input_text"}]}))]
#[case::input_image_missing_detail(json!({"type":"custom_tool_call_output","output":[{"type":"input_image","file_id":"f"}]}))]
#[case::bad_cache_mode(json!({"type":"custom_tool_call_output","output":[{"type":"input_text","text":"t","prompt_cache_breakpoint":{"mode":"implicit"}}]}))]
#[case::unknown_computer_action(json!({"type":"computer_call","action":{"type":"teleport"}}))]
#[case::click_missing_button(json!({"type":"computer_call","action":{"type":"click","x":1,"y":2}}))]
#[case::click_fractional_x(json!({"type":"computer_call","action":{"type":"click","button":"left","x":1.5,"y":2}}))]
#[case::drag_point_missing_y(json!({"type":"computer_call","actions":[{"type":"drag","path":[{"x":1}]}]}))]
#[case::keypress_keys_not_list(json!({"type":"computer_call","actions":[{"type":"keypress","keys":"enter"}]}))]
#[case::safety_check_not_object(json!({"type":"computer_call","pending_safety_checks":["sc_1"]}))]
#[case::unknown_screenshot_tag(json!({"type":"computer_call_output","output":{"type":"screenshot"}}))]
#[case::untagged_screenshot(json!({"type":"computer_call_output","output":{"file_id":"f"}}))]
#[case::tool_not_object(json!({"type":"tool_search_output","tools":["lookup"]}))]
#[case::unknown_local_shell_action(json!({"type":"local_shell_call","action":{"type":"spawn","command":[],"env":{}}}))]
#[case::local_shell_missing_env(json!({"type":"local_shell_call","action":{"type":"exec","command":["ls"]}}))]
#[case::local_shell_env_value(json!({"type":"local_shell_call","action":{"type":"exec","command":["ls"],"env":{"A":1}}}))]
#[case::shell_commands_not_list(json!({"type":"shell_call","action":{"commands":"ls"}}))]
#[case::unknown_shell_environment(json!({"type":"shell_call","environment":{"type":"container_auto"}}))]
#[case::container_missing_id(json!({"type":"shell_call","environment":{"type":"container_reference"}}))]
#[case::unknown_shell_outcome(json!({"type":"shell_call_output","output":[{"outcome":{"type":"killed"}}]}))]
#[case::exit_missing_code(json!({"type":"shell_call_output","output":[{"outcome":{"type":"exit"}}]}))]
#[case::negative_max_output(json!({"type":"shell_call_output","max_output_length":-1}))]
#[case::unknown_patch_operation(json!({"type":"apply_patch_call","operation":{"type":"rename_file","path":"a"}}))]
#[case::update_missing_diff(json!({"type":"apply_patch_call","operation":{"type":"update_file","path":"a"}}))]
#[case::approve_not_bool(json!({"type":"mcp_approval_response","approve":"true"}))]
#[case::compaction_content_not_string(json!({"type":"compaction","encrypted_content":7}))]
#[case::program_code_not_string(json!({"type":"program","code":["run()"]}))]
fn output_items_reject_malformed_typed_fields(#[case] wire: Value) {
assert!(serde_json::from_value::<ResponsesOutputItem>(wire).is_err());
}

86
litellm-rust/crates/llms-types/tests/serde_compat.rs
View file

@ -1,86 +0,0 @@
use litellm_llms_types::serde_compat::{FiniteF64, LaxI64};
use rstest::rstest;
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use serde_with::serde_as;
#[serde_as]
#[derive(Debug, Deserialize, Serialize, PartialEq)]
struct Numbers {
#[serde_as(deserialize_as = "Option<Vec<LaxI64>>")]
integers: Option<Vec<i64>>,
#[serde_as(deserialize_as = "Option<FiniteF64>")]
float: Option<f64>,
}
#[rstest]
fn adapters_compose_and_serialize_as_numbers() {
let numbers: Numbers = serde_json::from_value(json!({
"integers": ["9007199254740993.0", "1_000", " +2.000 ", 3.0, true],
"float": " 1.5 "
}))
.unwrap();
assert_eq!(
serde_json::to_value(numbers).unwrap(),
json!({"integers": [9_007_199_254_740_993_i64, 1000, 2, 3, 1], "float": 1.5})
);
}
#[rstest]
#[case::missing(json!({}))]
#[case::null(json!({"integers": null, "float": null}))]
fn optional_adapters_accept_missing_and_null_fields(#[case] input: Value) {
assert_eq!(
serde_json::from_value::<Numbers>(input).unwrap(),
Numbers {
integers: None,
float: None
}
);
}
#[rstest]
#[case::minimum(json!(i64::MIN), i64::MIN)]
#[case::maximum(json!(i64::MAX), i64::MAX)]
#[case::maximum_string(json!(i64::MAX.to_string()), i64::MAX)]
fn integers_preserve_bounds(#[case] input: Value, #[case] expected: i64) {
let numbers: Numbers = serde_json::from_value(json!({"integers": [input]})).unwrap();
assert_eq!(numbers.integers, Some(vec![expected]));
}
#[rstest]
#[case::unsigned_maximum(json!(u64::MAX))]
#[case::above_maximum(json!(9_223_372_036_854_775_808_u64))]
#[case::float_above_maximum(json!(9_223_372_036_854_775_808.0))]
#[case::below_minimum(json!("-9223372036854775809"))]
#[case::precise_fraction(json!("1.0000000000000001"))]
#[case::exponent(json!("1e3"))]
#[case::missing_fraction(json!("2."))]
#[case::missing_integer(json!(".0"))]
#[case::leading_separator(json!("_2"))]
#[case::repeated_separator(json!("2__0"))]
#[case::fraction(json!(2.5))]
#[case::null(json!(null))]
#[case::object(json!({}))]
fn integers_reject_invalid_values(#[case] input: Value) {
assert!(serde_json::from_value::<Numbers>(json!({"integers": [input]})).is_err());
}
#[rstest]
#[case::nan(json!("NaN"))]
#[case::positive_infinity(json!("inf"))]
#[case::negative_infinity(json!("-inf"))]
#[case::overflow(json!("1e999"))]
#[case::array(json!([]))]
fn floats_reject_nonfinite_and_invalid_values(#[case] input: Value) {
assert!(serde_json::from_value::<Numbers>(json!({"float": input})).is_err());
}
#[rstest]
#[case::integer(json!(2), 2.0)]
#[case::float(json!(2.5), 2.5)]
#[case::boolean(json!(true), 1.0)]
fn floats_accept_finite_numbers(#[case] input: Value, #[case] expected: f64) {
let numbers: Numbers = serde_json::from_value(json!({"float": input})).unwrap();
assert_eq!(numbers.float, Some(expected));
}

32
litellm-rust/crates/llms-types/tests/wire_type.rs
View file

@ -1,32 +0,0 @@
use litellm_llms_types::formats::chat_completions::ChatMessage;
use rstest::rstest;
use serde_json::json;
#[rstest]
fn wire_type_preserves_serialization() {
let message = ChatMessage {
role: "user".to_owned(),
content: None,
name: None,
extra: Default::default(),
};
assert_eq!(
serde_json::to_value(message).unwrap(),
json!({"role": "user"})
);
}
#[cfg(feature = "schema")]
#[rstest]
fn wire_type_supports_schema_generation() {
let schema = schemars::schema_for!(ChatMessage);
assert!(
schema
.to_value()
.get("properties")
.and_then(serde_json::Value::as_object)
.is_some_and(|properties| properties.contains_key("role"))
);
}

82
litellm-rust/crates/llms/AGENTS.md
View file

@ -1,39 +1,71 @@
litellm-llms mirrors `litellm/llms/`: base config traits, provider transformations, and the OCR request handler in `base_llm/ocr/handler.rs`. Transport code (clients, media fetching, header helpers, transport errors) lives in `litellm-http`. See `../inference/AGENTS.md` for how the crates layer.
# rules
## Python/Rust transformation pairs
## Scope
Use the base OCR and Mistral OCR pairs as the reference when aligning transformations. Use `src/<provider>/<format>/transformation.rs` for provider transformations. Python paths identify counterparts but do not dictate Rust module names
- This crate mirrors `litellm/llms/`: base config traits in `src/base_llm/<format>/`, provider transformations in `src/<provider>/<format>/transformation.rs`
- Transport (clients, media fetching, header helpers, transport errors) lives in `litellm-http`. See `../inference/AGENTS.md` for crate layering
Keep corresponding operation names and parameter names when their responsibilities match. Rust types retain the Python semantic name with Rust acronym casing (`BaseOCRConfig` / `BaseOcrConfig`, `MistralOCRConfig` / `MistralOcrConfig`). Private Python helpers can drop their leading underscore. Give Rust adapter helpers distinct responsibility names rather than duplicating trait method names
## Layering
Order OCR config methods as supported parameters, credential metadata and connection resolution, health-check input, parameter mapping, environment validation, URL construction, request transformation, async request transformation, response transformation, async response transformation, and error conversion. Put constants and data types before the config, private helpers after it in operation order, and tests last. Rust-only trait hooks follow the corresponding Python methods
- `litellm-llms-types` owns API data contracts
- `src/base_llm/<format>/` owns the adapter contract and provider-independent machinery. It never imports a provider or embeds provider policy in trait defaults, normalization or context defaults
- `src/<provider>/<format>/` owns that provider's implementation and policy
- `inference-<format>` owns call orchestration
- These are the intended boundaries, not a claim that all code already satisfies them
Use trait defaults for unchanged inherited behavior and explicit delegation for shared provider behavior. Keep typed inputs, ownership, `Result`, and async I/O idiomatic. A matching path or symbol identifies the counterpart, not a claim of full behavioral parity
## Provider folders
Use named `#[rstest]` cases for independent input/output scenarios instead of loops or repeated calls in one test. Inject reusable setup with `#[fixture]` arguments and use `#[with(...)]` for fixture overrides. Keep assertions about the same result together
- `src/<provider>/` holds provider-wide policy: credentials, auth, endpoints, model capabilities. `common_utils.rs` means shared across that provider's formats, not across providers
- Shared constants and wire types used by several of a provider's formats go in `litellm-llms-types/src/providers/<provider>/`
- A provider may explicitly reuse another provider's helper when its policy applies to the backend (Bedrock, Vertex and Azure reuse `anthropic/messages` shaping for Claude). That does not make the policy format-wide
- Pure payload rewrites belong with transformations, not transport handlers, even in a file named `handler.rs`
Shared OCR document and response contracts live in `litellm-llms-types::formats::ocr`. `BaseOcrConfig` and decoding into adapter errors remain in `src/base_llm/ocr/transformation.rs`. Rust context/environment types support the runtime. `BaseOcrConfig::prepare_request` corresponds to Python's HTTP-handler preparation rather than a `BaseOCRConfig` method, and `validate_request_body` is a Rust-only hook. `src/base_llm/ocr/error.rs` and `src/base_llm/ocr/document.rs` are Rust-only: the OCR error taxonomy shared with the route, and inline-document helpers shared by several providers
## AGENTS.md convention
For Mistral, `async_transform_ocr_request` uses the base default in both languages. `resolve_headers` and `build_ocr_url` implement the respective environment and URL operations, and `normalize_response` implements the typed part of response transformation. Existing auth key/header handling and top-level response-extra preservation differ between languages. Layout refactors must preserve those behaviors and verify them with the existing tests
- Every file is `# rules` then `# references`, both concise bullets. Split a long `# rules` into `##` topic sections
- A folder gets an AGENTS.md only when it adds something: a deviation, an explicit reuse of another provider, or upstream docs no other file lists. A folder without one follows its nearest parent
- Every `src/<provider>/<format>/` folder gets one, since it talks to its own upstream endpoint. `.greptile/files.json` lists them all
- A rule lives in the highest file where it holds. Never restate a parent's rule
- Each upstream URL appears in exactly one AGENTS.md, the one owning that contract. Others point to that file by path
- Format specs: `litellm-llms-types/src/formats/<format>/`
- Provider wire-type docs: `litellm-llms-types/src/providers/<provider>/`
- Provider-wide docs (auth, errors, regions): `src/<provider>/`
- One host's endpoint docs: `src/<provider>/<format>/`
For non-OCR pairs, order corresponding methods as parameter support/mapping, environment validation, URL construction, request transformation, and response transformation, followed by Rust-only runtime hooks. Auth resolution remains split between configs and route preparation in the `litellm-inference-<fmt>` crates. Chat `supported_openai_param_mappings` describes accepted OpenAI/provider name pairs, unlike Python's `get_supported_openai_params` name list. Audio `map_transcription_params` remains a Rust filtering helper
## Python pairs
Azure Messages maps to `llms/azure_ai/anthropic/messages_transformation.py`. Bedrock Converse maps to `llms/bedrock/chat/converse_transformation.py`. `AnthropicConfig`, `AmazonConverseConfig`, and the non-OCR base traits are partial ports. `OpenAiResponsesApiConfig` implements WebSocket transformations and a direct HTTP Responses path. Its HTTP path does not implement Python model-specific parameter rewriting or Responses-to-Chat emulation. Preserve their acceptance gates, passthrough behavior, and host fallback contracts when aligning layout
- Python paths identify counterparts but do not dictate Rust module names or class hierarchy. Preserve behavior and concepts, not structure
- Keep operation and parameter names when responsibilities match. Rust types keep the Python name with Rust acronym casing (`BaseOCRConfig` -> `BaseOcrConfig`). Private Python helpers drop the leading underscore
- Give Rust-only helpers distinct responsibility names instead of reusing trait method names
- Use trait defaults for unchanged inherited behavior and explicit delegation for shared provider behavior. Config traits represent real provider contracts, so do not recreate inheritance with extra traits
- Every provider format writes out its own `impl Base<Format>Config for <Provider><Format>Config`. Do not route several providers' methods through one cross-provider wrapper
- The base OCR and Mistral OCR pairs are the reference when aligning transformations
## Provider and format boundaries
## Method order
The same ownership rule applies to Messages, Responses, Chat Completions, OCR, and other API formats. `litellm-llms-types` owns shared API data contracts. `llms/src/base_llm/<format>/` owns provider adapter contracts and shared transformation machinery. `llms/src/<provider>/<format>/` owns provider implementations and policy. `inference-<format>` owns call orchestration. Repeating a format name identifies the API each layer handles, not duplicate ownership of its schema. These boundaries also apply between modules in the same crate
- OCR configs: supported params, credential metadata and connection resolution, health-check input, param mapping, env validation, URL, request, async request, response, async response, error conversion
- Other formats: param support and mapping, env validation, URL, request, response, then Rust-only runtime hooks
- Constants and data types before the config, private helpers after it in operation order, tests last
A provider adapter may explicitly reuse another provider's transformation helper when that policy applies to its backend, such as Bedrock's Claude adapter using Anthropic payload shaping. Reuse across hosts of the same model family does not make the policy format-wide. Keep provider policy out of shared trait defaults and generic normalization, and keep shared execution contexts limited to inputs the adapter contract actually needs. Pure payload rewrites belong with transformations, not transport handlers
## Known partial ports
- These are intended boundaries, not a claim that all existing code already satisfies them
- Preserve behavior and conceptual boundaries. Python names and layout are reference points, not requirements to reproduce its class hierarchy or helper structure
- Provider directories own provider behavior. API formats and their public data contracts are independent of the provider that originated them
- Shared `base_llm` contracts must not import provider implementations or provider-specific transformation policy
- Config traits represent actual provider contracts. Use composition and existing helpers instead of recreating inheritance with unnecessary traits or delegation layers
- Providers choose authentication and header policy. Shared auth and HTTP infrastructure apply those decisions
- Generic configuration lookup belongs in the existing settings utilities, not in a provider directory
- Closures are idiomatic Rust, but a `Vec<ContentBlock> -> Vec<ContentBlock>` helper is not automatically a useful abstraction
- Choose traversal for the operation: per-block mapping, filtering, or whole-message processing when blocks depend on one another
- Add an abstraction only when it clarifies a repeated responsibility
- Verify observable auth precedence, headers, serialization, passthrough, and transformations, not code structure
- `AnthropicConfig`, `AmazonConverseConfig` and the non-OCR base traits are partial. Preserve their acceptance gates, passthrough and host fallback contracts
- `OpenAiResponsesApiConfig` implements WebSocket transformations and direct HTTP Responses, without Python's model-specific param rewriting or Responses-to-Chat emulation
- Chat `supported_openai_param_mappings` lists OpenAI/provider name pairs, unlike Python's name-only list. Audio `map_transcription_params` is a Rust filtering helper
- Auth resolution is split between configs and route preparation in `inference-<format>`
## OCR specifics
- `BaseOcrConfig::prepare_request` matches Python's HTTP-handler preparation. `validate_request_body` is Rust-only
- `src/base_llm/ocr/error.rs` and `document.rs` are Rust-only: the OCR error taxonomy shared with the route and inline-document helpers
- Mistral auth key handling and top-level response-extra preservation differ from Python. Layout refactors must keep those behaviors and their tests
## Code
- Providers choose auth and header policy, shared auth and HTTP infrastructure apply it. Generic config lookup uses the existing settings utilities
- Pick traversal by operation (per-block map, filter, or whole-message when blocks depend on each other). Add an abstraction only for a repeated responsibility
## Tests
- Named `#[rstest]` cases instead of loops, `#[fixture]` for setup, `#[with(...)]` for overrides
- Verify observable auth precedence, headers, serialization, passthrough and transformations, never code structure

19
litellm-rust/crates/llms/src/anthropic/AGENTS.md
View file

@ -1,8 +1,11 @@
- This directory owns Anthropic provider behavior: credentials, OAuth policy, endpoints, beta requirements, model capabilities, and transformations
- `common_utils.rs` means shared across Anthropic operations, not shared across providers
- Put behavior specific to the Messages API in `messages/`
- Keep generic HTTP mechanics in `litellm-http`, configuration lookup in the existing settings utilities, and credential application in the shared auth layer
- Choose authentication policy and required headers here, then let shared infrastructure apply those decisions
- Consume shared API contracts from `litellm-llms-types`. Do not define public Messages protocol types under this provider
- Preserve Python's concepts and observable behavior where useful, without mechanically reproducing its class hierarchy, helpers, or file structure
- `ReplayedWebSearchResult` and `ReplayedWebSearchContent` are private partial models for replay flattening, not complete public protocol contracts. Keep them private while they serve that transformation
# rules
- Owns Anthropic provider policy: credentials, OAuth handling, endpoint resolution, beta requirements, model capabilities and transformations
- Paths, header names and default headers come from `litellm_llms_types::providers::anthropic`
- Choose auth and required headers here. Shared infrastructure applies them
- `ReplayedWebSearchResult` and `ReplayedWebSearchContent` are private partial models for replay flattening. Keep them private while they only serve that transformation
# references
- https://platform.claude.com/docs/en/api/overview.md
- https://platform.claude.com/docs/en/api/errors.md

6
litellm-rust/crates/llms/src/anthropic/batches/AGENTS.md
View file

@ -1 +1,7 @@
# rules
- Maps Anthropic Message Batches onto the shared batch format. `AnthropicMessageBatch` and the result records are adapter-only projections
# references
- https://platform.claude.com/docs/en/api/http/beta/messages/batches/create

13
litellm-rust/crates/llms/src/anthropic/batches/transformation.rs
View file

@ -1,5 +1,6 @@
use litellm_llms_types::formats::batches::{BatchRequestCounts, BatchResponse, BatchStatus};
use litellm_llms_types::formats::messages::MessagesResponse;
use litellm_llms_types::providers::anthropic::{BATCHES_PATH, MESSAGES_PATH};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use time::OffsetDateTime;
@ -7,8 +8,6 @@ use url::Url;
use crate::{Error, anthropic::common_utils::resolve_anthropic_api_base};
const BATCHES_PATH_SUFFIX: &str = "/v1/messages/batches";
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct AnthropicBatchRequestCounts {
#[serde(default)]
@ -106,12 +105,12 @@ fn batches_base_url(
) -> Result<Url, Error> {
let api_base = resolve_anthropic_api_base(api_base, env_lookup);
let api_base = api_base.trim_end_matches('/');
let complete_url = if api_base.ends_with(BATCHES_PATH_SUFFIX) {
let complete_url = if api_base.ends_with(BATCHES_PATH) {
api_base.to_string()
} else if let Some(base) = api_base.strip_suffix("/v1/messages") {
format!("{base}{BATCHES_PATH_SUFFIX}")
} else if let Some(base) = api_base.strip_suffix(MESSAGES_PATH) {
format!("{base}{BATCHES_PATH}")
} else {
format!("{api_base}{BATCHES_PATH_SUFFIX}")
format!("{api_base}{BATCHES_PATH}")
};
Url::parse(&complete_url).map_err(|error| {
Error::InvalidRequest(crate::ErrorDetail::invalid("Anthropic API base", error))
@ -187,7 +186,7 @@ impl AnthropicBatchesConfig for AnthropicBatchesTransformation {
BatchResponse {
id: response.id.clone(),
object: "batch".into(),
endpoint: "/v1/messages".into(),
endpoint: MESSAGES_PATH.into(),
input_file_id: "None".into(),
completion_window: "24h".into(),
status,

122
litellm-rust/crates/llms/src/anthropic/beta_headers.rs
View file

@ -1,10 +1,8 @@
use litellm_http::request::{with_header, without_headers};
use litellm_llms_types::providers::anthropic::{BetaProvider, BetaSet};
use litellm_llms_types::providers::anthropic::{BETA_HEADER, BetaProvider, BetaSet};
use crate::{anthropic::common_utils::existing_betas, base_llm::auth::Headers};
const BETA_HEADER: &str = "anthropic-beta";
/// How the `anthropic-beta` header is treated before a request leaves for a host.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum BetaPolicy {
@ -34,3 +32,121 @@ impl BetaPolicy {
with_header(headers, BETA_HEADER, accepted.to_string())
}
}
#[cfg(test)]
mod tests {
use crate::anthropic::beta_headers::BetaPolicy;
use litellm_llms_types::providers::anthropic::BetaProvider;
use rstest::rstest;
fn headers(pairs: &[(&str, &str)]) -> Vec<(String, String)> {
pairs
.iter()
.map(|(name, value)| (name.to_string(), value.to_string()))
.collect()
}
#[rstest]
fn forward_keeps_headers_as_is() {
let input = headers(&[
(
"Anthropic-Beta",
"example-beta-2099-01-01,web-search-2025-03-05",
),
("x-api-key", "k"),
]);
assert_eq!(BetaPolicy::Forward.apply(input.clone()), input);
}
#[rstest]
fn drop_removes_every_casing() {
assert_eq!(
BetaPolicy::Drop.apply(headers(&[
("anthropic-beta", "a"),
("x-api-key", "k"),
("ANTHROPIC-BETA", "b"),
])),
headers(&[("x-api-key", "k")])
);
}
#[rstest]
#[case::bedrock(BetaProvider::Bedrock, Some("tool-search-tool-2025-10-19"))]
#[case::bedrock_mantle(BetaProvider::BedrockMantle, Some("tool-search-tool-2025-10-19"))]
#[case::vertex_ai(BetaProvider::VertexAi, Some("tool-search-tool-2025-10-19"))]
#[case::anthropic(BetaProvider::Anthropic, Some("advanced-tool-use-2025-11-20"))]
#[case::azure_ai(BetaProvider::AzureAi, Some("advanced-tool-use-2025-11-20"))]
#[case::databricks(BetaProvider::Databricks, Some("advanced-tool-use-2025-11-20"))]
#[case::bedrock_converse(BetaProvider::BedrockConverse, None)]
fn filter_renames_per_host(
#[case] provider: BetaProvider,
#[case] expected_beta: Option<&str>,
) {
let expected = match expected_beta {
Some(beta) => headers(&[("x-api-key", "k"), ("anthropic-beta", beta)]),
None => headers(&[("x-api-key", "k")]),
};
assert_eq!(
BetaPolicy::Filter(provider).apply(headers(&[
("Anthropic-Beta", "advanced-tool-use-2025-11-20"),
("x-api-key", "k"),
])),
expected
);
}
#[rstest]
#[case::azure_ai(BetaProvider::AzureAi, "fast-mode-2026-02-01,example-beta-2099-01-01")]
#[case::anthropic(BetaProvider::Anthropic, "bash_20241022")]
#[case::vertex_ai(BetaProvider::VertexAi, " , ")]
fn filter_removes_header_when_nothing_survives(
#[case] provider: BetaProvider,
#[case] beta_value: &str,
) {
assert_eq!(
BetaPolicy::Filter(provider).apply(headers(&[
("x-api-key", "k"),
("anthropic-beta", beta_value)
])),
headers(&[("x-api-key", "k")])
);
}
#[rstest]
#[case::anthropic(
BetaProvider::Anthropic,
"web-search-2025-03-05",
"oauth-2025-04-20,web-search-2025-03-05,example-beta-2099-01-01",
"oauth-2025-04-20,web-search-2025-03-05"
)]
#[case::vertex_ai_renames_and_deduplicates(
BetaProvider::VertexAi,
"advanced-tool-use-2025-11-20",
"tool-search-tool-2025-10-19",
"tool-search-tool-2025-10-19"
)]
fn filter_deduplicates_and_sorts(
#[case] provider: BetaProvider,
#[case] first_beta: &str,
#[case] second_beta: &str,
#[case] expected_beta: &str,
) {
assert_eq!(
BetaPolicy::Filter(provider).apply(headers(&[
("ANTHROPIC-BETA", first_beta),
("x-api-key", "k"),
("anthropic-beta", second_beta),
])),
headers(&[("x-api-key", "k"), ("anthropic-beta", expected_beta)])
);
}
#[rstest]
#[case::forward(BetaPolicy::Forward)]
#[case::drop(BetaPolicy::Drop)]
#[case::filter(BetaPolicy::Filter(BetaProvider::Bedrock))]
fn headers_without_a_beta_are_untouched(#[case] policy: BetaPolicy) {
let input = headers(&[("x-api-key", "k"), ("anthropic-version", "2023")]);
assert_eq!(policy.apply(input.clone()), input);
}
}

4
litellm-rust/crates/llms/src/anthropic/chat/AGENTS.md Normal file
View file

@ -0,0 +1,4 @@
# rules
- Chat Completions translated onto Anthropic Messages. Both format specs live under `llms-types/src/formats/`
- `top_k` is not forwarded, because Python gates it per model inside `transform_request`

471
litellm-rust/crates/llms/src/anthropic/chat/transformation.rs
View file

@ -3,8 +3,11 @@ use litellm_core_utils::{
core_helpers::{finish_reason_for, unix_now, usage_from_parts},
prompt_templates::factory::{Conversation, build_conversation},
};
use litellm_llms_types::formats::chat_completions::{
ChatCompletionsChoice, ChatCompletionsChoiceMessage, ChatCompletionsResponse, ChatMessage,
use litellm_llms_types::{
formats::chat_completions::{
ChatCompletionsChoice, ChatCompletionsChoiceMessage, ChatCompletionsResponse, ChatMessage,
},
providers::anthropic::DEFAULT_HEADERS,
};
use serde::Deserialize;
use serde_json::{Map, Value, json};
@ -199,10 +202,7 @@ impl BaseConfig for AnthropicConfig {
}
fn default_headers(&self) -> &'static [(&'static str, &'static str)] {
&[
("anthropic-version", "2023-06-01"),
("content-type", "application/json"),
]
DEFAULT_HEADERS
}
/// An OAuth bearer is the whole credential: Python's `validate_environment`
@ -263,3 +263,462 @@ fn anthropic_body(
);
Value::Object(body)
}
#[cfg(test)]
mod tests {
use crate::{
Error,
anthropic::chat::transformation::ANTHROPIC_CHAT_COMPLETIONS_CONFIG,
base_llm::{
auth::AuthScheme,
chat::transformation::{BaseConfig, ProviderChatResponseData, Unsupported},
},
};
use litellm_llms_types::formats::chat_completions::{ChatCompletionsResponse, ChatMessage};
use rstest::rstest;
use serde_json::{Map, Value, json};
fn messages(value: Value) -> Vec<ChatMessage> {
serde_json::from_value(value).expect("valid messages")
}
fn params(value: Value) -> Map<String, Value> {
match value {
Value::Object(map) => map,
other => panic!("params must be an object, got {other}"),
}
}
fn transform(model: &str, msgs: Value, opts: Value) -> Value {
ANTHROPIC_CHAT_COMPLETIONS_CONFIG
.transform_request(model, messages(msgs), params(opts))
.expect("request transforms")
.body
}
fn transform_response(body: Value) -> Result<ChatCompletionsResponse, Error> {
ANTHROPIC_CHAT_COMPLETIONS_CONFIG
.transform_response("claude-sonnet-4-5", ProviderChatResponseData { body })
}
fn reason(msgs: Value, opts: Value) -> Option<Unsupported> {
ANTHROPIC_CHAT_COMPLETIONS_CONFIG.unsupported_reason(&messages(msgs), &params(opts))
}
#[test]
fn builds_the_messages_body_python_builds() {
let body = transform(
"claude-sonnet-4-5",
json!([
{"role": "system", "content": "be terse"},
{"role": "user", "content": "hi"}
]),
json!({"max_tokens": 128, "temperature": 0.2}),
);
assert_eq!(
body,
json!({
"model": "claude-sonnet-4-5",
"messages": [
{"role": "user", "content": [{"type": "text", "text": "hi"}]}
],
"system": [{"type": "text", "text": "be terse"}],
"max_tokens": 128,
"temperature": 0.2
})
);
}
#[test]
fn omits_system_when_no_system_message_is_present() {
let body = transform(
"claude-sonnet-4-5",
json!([{"role": "user", "content": "hi"}]),
json!({"max_tokens": 16}),
);
assert!(body.get("system").is_none());
}
#[test]
fn merges_consecutive_turns_and_wraps_every_text_in_a_block() {
let body = transform(
"claude-sonnet-4-5",
json!([
{"role": "user", "content": "one"},
{"role": "user", "content": [{"type": "text", "text": "two"}]},
{"role": "assistant", "content": "ack"}
]),
json!({"max_tokens": 16}),
);
assert_eq!(
body["messages"],
json!([
{"role": "user", "content": [
{"type": "text", "text": "one"},
{"type": "text", "text": "two"}
]},
{"role": "assistant", "content": [{"type": "text", "text": "ack"}]}
])
);
}
#[test]
fn right_strips_a_trailing_assistant_prefill_like_python() {
let body = transform(
"claude-sonnet-4-5",
json!([
{"role": "user", "content": "hi"},
{"role": "assistant", "content": "Argentina "}
]),
json!({"max_tokens": 16}),
);
assert_eq!(
body["messages"][1]["content"][0]["text"],
json!("Argentina")
);
}
#[test]
fn passes_every_supported_param_through_untouched() {
let body = transform(
"claude-sonnet-4-5",
json!([{"role": "user", "content": "hi"}]),
json!({
"max_tokens": 64,
"temperature": 0.1,
"top_p": 0.9,
"stop_sequences": ["STOP"]
}),
);
assert_eq!(body["max_tokens"], json!(64));
assert_eq!(body["temperature"], json!(0.1));
assert_eq!(body["top_p"], json!(0.9));
assert_eq!(body["stop_sequences"], json!(["STOP"]));
}
#[test]
fn declines_top_k_because_python_gates_it_by_model_below_this_point() {
// `temperature` and `top_p` arrive already resolved, because
// `map_openai_params` applies `_apply_sampling_param` to them before the
// gate runs. `top_k` bypasses that and is gated inside `transform_request`,
// the function this route replaces, so forwarding it would send `top_k` to
// a model that removed sampling params and take a 400 after the call, where
// Python drops it and succeeds.
assert_eq!(
reason(
json!([{"role": "user", "content": "hi"}]),
json!({"top_k": 40})
),
Some(Unsupported("unrecognized request parameter"))
);
}
#[test]
fn declines_streaming_before_anything_else() {
assert_eq!(
reason(
json!([{"role": "user", "content": "hi"}]),
json!({"stream": true, "max_tokens": 16})
),
Some(Unsupported("streaming"))
);
}
#[test]
fn accepts_an_explicit_stream_false() {
assert_eq!(
reason(
json!([{"role": "user", "content": "hi"}]),
json!({"stream": false, "max_tokens": 16})
),
None
);
}
#[rstest]
#[case::tools(json!({"tools": []}))]
#[case::tool_choice(json!({"tool_choice": {"type": "auto"}}))]
#[case::thinking(json!({"thinking": {"type": "enabled"}}))]
#[case::system(json!({"system": "injected"}))]
#[case::metadata(json!({"metadata": {"user_id": "u1"}}))]
#[case::output_config(json!({"output_config": {"effort": "high"}}))]
fn declines_any_param_outside_the_allowlist(#[case] param: Value) {
assert_eq!(
reason(json!([{"role": "user", "content": "hi"}]), param.clone()),
Some(Unsupported("unrecognized request parameter")),
"expected {param} to decline"
);
}
#[test]
fn declines_tool_calls_tool_results_and_multimodal_content() {
assert_eq!(
reason(
json!([
{"role": "user", "content": "hi"},
{"role": "assistant", "content": null, "tool_calls": [
{"id": "c1", "type": "function",
"function": {"name": "f", "arguments": "{}"}}
]}
]),
json!({})
),
Some(Unsupported("unrecognized message field"))
);
assert_eq!(
reason(
json!([
{"role": "user", "content": "hi"},
{"role": "tool", "tool_call_id": "c1", "content": "ok"}
]),
json!({})
),
Some(Unsupported("unrecognized message field"))
);
assert_eq!(
reason(
json!([
{"role": "user", "content": [
{"type": "image_url", "image_url": {"url": "https://x/y.png"}}
]}]),
json!({})
),
Some(Unsupported("non-text message content"))
);
assert_eq!(
reason(
json!([
{"role": "user", "content": [
{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}
]}]),
json!({})
),
Some(Unsupported("non-text message content"))
);
}
#[test]
fn declines_a_message_whose_content_list_is_empty() {
// An empty list passes every per-part check, so without this it would reach
// the provider as an empty `content` array and fail after the call rather
// than declining to Python before it.
assert_eq!(
reason(json!([{"role": "user", "content": []}]), json!({})),
Some(Unsupported("message without content"))
);
assert_eq!(
reason(
json!([{"role": "user", "content": [{"type": "text", "text": "hi"}]}]),
json!({})
),
None
);
}
#[test]
fn declines_a_conversation_that_does_not_open_on_a_user_turn() {
assert_eq!(
reason(
json!([
{"role": "system", "content": "be terse"},
{"role": "assistant", "content": "prefill"}
]),
json!({})
),
Some(Unsupported("conversation does not open on a user turn"))
);
}
#[test]
fn accepts_a_plain_text_conversation() {
assert_eq!(
reason(
json!([
{"role": "system", "content": "be terse"},
{"role": "user", "content": "hi"},
{"role": "assistant", "content": "hello"},
{"role": "user", "content": [{"type": "text", "text": "again"}]}
]),
json!({"max_tokens": 16, "temperature": 0.5})
),
None
);
}
#[test]
fn normalizes_a_text_response_into_openai_shape() {
let response = transform_response(json!({
"id": "msg_123",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-4-5-20260101",
"content": [{"type": "text", "text": "hello"}, {"type": "text", "text": " there"}],
"stop_reason": "end_turn",
"stop_sequence": null,
"usage": {"input_tokens": 11, "output_tokens": 4}
}))
.expect("response transforms");
assert_eq!(response.model, "claude-sonnet-4-5-20260101");
assert_eq!(response.choices.len(), 1);
assert_eq!(response.choices[0].index, 0);
assert_eq!(response.choices[0].message.role, "assistant");
assert_eq!(
response.choices[0].message.content.as_deref(),
Some("hello there")
);
assert_eq!(response.choices[0].finish_reason, "stop");
assert_eq!(response.usage.prompt_tokens, 11);
assert_eq!(response.usage.completion_tokens, 4);
assert_eq!(response.usage.total_tokens, 15);
}
#[test]
fn folds_cache_tokens_into_prompt_tokens_like_python() {
let response = transform_response(json!({
"model": "claude-sonnet-4-5",
"content": [{"type": "text", "text": "hi"}],
"stop_reason": "end_turn",
"usage": {
"input_tokens": 10,
"output_tokens": 2,
"cache_read_input_tokens": 5,
"cache_creation_input_tokens": 3
}
}))
.expect("response transforms");
assert_eq!(response.usage.prompt_tokens, 18);
assert_eq!(response.usage.total_tokens, 20);
assert_eq!(response.usage.prompt_tokens_details.cached_tokens, 5);
assert_eq!(
response.usage.prompt_tokens_details.cache_creation_tokens,
3
);
assert_eq!(response.usage.prompt_tokens_details.text_tokens, 10);
}
#[test]
fn maps_max_tokens_stop_reason_to_length() {
let response = transform_response(json!({
"model": "claude-sonnet-4-5",
"content": [{"type": "text", "text": "hi"}],
"stop_reason": "max_tokens",
"usage": {"input_tokens": 1, "output_tokens": 1}
}))
.expect("response transforms");
assert_eq!(response.choices[0].finish_reason, "length");
}
#[test]
fn a_refusal_returns_the_completion_python_returns() {
// `refusal` is a stop_reason, not a content block type, so the content is
// ordinary text and this normalizes rather than declining. Python maps it
// to content_filter in _FINISH_REASON_MAP and returns the completion.
let response = transform_response(json!({
"model": "claude-sonnet-4-5",
"content": [{"type": "text", "text": "I can't help with that."}],
"stop_reason": "refusal",
"usage": {"input_tokens": 9, "output_tokens": 6}
}))
.expect("a refusal still transforms");
assert_eq!(response.choices[0].finish_reason, "content_filter");
assert_eq!(
response.choices[0].message.content.as_deref(),
Some("I can't help with that.")
);
}
#[test]
fn reports_no_content_rather_than_an_empty_string() {
let response = transform_response(json!({
"model": "claude-sonnet-4-5",
"content": [],
"stop_reason": "end_turn",
"usage": {"input_tokens": 1, "output_tokens": 0}
}))
.expect("response transforms");
assert_eq!(response.choices[0].message.content, None);
}
#[test]
fn response_carries_no_id_so_python_keeps_its_chatcmpl_id() {
let response = transform_response(json!({
"id": "msg_should_not_leak",
"model": "claude-sonnet-4-5",
"content": [{"type": "text", "text": "hi"}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 1, "output_tokens": 1}
}))
.expect("response transforms");
let value = serde_json::to_value(response).expect("serializable");
assert!(
value.get("id").is_none(),
"the rust response must not carry an id, got {value}"
);
}
#[test]
fn declines_a_response_carrying_a_non_text_block() {
let err = transform_response(json!({
"model": "claude-sonnet-4-5",
"content": [{"type": "tool_use", "id": "t1", "name": "f", "input": {}}],
"stop_reason": "tool_use",
"usage": {"input_tokens": 1, "output_tokens": 1}
}))
.expect_err("non-text block");
assert_eq!(err, Error::Unsupported("non-text response content block"));
}
#[rstest::rstest]
#[case::not_an_object(json!("nope"))]
#[case::missing_content(json!({"model": "test-model", "usage": {"input_tokens": 1, "output_tokens": 1}}))]
#[case::missing_usage(json!({"model": "test-model", "content": []}))]
#[case::missing_model(json!({"content": [], "usage": {"input_tokens": 1, "output_tokens": 1}}))]
fn errors_on_a_response_missing_required_fields(#[case] body: Value) {
let error = transform_response(body).expect_err("invalid response");
assert!(matches!(error, Error::InvalidResponse(_)));
let source = std::iter::successors(Some(&error as &dyn std::error::Error), |error| {
error.source()
})
.find_map(|error| error.downcast_ref::<serde_json::Error>())
.expect("the JSON decoding source is preserved");
assert_eq!(
error.to_string(),
format!("invalid response: invalid messages response: {source}")
);
}
#[test]
fn resolves_the_messages_url_and_x_api_key_auth() {
let config = &ANTHROPIC_CHAT_COMPLETIONS_CONFIG;
assert_eq!(
config
.get_complete_url(None, "claude-sonnet-4-5", &Map::new(), &|_| None)
.expect("url builds"),
"https://api.anthropic.com/v1/messages"
);
let validated = config
.validate_environment(
Vec::new(),
Some("sk-x"),
"claude-sonnet-4-5",
&Map::new(),
&|_| None,
)
.expect("auth resolves");
assert!(matches!(
validated.auth,
AuthScheme::Credential {
placement: litellm_auth::CredentialPlacement::Header("x-api-key"),
ref secret
} if secret.expose() == "sk-x"
));
assert_eq!(
config.default_headers(),
&[
("anthropic-version", "2023-06-01"),
("content-type", "application/json"),
]
);
}
}

18
litellm-rust/crates/llms/src/anthropic/common_utils.rs
View file

@ -9,13 +9,15 @@ use litellm_llms_types::{
ContentBlock, ContentBlockType, EffortLevel, Message, MessageContent, MessagesTool,
SystemPrompt,
},
providers::anthropic::{AnthropicBeta, BetaSet},
providers::anthropic::{
API_BASE, API_KEY_HEADER, AnthropicBeta, BETA_HEADER, BetaSet,
DIRECT_BROWSER_ACCESS_HEADER, MESSAGES_PATH,
},
recognized::Recognized,
};
use serde::Deserialize;
use serde_json::Value;
use crate::base_llm::messages::transformation::MESSAGES_PATH_SUFFIX;
use crate::{
anthropic::ANTHROPIC_OAUTH_TOKEN_PREFIX,
base_llm::auth::{AuthScheme, Headers},
@ -25,14 +27,10 @@ pub const ANTHROPIC_API_KEY_ENV: &str = "ANTHROPIC_API_KEY";
pub const ANTHROPIC_AUTH_TOKEN_ENV: &str = "ANTHROPIC_AUTH_TOKEN";
pub const ENCRYPTED_REASONING_SIGNATURE_PREFIX: &str = "litellm_encrypted_reasoning:";
const THOUGHT_SIGNATURE_SEPARATOR: &str = "__thought__";
const BETA_HEADER: &str = "anthropic-beta";
pub const ANTHROPIC_API_BASE_ENV: &str = "ANTHROPIC_API_BASE";
pub const ANTHROPIC_BASE_URL_ENV: &str = "ANTHROPIC_BASE_URL";
pub const DEFAULT_ANTHROPIC_API_BASE: &str = "https://api.anthropic.com";
pub const API_KEY_PLACEMENT: CredentialPlacement = CredentialPlacement::Header("x-api-key");
const API_KEY_HEADER: &str = API_KEY_PLACEMENT.header_name();
pub const API_KEY_PLACEMENT: CredentialPlacement = CredentialPlacement::Header(API_KEY_HEADER);
const AUTHORIZATION: &str = CredentialPlacement::Bearer.header_name();
const DIRECT_BROWSER_ACCESS_HEADER: &str = "anthropic-dangerous-direct-browser-access";
pub fn supports_effort_tier(capabilities: &MessagesModelCapabilities, level: EffortLevel) -> bool {
match level {
@ -145,7 +143,7 @@ pub fn resolve_anthropic_api_base(
env_lookup,
&[ANTHROPIC_API_BASE_ENV, ANTHROPIC_BASE_URL_ENV],
)
.unwrap_or_else(|| DEFAULT_ANTHROPIC_API_BASE.to_string())
.unwrap_or_else(|| API_BASE.to_string())
}
pub fn complete_anthropic_url(
@ -155,10 +153,10 @@ pub fn complete_anthropic_url(
let api_base = resolve_anthropic_api_base(api_base, env_lookup);
let api_base = api_base.trim_end_matches('/');
if api_base.ends_with(MESSAGES_PATH_SUFFIX) {
if api_base.ends_with(MESSAGES_PATH) {
return api_base.to_string();
}
format!("{api_base}{MESSAGES_PATH_SUFFIX}")
format!("{api_base}{MESSAGES_PATH}")
}
pub fn existing_betas(headers: &[(String, String)]) -> BetaSet {

6
litellm-rust/crates/llms/src/anthropic/count_tokens/AGENTS.md
View file

@ -1 +1,7 @@
# rules
- Builds the count-tokens request against the public Anthropic host. The request and response structs are adapter-only projections
# references
- https://platform.claude.com/docs/en/api/http/messages/count_tokens

22
litellm-rust/crates/llms/src/anthropic/count_tokens/transformation.rs
View file

@ -1,10 +1,14 @@
use litellm_llms_types::formats::messages::{Message, SystemPrompt};
use litellm_llms_types::{
formats::messages::{Message, SystemPrompt},
providers::anthropic::{
API_BASE, API_KEY_HEADER, API_VERSION, BETA_HEADER, COUNT_TOKENS_PATH, VERSION_HEADER,
},
};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use crate::{Error, anthropic::ANTHROPIC_OAUTH_TOKEN_PREFIX};
const COUNT_TOKENS_ENDPOINT: &str = "https://api.anthropic.com/v1/messages/count_tokens";
const TOKEN_COUNTING_BETA: &str = "token-counting-2024-11-01";
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
@ -23,7 +27,7 @@ pub struct AnthropicCountTokensResponse {
}
pub trait AnthropicCountTokensConfig {
fn endpoint(&self) -> &'static str;
fn endpoint(&self) -> String;
fn validate_request(&self, model: &str, messages: &[Message]) -> Result<(), Error>;
@ -44,8 +48,8 @@ pub const ANTHROPIC_COUNT_TOKENS_TRANSFORMATION: AnthropicCountTokensTransformat
AnthropicCountTokensTransformation;
impl AnthropicCountTokensConfig for AnthropicCountTokensTransformation {
fn endpoint(&self) -> &'static str {
COUNT_TOKENS_ENDPOINT
fn endpoint(&self) -> String {
format!("{API_BASE}{COUNT_TOKENS_PATH}")
}
fn transform_request(
@ -79,13 +83,13 @@ impl AnthropicCountTokensConfig for AnthropicCountTokensTransformation {
let auth = if api_key.starts_with(ANTHROPIC_OAUTH_TOKEN_PREFIX) {
("authorization", format!("Bearer {api_key}"))
} else {
("x-api-key", api_key.to_string())
(API_KEY_HEADER, api_key.to_string())
};
vec![
("content-type", "application/json".to_string()),
auth,
("anthropic-version", "2023-06-01".to_string()),
("anthropic-beta", TOKEN_COUNTING_BETA.to_string()),
(VERSION_HEADER, API_VERSION.to_string()),
(BETA_HEADER, TOKEN_COUNTING_BETA.to_string()),
]
}
}
@ -127,7 +131,7 @@ mod tests {
);
assert_eq!(
ANTHROPIC_COUNT_TOKENS_TRANSFORMATION.endpoint(),
COUNT_TOKENS_ENDPOINT
"https://api.anthropic.com/v1/messages/count_tokens"
);
}

12
litellm-rust/crates/llms/src/anthropic/messages/AGENTS.md
View file

@ -1,9 +1,5 @@
This directory owns Anthropic's implementation of the Messages adapter contract in `base_llm/messages`. Shared Messages API data contracts belong in `litellm-llms-types::formats::messages`, and call orchestration belongs in `inference-messages`. Sharing the `llms` crate with `base_llm/messages` does not erase this boundary
# rules
Payload shaping, metadata filtering, tool-ID rewriting, web-search replay handling, thinking translation, and beta selection are provider policy. Keep them here or in Anthropic helpers shared by its operations. Pure payload shaping belongs with transformations, even if an existing file is named `handler.rs`
Bedrock and Azure adapters may explicitly reuse these helpers where Anthropic policy applies to their Claude backend. That reuse does not make the policy part of the shared Messages contract or a default for every provider. Shared `base_llm` code must never depend on this implementation
`web_search_result`, `web_search_tool_result_error`, and encrypted-content fields are protocol data owned by `litellm-llms-types`. Keep those schemas separate from decisions about flattening, encrypted results, beta requirements, and model capabilities
Protocol reference: [Messages API](https://platform.claude.com/docs/en/api/http/messages/create)
- Format spec lives in `llms-types/src/formats/messages/AGENTS.md`
- Payload shaping, metadata filtering, tool-ID rewriting, web-search replay, thinking translation and beta selection live here or in Anthropic-wide helpers
- Bedrock, Vertex, Azure and DeepSeek reuse these helpers for their Claude-compatible hosts. Keep the helpers free of those hosts' differences

7
litellm-rust/crates/llms/src/anthropic/messages/transformation.rs
View file

@ -3,7 +3,7 @@ use litellm_llms_types::{
formats::messages::{
ContextEdit, ContextManagement, Message, MessagesOptionalParams, MessagesRequest, Speed,
},
providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet},
providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet, DEFAULT_HEADERS},
recognized::Recognized,
};
use litellm_router_types::LitellmParams;
@ -26,11 +26,6 @@ use crate::{
},
};
pub(crate) const DEFAULT_HEADERS: &[(&str, &str)] = &[
("anthropic-version", "2023-06-01"),
("content-type", "application/json"),
];
pub struct AnthropicMessagesConfig;
pub const ANTHROPIC_MESSAGES_CONFIG: AnthropicMessagesConfig = AnthropicMessagesConfig;

6
litellm-rust/crates/llms/src/aws_textract/ocr/AGENTS.md
View file

@ -1,3 +1,9 @@
# rules
- Translates Textract DetectDocumentText and AnalyzeDocument output into the shared OCR format. Only synchronous single-page calls are supported
# references
- https://docs.aws.amazon.com/textract/latest/APIReference/Welcome.md
- https://docs.aws.amazon.com/textract/latest/APIReference/API_Operations.md
- https://docs.aws.amazon.com/textract/latest/APIReference/API_DetectDocumentText.md

8
litellm-rust/crates/llms/src/azure_ai/messages/AGENTS.md
View file

@ -1,3 +1,7 @@
This directory owns Azure's Messages adapter: its endpoints, authentication policy, headers, and transformations. Implement the shared adapter contract from `base_llm/messages`, consume API data contracts from `litellm-llms-types::formats::messages`, and leave call orchestration to `inference-messages`
# rules
The Claude adapter may explicitly reuse payload policy from `anthropic/messages` when it applies to Azure's Claude backend. Keep Azure-specific differences here. Sharing that helper does not make Anthropic policy a format-wide default or justify a dependency from `base_llm/messages` on provider implementations
- Claude on Azure AI Foundry: reuses `anthropic/messages` shaping with the Azure `/anthropic` path, Azure credential env vars, and `x-api-key` or bearer auth
# references
- https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/how-to/use-foundry-models-claude

23
litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs
View file

@ -1,14 +1,18 @@
use litellm_auth::{CredentialPlacement, SecretValue};
use litellm_auth::SecretValue;
use litellm_http::request::{has_bearer_auth, has_header};
use litellm_llms_types::formats::messages::MessagesRequest;
use litellm_llms_types::{
formats::messages::MessagesRequest,
providers::anthropic::{DEFAULT_HEADERS, MESSAGES_PATH},
};
use litellm_router_types::LitellmParams;
use crate::{
Error,
anthropic::messages::{
handler::shape_anthropic_messages_request,
transformation::{
DEFAULT_HEADERS, transform_messages_request, update_headers_with_anthropic_beta,
anthropic::{
common_utils::API_KEY_PLACEMENT,
messages::{
handler::shape_anthropic_messages_request,
transformation::{transform_messages_request, update_headers_with_anthropic_beta},
},
},
azure_ai::common_utils::{
@ -19,12 +23,11 @@ use crate::{
messages::{
context::MessagesTransformContext,
normalization::{normalize_system_role_messages, strip_cache_control_scope},
transformation::{BaseMessagesConfig, MESSAGES_PATH_SUFFIX},
transformation::BaseMessagesConfig,
},
},
};
const API_KEY_PLACEMENT: CredentialPlacement = CredentialPlacement::Header("x-api-key");
const ANTHROPIC_PATH_SEGMENT: &str = "/anthropic";
pub struct AzureAnthropicMessagesConfig;
@ -113,7 +116,7 @@ pub fn complete_azure_anthropic_url(
let api_base = api_base.trim_end_matches('/');
if api_base.ends_with(MESSAGES_PATH_SUFFIX) {
if api_base.ends_with(MESSAGES_PATH) {
return Ok(api_base.to_string());
}
@ -121,7 +124,7 @@ pub fn complete_azure_anthropic_url(
Some((prefix, _)) => format!("{prefix}{ANTHROPIC_PATH_SEGMENT}"),
None => format!("{api_base}{ANTHROPIC_PATH_SEGMENT}"),
};
Ok(format!("{with_anthropic}{MESSAGES_PATH_SUFFIX}"))
Ok(format!("{with_anthropic}{MESSAGES_PATH}"))
}
#[cfg(test)]

9
litellm-rust/crates/llms/src/azure_ai/ocr/AGENTS.md Normal file
View file

@ -0,0 +1,9 @@
# rules
- `transformation.rs` sends the Mistral OCR request to Azure AI's `/providers/mistral/azure/ocr` path
- `cohere_parse_transformation.rs` reuses `cohere/ocr` against Azure AI's `/providers/cohere/v2/parse` path
- `document_intelligence/` translates Azure Document Intelligence's async analyze operation into the shared OCR format
# references
- https://learn.microsoft.com/en-us/rest/api/aiservices/document-models/analyze-document

10
litellm-rust/crates/llms/src/base_llm/messages/AGENTS.md
View file

@ -1,5 +1,7 @@
This directory owns the shared Messages provider adapter contract, its execution inputs such as `MessagesTransformContext`, and provider-independent transformation machinery. Public request, response, content-block, and event schemas belong in `litellm-llms-types::formats::messages`. Call orchestration belongs in `inference-messages`, and provider implementations belong in `llms/src/<provider>/messages`
# rules
Do not import provider implementations or embed their policy in shared trait defaults, normalization, or context defaults. A context carries inputs the shared adapter contract needs, not every provider's settings. Thinking-budget choices and model-specific restrictions do not become format rules merely because several providers host Claude
Shared normalization must implement LiteLLM's provider-independent Messages input contract. Provider-specific metadata filtering, tool-ID rewriting, web-search replay policy, beta selection, and thinking translation belong in the provider implementation. Let each adapter explicitly opt into applicable shared provider helpers
- Owns the Messages adapter contract, its execution inputs such as `MessagesTransformContext`, and provider-independent machinery
- A context carries only inputs the contract needs, not every provider's settings
- Shared normalization implements LiteLLM's provider-independent Messages input contract
- Metadata filtering, tool-ID rewriting, web-search replay, beta selection, thinking translation and thinking budgets are provider policy. Adapters opt into shared provider helpers explicitly
- Restrictions common to Claude hosts are still provider policy, not format rules

58
litellm-rust/crates/llms/src/base_llm/messages/normalization.rs
View file

@ -103,3 +103,61 @@ pub fn strip_cache_control_scope(request: MessagesRequest) -> MessagesRequest {
..request
}
}
#[cfg(test)]
mod tests {
use crate::base_llm::messages::normalization::normalize_system_role_messages;
use litellm_llms_types::formats::messages::MessagesRequest;
use rstest::rstest;
use serde_json::{Value, json};
#[rstest]
#[case::text_system(json!("existing"), json!([{"type": "text", "text": "existing"}]))]
#[case::block_system(json!([{ "type": "future", "payload": 7 }]), json!([{ "type": "future", "payload": 7 }]))]
#[case::no_system(Value::Null, json!([]))]
fn hoisting_preserves_block_fields_order_and_unrelated_request_fields(
#[case] system: Value,
#[case] initial_blocks: Value,
) {
let cache_control = json!({"type": "ephemeral", "scope": "global", "future": true});
let leading_block =
json!({"type": "text", "text": "second", "cache_control": cache_control});
let user = json!({"role": "user", "content": "hello", "future_message": 42});
let later_turn = json!({"role": "system", "content": "mid-conversation reminder"});
let request: MessagesRequest = serde_json::from_value(json!({
"model": "test-model",
"max_tokens": 64,
"system": system,
"messages": [
{"role": "system", "content": "first"},
{"role": "system", "content": [leading_block]},
user,
later_turn
],
"future_request": {"nested": true}
}))
.unwrap();
let normalized = normalize_system_role_messages(request, true);
let expected_blocks: Vec<Value> = initial_blocks
.as_array()
.unwrap()
.iter()
.cloned()
.chain([json!({"type": "text", "text": "first"}), leading_block])
.collect();
assert_eq!(
serde_json::to_value(&normalized).unwrap(),
json!({
"model": "test-model",
"max_tokens": 64,
"system": expected_blocks,
"messages": [user, later_turn],
"future_request": {"nested": true}
})
);
assert_eq!(
normalize_system_role_messages(normalized.clone(), true),
normalized
);
}
}

2
litellm-rust/crates/llms/src/base_llm/messages/transformation.rs
View file

@ -7,8 +7,6 @@ use super::context::MessagesTransformContext;
pub use crate::base_llm::auth::{Headers, ValidatedEnvironment};
use crate::{Error, base_llm::messages::streaming::StreamDecoder};
pub const MESSAGES_PATH_SUFFIX: &str = "/v1/messages";
pub trait BaseMessagesConfig: Sync {
fn shape_request(
&self,

80
litellm-rust/crates/llms/src/base_llm/ocr/handler.rs
View file

@ -303,7 +303,13 @@ pub fn body_document(body: &Value) -> Result<OcrDocument, Error> {
#[cfg(test)]
mod tests {
use crate::base_llm::ocr::{error::Error, handler::read_response_bytes};
use rstest::rstest;
use std::time::Duration;
use tokio::{
io::{AsyncReadExt, AsyncWriteExt},
net::TcpListener,
};
use super::*;
@ -328,4 +334,78 @@ mod tests {
));
server.abort();
}
/// Answers one request with raw `response` bytes and then holds the connection open, so a
/// read that waits for the rest of an oversized body hangs instead of passing.
async fn read_bounded(response: String, limit: usize) -> Result<bytes::Bytes, Error> {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let mut request = [0; 4096];
assert!(socket.read(&mut request).await.unwrap() > 0);
socket.write_all(response.as_bytes()).await.unwrap();
std::future::pending::<()>().await;
});
let response = litellm_http::Client::plain_for_test()
.get(format!("http://{address}"))
.send()
.await
.unwrap();
let result =
tokio::time::timeout(Duration::from_secs(2), read_response_bytes(response, limit))
.await;
server.abort();
result.expect("bounded reads must finish without waiting for the rest of an oversized body")
}
#[rstest]
#[case::declared("HTTP/1.1 200 OK\r\nContent-Length: 8\r\n\r\nabcdefgh")]
#[case::chunked(
"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nabcd\r\n4\r\nefgh\r\n0\r\n\r\n"
)]
#[tokio::test]
async fn a_body_of_exactly_the_limit_is_read(#[case] response: &str) {
assert_eq!(read_bounded(response.into(), 8).await.unwrap(), "abcdefgh");
}
#[rstest]
#[case::declared("HTTP/1.1 200 OK\r\nContent-Length: 9\r\n\r\n")]
#[case::chunked(
"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nabcd\r\n5\r\nefghi\r\n"
)]
#[tokio::test]
async fn a_body_over_the_limit_is_rejected(#[case] response: &str) {
assert!(matches!(
read_bounded(response.into(), 8).await,
Err(Error::TooLarge { limit: 8 })
));
}
#[rstest]
#[case::declared("Content-Length: 1000000")]
#[case::chunked("Transfer-Encoding: chunked")]
#[tokio::test]
async fn an_oversized_error_keeps_its_status_and_a_bounded_body_without_draining(
#[case] headers: &str,
) {
let prefix = "x".repeat(4096);
let body = match headers.starts_with("Transfer") {
true => format!("{:x}\r\n{prefix}\r\n", prefix.len()),
false => prefix.clone(),
};
let error = read_bounded(
format!("HTTP/1.1 429 Too Many Requests\r\n{headers}\r\n\r\n{body}"),
prefix.len(),
)
.await
.unwrap_err();
let Error::Transport(litellm_http::transport::Error::Http { status, body }) = error else {
panic!("unexpected error: {error}");
};
assert_eq!(status, 429);
assert_eq!(body, prefix);
}
}

9
litellm-rust/crates/llms/src/bedrock/AGENTS.md Normal file
View file

@ -0,0 +1,9 @@
# rules
- Owns Bedrock-wide policy: region resolution, the runtime endpoint template, and SigV4 or bearer-token auth through `litellm-auth-aws`
- Paths and wire types shared by Bedrock's formats come from `litellm_llms_types::providers::bedrock`
# references
- https://docs.aws.amazon.com/general/latest/gr/bedrock.html
- https://docs.aws.amazon.com/bedrock/latest/userguide/api-keys.html

8
litellm-rust/crates/llms/src/bedrock/audio_transcription/AGENTS.md Normal file
View file

@ -0,0 +1,8 @@
# rules
- Transcription over Converse: the audio goes in as a Converse content block and the transcript is the response text
- Non-text response blocks are an error, never silently dropped
# references
- https://docs.aws.amazon.com/bedrock/latest/userguide/conversation-inference.html

6
litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs
View file

@ -5,7 +5,10 @@ use litellm_auth_aws::{
resolve_bedrock_region,
};
use litellm_core_utils::core_helpers::json_type_name;
use litellm_llms_types::formats::audio_transcription::AudioTranscriptionResponseData;
use litellm_llms_types::{
formats::audio_transcription::AudioTranscriptionResponseData,
providers::bedrock::ConverseResponse,
};
use serde::Deserialize;
use serde_json::{Map, Value, json};
use strum::IntoStaticStr;
@ -19,7 +22,6 @@ use crate::{
},
auth::AuthScheme,
},
bedrock::chat::converse_transformation::ConverseResponse,
};
const SUPPORTED_PARAMS: &[&str] = &["language", "prompt", "temperature", "response_format"];

5
litellm-rust/crates/llms/src/bedrock/chat/AGENTS.md Normal file
View file

@ -0,0 +1,5 @@
# rules
- Chat Completions over Converse. Converse wire docs live in `llms-types/src/providers/bedrock/AGENTS.md`
- `topK` is not forwarded, because Python's placement depends on the model catalog this crate cannot see
- `invoke_handler.rs` decodes InvokeModel event streams for Claude, reusing `anthropic/chat` response handling

712
litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs
View file

@ -9,11 +9,13 @@ use litellm_core_utils::{
core_helpers::{finish_reason_for, unix_now, usage_from_parts},
prompt_templates::factory::{Conversation, TurnRole, build_conversation},
};
use litellm_llms_types::formats::chat_completions::{
ChatCompletionsChoice, ChatCompletionsChoiceMessage, ChatCompletionsResponse,
ChatCompletionsUsage, ChatMessage, ChatMessageContent,
use litellm_llms_types::{
formats::chat_completions::{
ChatCompletionsChoice, ChatCompletionsChoiceMessage, ChatCompletionsResponse,
ChatCompletionsUsage, ChatMessage, ChatMessageContent,
},
providers::bedrock::{CONVERSE_PATH, ConverseResponse},
};
use serde::Deserialize;
use serde_json::{Map, Value, json};
use crate::{
@ -62,61 +64,6 @@ const CONFIG_PARAMS: &[&str] = &[
AWS_BEDROCK_RUNTIME_ENDPOINT,
];
const CONVERSE_PATH_SUFFIX: &str = "/converse";
#[derive(Deserialize)]
pub(crate) struct ConverseResponse {
output: ConverseOutput,
// Converse always reports usage, but the transcription route tolerates its
// absence; the chat transform checks for the field itself.
#[serde(default)]
usage: ConverseUsage,
#[serde(rename = "stopReason")]
stop_reason: Option<String>,
}
#[derive(Deserialize)]
struct ConverseOutput {
message: ConverseMessage,
}
#[derive(Deserialize)]
struct ConverseMessage {
content: Vec<ConverseContentBlock>,
}
enum ConverseContentBlock {
Text { text: String },
Other(serde::de::IgnoredAny),
}
impl<'de> Deserialize<'de> for ConverseContentBlock {
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
let value = Value::deserialize(deserializer)?;
if let Some(text) = value.get("text") {
let text = text.as_str().ok_or_else(|| {
serde::de::Error::custom("invalid type for `text`, expected a string")
})?;
return Ok(Self::Text {
text: text.to_owned(),
});
}
Ok(Self::Other(serde::de::IgnoredAny))
}
}
#[derive(Default, Deserialize)]
#[serde(rename_all = "camelCase")]
struct ConverseUsage {
input_tokens: u64,
output_tokens: u64,
#[serde(default)]
cache_read_input_tokens: u64,
#[serde(default)]
cache_write_input_tokens: u64,
total_tokens: Option<u64>,
}
enum ConverseStopReason {
Value(String),
Unknown(String),
@ -141,28 +88,6 @@ impl ConverseStopReason {
}
}
impl ConverseResponse {
pub(crate) fn content_text(&self) -> String {
self.output
.message
.content
.iter()
.filter_map(|block| match block {
ConverseContentBlock::Text { text } => Some(text.as_str()),
ConverseContentBlock::Other(_) => None,
})
.collect()
}
pub(crate) fn message_content_is_non_text(&self) -> bool {
self.output
.message
.content
.iter()
.any(|block| matches!(block, ConverseContentBlock::Other(_)))
}
}
pub struct AmazonConverseConfig;
pub const BEDROCK_CHAT_COMPLETIONS_CONFIG: AmazonConverseConfig = AmazonConverseConfig;
@ -205,10 +130,10 @@ impl BaseConfig for AmazonConverseConfig {
// A host that already built the full Converse URL (LiteLLM's Python
// path encodes the model id itself) passes it through untouched, the
// way the Anthropic config leaves a complete `/v1/messages` URL alone.
if endpoint.ends_with(CONVERSE_PATH_SUFFIX) {
if endpoint.ends_with(CONVERSE_PATH) {
return Ok(endpoint.to_string());
}
Ok(format!("{endpoint}/model/{model_id}{CONVERSE_PATH_SUFFIX}"))
Ok(format!("{endpoint}/model/{model_id}{CONVERSE_PATH}"))
}
fn transform_request(
@ -426,3 +351,624 @@ fn has_blank_text(message: &ChatMessage) -> bool {
}),
}
}
#[cfg(test)]
mod tests {
use crate::{
Error,
base_llm::{
auth::AuthScheme,
chat::transformation::{BaseConfig, ProviderChatResponseData, Unsupported},
},
bedrock::chat::converse_transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG,
};
use litellm_auth::CredentialPlacement;
use litellm_llms_types::formats::chat_completions::{ChatCompletionsResponse, ChatMessage};
use rstest::rstest;
use serde_json::{Map, Value, json};
fn messages(value: Value) -> Vec<ChatMessage> {
serde_json::from_value(value).expect("valid messages")
}
fn params(value: Value) -> Map<String, Value> {
match value {
Value::Object(map) => map,
other => panic!("params must be an object, got {other}"),
}
}
fn transform(msgs: Value, opts: Value) -> Value {
BEDROCK_CHAT_COMPLETIONS_CONFIG
.transform_request(
"anthropic.claude-sonnet-4-5-v1:0",
messages(msgs),
params(opts),
)
.expect("request transforms")
.body
}
fn transform_response(body: Value) -> Result<ChatCompletionsResponse, Error> {
BEDROCK_CHAT_COMPLETIONS_CONFIG.transform_response(
"anthropic.claude-sonnet-4-5-v1:0",
ProviderChatResponseData { body },
)
}
fn reason(msgs: Value, opts: Value) -> Option<Unsupported> {
BEDROCK_CHAT_COMPLETIONS_CONFIG.unsupported_reason(&messages(msgs), &params(opts))
}
#[test]
fn builds_the_converse_body_python_builds() {
let body = transform(
json!([
{"role": "system", "content": "be terse"},
{"role": "user", "content": "hi"}
]),
json!({"maxTokens": 128, "temperature": 0.2}),
);
assert_eq!(
body,
json!({
"inferenceConfig": {"maxTokens": 128, "temperature": 0.2},
"messages": [{"role": "user", "content": [{"text": "hi"}]}],
"system": [{"text": "be terse"}]
})
);
}
#[test]
fn always_emits_inference_config_even_when_empty() {
let body = transform(json!([{"role": "user", "content": "hi"}]), json!({}));
assert_eq!(body["inferenceConfig"], json!({}));
assert!(body.get("system").is_none());
}
#[test]
fn places_only_inference_params_in_inference_config() {
let body = transform(
json!([{"role": "user", "content": "hi"}]),
json!({
"maxTokens": 64,
"temperature": 0.1,
"topP": 0.9,
"stopSequences": ["STOP"]
}),
);
assert_eq!(
body["inferenceConfig"],
json!({"maxTokens": 64, "temperature": 0.1, "topP": 0.9, "stopSequences": ["STOP"]})
);
assert!(body.get("additionalModelRequestFields").is_none());
}
#[test]
fn merges_consecutive_user_turns_into_one_message() {
let body = transform(
json!([
{"role": "user", "content": "one"},
{"role": "user", "content": [{"type": "text", "text": "two"}]},
{"role": "assistant", "content": "ack"},
{"role": "user", "content": "three"}
]),
json!({}),
);
assert_eq!(
body["messages"],
json!([
{"role": "user", "content": [{"text": "one"}, {"text": "two"}]},
{"role": "assistant", "content": [{"text": "ack"}]},
{"role": "user", "content": [{"text": "three"}]}
])
);
}
#[test]
fn declines_streaming() {
assert_eq!(
reason(
json!([{"role": "user", "content": "hi"}]),
json!({"stream": true})
),
Some(Unsupported("streaming"))
);
}
#[test]
fn declines_top_k_because_python_routes_it_by_base_model() {
assert_eq!(
reason(
json!([{"role": "user", "content": "hi"}]),
json!({"topK": 40})
),
Some(Unsupported("unrecognized request parameter"))
);
}
#[rstest]
#[case::tools(json!({"tools": []}))]
#[case::tool_choice(json!({"tool_choice": {"auto": {}}}))]
#[case::thinking(json!({"thinking": {"type": "enabled"}}))]
#[case::request_metadata(json!({"requestMetadata": {"k": "v"}}))]
#[case::output_config(json!({"outputConfig": {}}))]
#[case::parallel_tool_use_config(json!({"_parallel_tool_use_config": {}}))]
fn declines_tools_and_other_params_outside_the_allowlist(#[case] param: Value) {
assert_eq!(
reason(json!([{"role": "user", "content": "hi"}]), param.clone()),
Some(Unsupported("unrecognized request parameter")),
"expected {param} to decline"
);
}
#[rstest]
#[case::empty_string(json!(""))]
#[case::whitespace_string(json!(" "))]
#[case::whitespace_text_block(json!([{"type": "text", "text": " "}]))]
fn declines_blank_text_rather_than_substituting_the_anthropic_placeholder(
#[case] content: Value,
) {
assert_eq!(
reason(
json!([{"role": "user", "content": content}, {"role": "user", "content": "hi"}]),
json!({})
),
Some(Unsupported("blank message text")),
"expected blank content {content} to decline"
);
}
#[test]
fn declines_a_message_whose_content_list_is_empty() {
// The blank-text check scans parts, so an empty list clears it; Converse
// rejects an empty `content` array, which is a decline the core owes the
// host before the call rather than an error after it.
assert_eq!(
reason(json!([{"role": "user", "content": []}]), json!({})),
Some(Unsupported("message without content"))
);
assert_eq!(
reason(
json!([{"role": "user", "content": [{"type": "text", "text": "hi"}]}]),
json!({})
),
None
);
}
#[test]
fn declines_a_conversation_that_opens_or_closes_on_an_assistant_turn() {
assert_eq!(
reason(
json!([
{"role": "assistant", "content": "prefill"},
{"role": "user", "content": "hi"}
]),
json!({})
),
Some(Unsupported(
"conversation does not run user turn to user turn"
))
);
assert_eq!(
reason(
json!([
{"role": "user", "content": "hi"},
{"role": "assistant", "content": "prefill"}
]),
json!({})
),
Some(Unsupported(
"conversation does not run user turn to user turn"
))
);
}
#[test]
fn accepts_a_user_to_user_text_conversation() {
assert_eq!(
reason(
json!([
{"role": "system", "content": "be terse"},
{"role": "user", "content": "hi"},
{"role": "assistant", "content": "hello"},
{"role": "user", "content": "again"}
]),
json!({"maxTokens": 16})
),
None
);
}
#[test]
fn builds_the_converse_url_from_the_region_in_the_model_id() {
let config = &BEDROCK_CHAT_COMPLETIONS_CONFIG;
assert_eq!(
config
.get_complete_url(None, "us-east-1/anthropic.claude-v2", &Map::new(), &|_| {
None
})
.expect("url builds"),
"https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-v2/converse"
);
}
#[test]
fn falls_back_to_the_region_env_then_the_default_region() {
let config = &BEDROCK_CHAT_COMPLETIONS_CONFIG;
let with_env = |key: &str| (key == "AWS_REGION_NAME").then(|| "eu-west-1".to_string());
assert_eq!(
config
.get_complete_url(None, "anthropic.claude-v2", &Map::new(), &with_env)
.expect("url builds"),
"https://bedrock-runtime.eu-west-1.amazonaws.com/model/anthropic.claude-v2/converse"
);
assert_eq!(
config
.get_complete_url(None, "anthropic.claude-v2", &Map::new(), &|_| None)
.expect("url builds"),
"https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-v2/converse"
);
}
#[test]
fn prefers_an_explicit_runtime_endpoint_over_the_api_base() {
let config = &BEDROCK_CHAT_COMPLETIONS_CONFIG;
let overrides = params(json!({"aws_bedrock_runtime_endpoint": "https://vpce.internal/"}));
assert_eq!(
config
.get_complete_url(
Some("https://ignored.example"),
"anthropic.claude-v2",
&overrides,
&|_| None
)
.expect("url builds"),
"https://vpce.internal/model/anthropic.claude-v2/converse"
);
}
/// The bearer token a config named, or `None` for a SigV4 scheme in the given region.
fn bearer_or_region(auth: AuthScheme) -> Result<String, String> {
match auth {
AuthScheme::Credential {
placement: CredentialPlacement::Bearer,
secret,
} => Ok(secret.expose().to_string()),
AuthScheme::AwsSigV4 {
region,
service: "bedrock",
..
} => Err(region),
other => panic!("unexpected auth {other:?}"),
}
}
#[test]
fn signs_with_sigv4_in_the_resolved_region() {
let validated = BEDROCK_CHAT_COMPLETIONS_CONFIG
.validate_environment(
Vec::new(),
None,
"eu-central-1/anthropic.claude-v2",
&Map::new(),
&|_| None,
)
.expect("auth resolves");
assert_eq!(
bearer_or_region(validated.auth),
Err("eu-central-1".to_string())
);
}
#[test]
fn a_bearer_token_outranks_sigv4_the_way_python_resolves_it() {
// Python's get_request_headers reads `api_key` as the Bedrock bearer token
// and only falls back to the env when the caller passed none, so each case
// pins one of its precedence rules. Signing as the host principal when a
// bearer identity is configured would cross an account and quota boundary.
let bedrock_env =
|key: &str| (key == "AWS_BEARER_TOKEN_BEDROCK").then(|| "from-env".to_string());
let no_env = |_: &str| None;
let resolve = |api_key, env: &dyn Fn(&str) -> Option<String>| {
bearer_or_region(
BEDROCK_CHAT_COMPLETIONS_CONFIG
.validate_environment(
Vec::new(),
api_key,
"eu-central-1/anthropic.claude-v2",
&Map::new(),
env,
)
.expect("auth resolves")
.auth,
)
};
let bearer = |token: &str| Ok(token.to_string());
let sigv4 = Err("eu-central-1".to_string());
// A caller-supplied key is the bearer token, and outranks the env.
assert_eq!(
resolve(Some("bedrock-api-key"), &bedrock_env),
bearer("bedrock-api-key")
);
// No key, so the env supplies it.
assert_eq!(resolve(None, &bedrock_env), bearer("from-env"));
// An empty key is not a bearer token, and deliberately does NOT reach for
// the env, which is what Python's `is not None` check does.
assert_eq!(resolve(Some(""), &bedrock_env), sigv4);
// Whitespace is truthy in Python, so it stays a bearer token rather than
// silently becoming a host-credentialed SigV4 request.
assert_eq!(resolve(Some(" "), &no_env), bearer(" "));
// Neither present, so SigV4 as before.
assert_eq!(resolve(None, &no_env), sigv4);
}
#[test]
fn normalizes_a_converse_response_into_openai_shape() {
let response = transform_response(json!({
"output": {"message": {"role": "assistant", "content": [
{"text": "hello"}, {"text": " there"}
]}},
"stopReason": "end_turn",
"usage": {"inputTokens": 11, "outputTokens": 4, "totalTokens": 15}
}))
.expect("response transforms");
assert_eq!(response.model, "anthropic.claude-sonnet-4-5-v1:0");
assert_eq!(
response.choices[0].message.content.as_deref(),
Some("hello there")
);
assert_eq!(response.choices[0].finish_reason, "stop");
assert_eq!(response.usage.prompt_tokens, 11);
assert_eq!(response.usage.completion_tokens, 4);
assert_eq!(response.usage.total_tokens, 15);
}
#[test]
fn maps_converse_stop_reasons_python_maps() {
for (provider_reason, expected) in [
("end_turn", "stop"),
("stop_sequence", "stop"),
("max_tokens", "length"),
("guardrail_intervened", "content_filter"),
// Converse emits this one, and Python's `_FINISH_REASON_MAP` carries
// it. Folding it into `stop` reports a filtered completion as a normal
// one to anything keying on the finish reason.
("content_filtered", "content_filter"),
("content_filter", "content_filter"),
] {
let response = transform_response(json!({
"output": {"message": {"content": [{"text": "x"}]}},
"stopReason": provider_reason,
"usage": {"inputTokens": 1, "outputTokens": 1}
}))
.expect("response transforms");
assert_eq!(
response.choices[0].finish_reason, expected,
"stopReason {provider_reason}"
);
}
}
#[test]
fn reports_an_empty_converse_answer_as_an_empty_string_not_null() {
// Converse assigns the joined text unconditionally
// (`chat_completion_message["content"] = content_str`), unlike Anthropic's
// `merged_text or None`, so an empty answer is `""` on both paths. A caller
// calling `.strip()` on it would break on the Rust path alone. Reachable
// through a filtered or guardrail-intervened response.
for content in [json!([]), json!([{"text": ""}])] {
let response = transform_response(json!({
"output": {"message": {"content": content}},
"stopReason": "content_filtered",
"usage": {"inputTokens": 1, "outputTokens": 0}
}))
.expect("response transforms");
assert_eq!(response.choices[0].message.content, Some(String::new()));
}
}
#[test]
fn reports_the_total_tokens_converse_sent_rather_than_recomputing_them() {
// Python reads `usage["totalTokens"]` straight through here, where Anthropic
// has no such field and adds the two counts instead. The two agree while the
// gate declines every cache_control request, so this is what keeps them
// agreeing if that ever widens.
let response = transform_response(json!({
"output": {"message": {"content": [{"text": "x"}]}},
"stopReason": "end_turn",
"usage": {"inputTokens": 10, "outputTokens": 4, "cacheReadInputTokens": 7, "totalTokens": 14}
}))
.expect("response transforms");
assert_eq!(
response.usage.total_tokens, 14,
"provider total was recomputed"
);
assert_eq!(response.usage.prompt_tokens, 17);
assert_eq!(response.usage.completion_tokens, 4);
}
#[test]
fn falls_back_to_the_computed_total_when_converse_omits_it() {
// Python raises a KeyError on a body with no `totalTokens`. Reporting a zero
// instead would be a worse divergence than the one above, so the computed
// total stands in.
let response = transform_response(json!({
"output": {"message": {"content": [{"text": "x"}]}},
"stopReason": "end_turn",
"usage": {"inputTokens": 10, "outputTokens": 4}
}))
.expect("response transforms");
assert_eq!(response.usage.total_tokens, 14);
}
#[test]
fn declines_a_cache_control_message_so_widening_the_gate_is_a_red_test() {
// Converse only reports cache token counts when the request carries a
// cachePoint block, which is why the provider total and the computed one
// cannot disagree today. This is the tripwire: whoever widens the gate to
// admit prompt caching has to come back and re-check the usage mapping
// rather than discovering a silent number change in production.
assert_eq!(
reason(
json!([{"role": "user", "content": [
{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}
]}]),
json!({})
),
Some(Unsupported("non-text message content"))
);
}
#[test]
fn folds_converse_cache_tokens_into_prompt_tokens() {
let response = transform_response(json!({
"output": {"message": {"content": [{"text": "x"}]}},
"stopReason": "end_turn",
"usage": {
"inputTokens": 10,
"outputTokens": 2,
"cacheReadInputTokens": 5,
"cacheWriteInputTokens": 3
}
}))
.expect("response transforms");
assert_eq!(response.usage.prompt_tokens, 18);
assert_eq!(response.usage.prompt_tokens_details.cached_tokens, 5);
assert_eq!(
response.usage.prompt_tokens_details.cache_creation_tokens,
3
);
assert_eq!(response.usage.prompt_tokens_details.text_tokens, 10);
}
#[test]
fn declines_a_response_carrying_a_tool_use_block() {
let err = transform_response(json!({
"output": {"message": {"content": [
{"toolUse": {"toolUseId": "t1", "name": "f", "input": {}}}
]}},
"stopReason": "tool_use",
"usage": {"inputTokens": 1, "outputTokens": 1}
}))
.expect_err("tool use block");
assert_eq!(err, Error::Unsupported("non-text response content block"));
}
#[test]
fn errors_on_a_response_missing_required_fields() {
assert_eq!(
transform_response(json!("nope")).expect_err("not an object"),
Error::InvalidResponse("converse response is not an object".to_string().into())
);
assert_eq!(
transform_response(json!({"usage": {}})).expect_err("no output"),
Error::InvalidResponse("invalid Converse response: missing field `output`".into())
);
assert_eq!(
transform_response(json!({"output": {"message": {"content": []}}}))
.expect_err("no usage"),
Error::InvalidResponse("invalid Converse response: missing field `usage`".into())
);
}
#[test]
fn rejects_malformed_text_and_token_counts() {
for response in [
json!({"output": {"message": {"content": [{"text": 123}]}}, "usage": {"inputTokens": 1, "outputTokens": 1}}),
json!({"output": {"message": {"content": [{"text": "x"}]}}, "usage": {"inputTokens": "1", "outputTokens": 1}}),
] {
assert!(matches!(
transform_response(response),
Err(Error::InvalidResponse(message)) if message.to_string().starts_with("invalid Converse response:")
));
}
}
#[test]
fn accepts_aws_call_configuration_without_serializing_it() {
let call_config = json!({
"maxTokens": 16,
"aws_access_key_id": "AKIA",
"aws_secret_access_key": "secret",
"aws_session_token": "token",
"aws_region_name": "us-east-1",
"aws_profile_name": "litellm-stage",
"aws_role_name": "role",
"aws_session_name": "session",
"aws_web_identity_token": "wit",
"aws_sts_endpoint": "https://sts.example",
"aws_external_id": "ext",
"aws_bedrock_runtime_endpoint": "https://vpce.internal"
});
assert_eq!(
reason(
json!([{"role": "user", "content": "hi"}]),
call_config.clone()
),
None
);
let body = transform(json!([{"role": "user", "content": "hi"}]), call_config);
assert_eq!(
body,
json!({
"inferenceConfig": {"maxTokens": 16},
"messages": [{"role": "user", "content": [{"text": "hi"}]}]
}),
"aws call configuration must not reach the Converse body"
);
}
#[test]
fn leaves_a_complete_converse_url_untouched() {
let config = &BEDROCK_CHAT_COMPLETIONS_CONFIG;
let already_built = "https://bedrock-runtime.us-east-1.amazonaws.com/model/us.anthropic.claude-v2%3A0/converse";
assert_eq!(
config
.get_complete_url(
Some(already_built),
"anthropic.claude-v2",
&Map::new(),
&|_| None
)
.expect("url builds"),
already_built,
"a host that encoded the model id itself must not have it re-derived"
);
}
#[rstest]
#[case::full_static_pair(
json!({"aws_access_key_id": "AKIAHOST", "aws_secret_access_key": "hostsecret", "aws_session_token": "hosttoken"}),
Some(("AKIAHOST", "hostsecret", Some("hosttoken")))
)]
#[case::pair_without_session_token(
json!({"aws_access_key_id": "AKIAHOST", "aws_secret_access_key": "hostsecret"}),
Some(("AKIAHOST", "hostsecret", None))
)]
#[case::key_id_alone(json!({"aws_access_key_id": "AKIA"}), None)]
#[case::blank_key_id(json!({"aws_access_key_id": " ", "aws_secret_access_key": "s"}), None)]
#[case::nothing(json!({}), None)]
fn host_supplied_credentials_need_a_full_static_pair(
#[case] optional_params: Value,
#[case] expected: Option<(&str, &str, Option<&str>)>,
) {
use litellm_auth::AwsParams;
use litellm_auth_aws::host_supplied_credentials;
let credentials =
host_supplied_credentials(&AwsParams::from_optional_params(&params(optional_params)));
assert_eq!(
credentials.as_ref().map(|credentials| (
credentials.access_key_id(),
credentials.secret_access_key(),
credentials.session_token(),
)),
expected
);
}
}

9
litellm-rust/crates/llms/src/bedrock/messages/AGENTS.md
View file

@ -1,3 +1,8 @@
This directory owns Bedrock's Messages adapter: its endpoints, authentication policy, wire adaptation, and response decoding. Implement the shared adapter contract from `base_llm/messages`, consume API data contracts from `litellm-llms-types::formats::messages`, and leave call orchestration to `inference-messages`
# rules
The Claude adapter may explicitly reuse payload policy from `anthropic/messages` when it applies to Bedrock's Claude backend. Keep Bedrock-specific differences here. Sharing that helper does not make Anthropic policy a format-wide default or justify a dependency from `base_llm/messages` on provider implementations
- Claude on Bedrock InvokeModel: reuses `anthropic/messages` shaping, moves the model into the URL, and decodes the AWS event stream
- Invocation metrics in the response map onto Messages usage here
# references
- https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages.html

7
litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.rs
View file

@ -19,6 +19,9 @@ use litellm_llms_types::formats::messages::{
MessagesRequest,
streaming::{MessagesStreamEvent, MessagesStreamUsage},
};
use litellm_llms_types::providers::bedrock::{
INVOCATION_METRICS_KEY, INVOKE_PATH, INVOKE_STREAM_PATH,
};
use litellm_router_types::LitellmParams;
use serde_json::{Map, Value};
@ -35,8 +38,6 @@ use crate::{
bedrock::chat::invoke_handler::{decode_invoke_anthropic_chunk, invoke_chunk_stream},
};
const INVOCATION_METRICS_KEY: &str = "amazon-bedrock-invocationMetrics";
const METRICS_USAGE_KEYS: [(&str, &str); 4] = [
("input_tokens", "inputTokenCount"),
("output_tokens", "outputTokenCount"),
@ -44,8 +45,6 @@ const METRICS_USAGE_KEYS: [(&str, &str); 4] = [
("cache_creation_input_tokens", "cacheWriteInputTokenCount"),
];
const INVOKE_PATH: &str = "invoke";
const INVOKE_STREAM_PATH: &str = "invoke-with-response-stream";
const INVOKE_MODEL_PREFIX: &str = "invoke/";
const SECRET_NAMES: &[&str] = &[

7
litellm-rust/crates/llms/src/cohere/ocr/AGENTS.md Normal file
View file

@ -0,0 +1,7 @@
# rules
- Translates Cohere Parse output into the shared OCR format. Azure AI reuses this adapter for its hosted Cohere
# references
- https://docs.cohere.com/reference/parse

9
litellm-rust/crates/llms/src/deepseek/messages/AGENTS.md Normal file
View file

@ -0,0 +1,9 @@
# rules
- DeepSeek's Anthropic-compatible host: reuses `anthropic/messages` shaping with `x-api-key` auth
- Drops the explicit `type: custom` tool discriminator and Claude Code billing system blocks, which DeepSeek rejects
- Peels OpenAI-style suffixes off a configured base before appending `/anthropic/v1/messages`
# references
- https://api-docs.deepseek.com/guides/anthropic_api

18
litellm-rust/crates/llms/src/deepseek/messages/transformation.rs
View file

@ -2,6 +2,7 @@ use litellm_auth::{CredentialPlacement, SecretValue};
use litellm_core_utils::settings::resolve_non_empty;
use litellm_llms_types::{
formats::messages::{MessagesOptionalParams, MessagesRequest, MessagesTool},
providers::anthropic::{DEFAULT_HEADERS, MESSAGES_PATH},
recognized::Recognized,
};
use litellm_router_types::LitellmParams;
@ -13,17 +14,12 @@ use crate::{
common_utils::{filter_billing_headers_from_system, has_anthropic_credential},
messages::{
handler::shape_anthropic_messages_request,
transformation::{
DEFAULT_HEADERS, transform_messages_request, update_headers_with_anthropic_beta,
},
transformation::{transform_messages_request, update_headers_with_anthropic_beta},
},
},
base_llm::{
auth::{AuthScheme, Headers, ValidatedEnvironment},
messages::{
context::MessagesTransformContext,
transformation::{BaseMessagesConfig, MESSAGES_PATH_SUFFIX},
},
messages::{context::MessagesTransformContext, transformation::BaseMessagesConfig},
},
};
@ -166,18 +162,18 @@ pub fn complete_deepseek_anthropic_url(
) -> String {
let api_base = get_api_base(api_base, env_lookup);
let api_base = api_base.trim_end_matches('/');
if api_base.ends_with(MESSAGES_PATH_SUFFIX) && api_base.contains("/anthropic/") {
if api_base.ends_with(MESSAGES_PATH) && api_base.contains("/anthropic/") {
return api_base.to_string();
}
let api_base = [MESSAGES_PATH_SUFFIX, "/v1", "/beta"]
let api_base = [MESSAGES_PATH, "/v1", "/beta"]
.into_iter()
.fold(api_base, |base, suffix| {
base.strip_suffix(suffix).unwrap_or(base)
});
if api_base.ends_with(ANTHROPIC_PATH_SEGMENT) || api_base.contains("/anthropic/") {
return format!("{api_base}{MESSAGES_PATH_SUFFIX}");
return format!("{api_base}{MESSAGES_PATH}");
}
format!("{api_base}{ANTHROPIC_PATH_SEGMENT}{MESSAGES_PATH_SUFFIX}")
format!("{api_base}{ANTHROPIC_PATH_SEGMENT}{MESSAGES_PATH}")
}
/// DeepSeek rejects Anthropic's explicit `{"type": "custom"}` tool discriminator, so it is

8
litellm-rust/crates/llms/src/mistral/ocr/AGENTS.md Normal file
View file

@ -0,0 +1,8 @@
# rules
- Mistral is the origin of the shared OCR shape, documented in `llms-types/src/formats/ocr/AGENTS.md`. This adapter adds only auth and URL handling
- Azure AI and Vertex AI reuse `MistralOcrRequest` for their hosted Mistral OCR
# references
- https://docs.mistral.ai/capabilities/document_ai/basic_ocr

Some files were not shown because too many files have changed in this diff Show more