mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
wip
This commit is contained in:
parent
c6033b32c1
commit
ec0ffb9d42
19 changed files with 50 additions and 138 deletions
|
|
@ -1,75 +1 @@
|
|||
# CLAUDE.md
|
||||
|
||||
Rules for `litellm-rust/crates/core`.
|
||||
|
||||
## Responsibility
|
||||
|
||||
`core` is the LiteLLM SDK in Rust: it makes the LLM call. Every top-level
|
||||
LiteLLM call has a public entrypoint here, named after the route
|
||||
(`messages::messages()` is the Rust equivalent of `litellm.messages()`), and
|
||||
calling it returns a typed non-streaming response.
|
||||
|
||||
Allowed:
|
||||
- The public entrypoint for a route, plus its `<route>_stream` variant when the
|
||||
route supports streaming.
|
||||
- Provider resolution, auth header construction, URL building, and the provider
|
||||
HTTP call (shared reused client, connect + request timeouts).
|
||||
- Shared request/response structs.
|
||||
- Typed errors with stable, non-sensitive messages.
|
||||
- Deterministic validation helpers.
|
||||
- Serialization helpers that intentionally mirror Python output shape.
|
||||
- Route templates that match Python base config responsibilities, such as
|
||||
`messages::transformation::AnthropicMessagesProviderConfig`.
|
||||
|
||||
Not allowed:
|
||||
- Serving HTTP: axum routers, extractors, and other transport concerns.
|
||||
- Filesystem, database, or cache access.
|
||||
- Config file reading or rollout state; the host resolves those and passes them
|
||||
in. Env reads are limited to credential fallback in a route's `prepare.rs`.
|
||||
- Logging callbacks, tracing spans, spend writes, or customer callbacks.
|
||||
- Provider-specific branching that belongs in `providers`.
|
||||
- Panics for user/provider-controlled input.
|
||||
|
||||
## Typed Contracts (core rule)
|
||||
|
||||
Trait and function boundaries MUST be strongly typed. No stringly-typed JSON
|
||||
(`&str` / `String` / `Vec<String>` / bare `serde_json::Value`) as a transform
|
||||
input or output. Parse wire bytes into typed structs/enums at the host edge;
|
||||
`core` and `providers` operate only on those types (e.g. `RealtimeEvent`,
|
||||
`RealtimeTransformResult`, `OcrRequestData`). A `type`-style discriminator is a
|
||||
typed field on a struct, not a raw string threaded through the API.
|
||||
|
||||
## Structure
|
||||
|
||||
Use route names directly under `src/`: `messages`, `ocr`, future
|
||||
`chat_completions`, `embeddings`, and similar top-level LiteLLM calls. Do not
|
||||
invent broad names like `engine` for route contracts.
|
||||
|
||||
`src/messages` is the reference shape for a route module:
|
||||
|
||||
```
|
||||
mod.rs pub async fn messages(..) (+ messages_stream)
|
||||
types.rs request/response types
|
||||
transformation.rs the provider template trait
|
||||
prepare.rs provider resolution, auth headers, URL
|
||||
handler.rs the provider call
|
||||
client.rs the shared reqwest client
|
||||
```
|
||||
|
||||
OCR keeps each concern independently changeable:
|
||||
|
||||
```
|
||||
ocr/formats/<format>/ typed wire shape and transforms
|
||||
ocr/backends/<backend>.rs shared auth, upload, polling, and hosting behavior
|
||||
ocr/integrations/<name>.rs one typed format/backend pairing
|
||||
ocr/registry.rs provider/model resolution and generated dispatch
|
||||
```
|
||||
|
||||
## Parity Rules
|
||||
|
||||
- Every shared type used by a provider transform needs unit tests for
|
||||
serialization shape.
|
||||
- If Python parity requires always emitting a `null` field instead of omitting
|
||||
it, document that in code and pin it with a test.
|
||||
- Error enums should preserve enough detail for Python/HTTP hosts to map errors
|
||||
consistently without exposing document contents or upstream bodies.
|
||||
See ./AGENTS.md
|
||||
|
|
|
|||
|
|
@ -118,7 +118,10 @@ impl MediaFetcher {
|
|||
}
|
||||
|
||||
async fn validate_url(&self, url: &Url) -> Result<(), MediaError> {
|
||||
if !matches!(url.scheme(), "http" | "https") {
|
||||
if !matches!(url.scheme(), "http" | "https")
|
||||
|| !url.username().is_empty()
|
||||
|| url.password().is_some()
|
||||
{
|
||||
return Err(MediaError::BlockedUrl);
|
||||
}
|
||||
let host = url.host_str().ok_or(MediaError::BlockedUrl)?;
|
||||
|
|
@ -355,4 +358,15 @@ mod tests {
|
|||
};
|
||||
assert!(error.to_string().contains("network policy"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejects_url_credentials_before_network_access() {
|
||||
let fetcher = MediaFetcher::new().expect("media fetcher builds");
|
||||
let url =
|
||||
Url::parse("https://user:password@8.8.8.8/document").expect("credentialed URL parses");
|
||||
assert!(matches!(
|
||||
fetcher.validate_url(&url).await,
|
||||
Err(MediaError::BlockedUrl)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -36,7 +36,6 @@ macro_rules! ocr_format {
|
|||
|
||||
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
|
||||
fn map_params(
|
||||
&self,
|
||||
params: Self::InputParams,
|
||||
) -> Result<Self::MappedParams, $crate::ocr::error::OcrRequestError> {
|
||||
$map(params)
|
||||
|
|
@ -44,7 +43,6 @@ macro_rules! ocr_format {
|
|||
|
||||
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
|
||||
fn transform_request(
|
||||
&self,
|
||||
model: &str,
|
||||
document: Self::PreparedDocument,
|
||||
params: &Self::MappedParams,
|
||||
|
|
@ -54,7 +52,6 @@ macro_rules! ocr_format {
|
|||
|
||||
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
|
||||
fn transform_response(
|
||||
&self,
|
||||
model: &str,
|
||||
response: Self::ResponseBody,
|
||||
params: &Self::MappedParams,
|
||||
|
|
|
|||
|
|
@ -26,15 +26,13 @@ pub trait OcrFormat: Send + Sync + Sized + 'static {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn map_params(&self, params: Self::InputParams) -> Result<Self::MappedParams, OcrRequestError>;
|
||||
fn map_params(params: Self::InputParams) -> Result<Self::MappedParams, OcrRequestError>;
|
||||
fn transform_request(
|
||||
&self,
|
||||
model: &str,
|
||||
document: Self::PreparedDocument,
|
||||
params: &Self::MappedParams,
|
||||
) -> Result<Self::RequestBody, OcrRequestError>;
|
||||
fn transform_response(
|
||||
&self,
|
||||
model: &str,
|
||||
response: Self::ResponseBody,
|
||||
params: &Self::MappedParams,
|
||||
|
|
|
|||
|
|
@ -104,7 +104,6 @@ where
|
|||
I: OcrIntegration,
|
||||
{
|
||||
let backend = &request.integration;
|
||||
let format = &I::FORMAT;
|
||||
let prepared_backend = backend
|
||||
.prepare(
|
||||
&request.connection,
|
||||
|
|
@ -133,7 +132,7 @@ where
|
|||
let document = backend
|
||||
.prepare_document(client, document, &request.connection, &headers)
|
||||
.await?;
|
||||
let body = format.transform_request(&request.model, document.into(), &request.params)?;
|
||||
let body = I::Format::transform_request(&request.model, document.into(), &request.params)?;
|
||||
let body = if I::GUARDRAIL_STAGE == GuardrailStage::RequestBody && hooks.has_guardrails() {
|
||||
let guarded = hooks
|
||||
.during_call(OcrDuringCallRequest {
|
||||
|
|
@ -215,7 +214,6 @@ async fn send_ocr_call<I: OcrIntegration>(
|
|||
body,
|
||||
} = request;
|
||||
let backend = &integration;
|
||||
let format = I::FORMAT;
|
||||
let builder = client
|
||||
.provider_http()
|
||||
.post(&url)
|
||||
|
|
@ -229,7 +227,7 @@ async fn send_ocr_call<I: OcrIntegration>(
|
|||
let decoded = backend
|
||||
.read_response(client, response, &url, &headers, &connection, ¶ms)
|
||||
.await?;
|
||||
let response = format.transform_response(&model, decoded.data, ¶ms)?;
|
||||
let response = I::Format::transform_response(&model, decoded.data, ¶ms)?;
|
||||
Ok(OcrResponseData {
|
||||
provider_native_response: decoded.native,
|
||||
..response
|
||||
|
|
|
|||
|
|
@ -118,7 +118,7 @@ where
|
|||
})?,
|
||||
"guardrail.optional_params",
|
||||
)
|
||||
.and_then(|params| I::FORMAT.map_params(params))?;
|
||||
.and_then(I::Format::map_params)?;
|
||||
Ok(Ok(MappedOcrRequest {
|
||||
document: changed.document,
|
||||
params,
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ impl OcrIntegration for AzureDocumentIntelligence {
|
|||
type Backend = AzureBackend;
|
||||
type Format = AzureDocumentIntelligenceOcrFormat;
|
||||
type DocumentPreparation = super::PassThrough;
|
||||
const FORMAT: Self::Format = AzureDocumentIntelligenceOcrFormat;
|
||||
|
||||
async fn prepare(
|
||||
&self,
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@ impl OcrIntegration for AzureMistral {
|
|||
type Backend = AzureBackend;
|
||||
type Format = MistralOcrFormat;
|
||||
type DocumentPreparation = super::RequireInline;
|
||||
const FORMAT: Self::Format = MistralOcrFormat;
|
||||
|
||||
async fn prepare(
|
||||
&self,
|
||||
|
|
|
|||
|
|
@ -13,7 +13,6 @@ impl OcrIntegration for MistralDirect {
|
|||
type Backend = MistralBackend;
|
||||
type Format = MistralOcrFormat;
|
||||
type DocumentPreparation = super::PassThrough;
|
||||
const FORMAT: Self::Format = MistralOcrFormat;
|
||||
|
||||
async fn prepare(
|
||||
&self,
|
||||
|
|
|
|||
|
|
@ -39,7 +39,6 @@ pub trait OcrIntegration: Send + Sync + Sized + 'static {
|
|||
type DocumentPreparation: DocumentPreparation<
|
||||
Output: Into<<Self::Format as OcrFormat>::PreparedDocument>,
|
||||
>;
|
||||
const FORMAT: Self::Format;
|
||||
const GUARDRAIL_STAGE: GuardrailStage = GuardrailStage::RequestBody;
|
||||
|
||||
fn supports_native_request_format(&self) -> bool {
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ impl OcrIntegration for ReductoLegacy {
|
|||
type Backend = ReductoBackend;
|
||||
type Format = ReductoParseLegacyFormat;
|
||||
type DocumentPreparation = super::ReductoUpload;
|
||||
const FORMAT: Self::Format = ReductoParseLegacyFormat;
|
||||
const GUARDRAIL_STAGE: super::GuardrailStage = super::GuardrailStage::Document;
|
||||
|
||||
async fn prepare(
|
||||
&self,
|
||||
|
|
@ -29,8 +29,6 @@ impl OcrIntegration for ReductoLegacy {
|
|||
})
|
||||
}
|
||||
|
||||
const GUARDRAIL_STAGE: super::GuardrailStage = super::GuardrailStage::Document;
|
||||
|
||||
fn preserve_native_response(&self, _params: &ReductoLegacyParams) -> bool {
|
||||
true
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ impl OcrIntegration for ReductoV3 {
|
|||
type Backend = ReductoBackend;
|
||||
type Format = ReductoParseV3Format;
|
||||
type DocumentPreparation = super::ReductoUpload;
|
||||
const FORMAT: Self::Format = ReductoParseV3Format;
|
||||
const GUARDRAIL_STAGE: super::GuardrailStage = super::GuardrailStage::Document;
|
||||
|
||||
async fn prepare(
|
||||
&self,
|
||||
|
|
@ -29,8 +29,6 @@ impl OcrIntegration for ReductoV3 {
|
|||
})
|
||||
}
|
||||
|
||||
const GUARDRAIL_STAGE: super::GuardrailStage = super::GuardrailStage::Document;
|
||||
|
||||
fn preserve_native_response(&self, _params: &ReductoV3Params) -> bool {
|
||||
true
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@ impl OcrIntegration for VertexDeepSeek {
|
|||
type Backend = VertexBackend;
|
||||
type Format = DeepSeekOcrFormat;
|
||||
type DocumentPreparation = super::PassThrough;
|
||||
const FORMAT: Self::Format = DeepSeekOcrFormat;
|
||||
|
||||
async fn prepare(
|
||||
&self,
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@ impl OcrIntegration for VertexMistral {
|
|||
type Backend = VertexBackend;
|
||||
type Format = MistralOcrFormat;
|
||||
type DocumentPreparation = super::RequireInline;
|
||||
const FORMAT: Self::Format = MistralOcrFormat;
|
||||
|
||||
async fn prepare(
|
||||
&self,
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ pub(crate) fn prepare_ocr_call<I>(
|
|||
where
|
||||
I: OcrIntegration,
|
||||
{
|
||||
let params = I::FORMAT.map_params(params)?;
|
||||
let params = I::Format::map_params(params)?;
|
||||
Ok(MappedOcrRequest {
|
||||
integration,
|
||||
model,
|
||||
|
|
|
|||
|
|
@ -76,9 +76,10 @@ fn invalid_page_types_return_typed_errors(#[case] pages: Value) {
|
|||
fn invalid_page_values_never_panic(#[case] pages: Value) {
|
||||
let input = serde_json::from_value(json!({"pages":pages})).unwrap();
|
||||
assert!(
|
||||
crate::ocr::registry::AzureDocumentIntelligence::FORMAT
|
||||
.map_params(input)
|
||||
.is_err()
|
||||
crate::ocr::formats::document_intelligence::AzureDocumentIntelligenceOcrFormat::map_params(
|
||||
input
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
#[tokio::test]
|
||||
|
|
@ -109,9 +110,10 @@ async fn document_intelligence_maps_features(#[case] features: Value, #[case] ex
|
|||
#[case(json!(""))]
|
||||
fn document_intelligence_rejects_invalid_features(#[case] features: Value) {
|
||||
assert!(
|
||||
crate::ocr::registry::AzureDocumentIntelligence::FORMAT
|
||||
.map_params(serde_json::from_value(json!({"features":features})).unwrap())
|
||||
.is_err()
|
||||
crate::ocr::formats::document_intelligence::AzureDocumentIntelligenceOcrFormat::map_params(
|
||||
serde_json::from_value(json!({"features":features})).unwrap()
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
#[tokio::test]
|
||||
|
|
|
|||
|
|
@ -13,8 +13,7 @@ use serde_json::json;
|
|||
#[case(json!({"bbox_annotation_format":{"type":"json_schema","schema":{"type":"object"}},"document_annotation_format":{"type":"json_schema"},"document_annotation_prompt":"extract"}))]
|
||||
fn transform_ocr_request_includes_each_optional_param(#[case] options: serde_json::Value) {
|
||||
let document = json!({"type":"document_url","document_url":"https://example.com/a.pdf"});
|
||||
let result = format_body(
|
||||
&MistralOcrFormat,
|
||||
let result = format_body::<MistralOcrFormat>(
|
||||
"model",
|
||||
serde_json::from_value::<OcrDocument>(document.clone()).unwrap(),
|
||||
options.clone(),
|
||||
|
|
@ -29,8 +28,7 @@ fn transform_ocr_request_includes_each_optional_param(#[case] options: serde_jso
|
|||
|
||||
#[test]
|
||||
fn map_ocr_params_drops_unknown_params() {
|
||||
let result = format_body(
|
||||
&MistralOcrFormat,
|
||||
let result = format_body::<MistralOcrFormat>(
|
||||
"model",
|
||||
serde_json::from_value::<OcrDocument>(
|
||||
json!({"type":"image_url","image_url":"data:image/png;base64,YWJj"}),
|
||||
|
|
@ -46,7 +44,7 @@ fn map_ocr_params_drops_unknown_params() {
|
|||
#[test]
|
||||
fn transform_ocr_response_preserves_ocr4_page_fields() {
|
||||
let page = json!({"index":0,"markdown":"hello","header":"head","footer":"foot","confidence_scores":{"mean":0.99},"blocks":[{"type":"text","content":"hello"}],"images":[{"id":"img-1","image_base64":"YWJj","top_left_x":12}],"dimensions":{"width":100,"height":200,"dpi":96}});
|
||||
let response = transform_format(&MistralOcrFormat, "model", json!({"pages":[page.clone()],"model":"returned-model","usage_info":{"pages_processed":1,"future_counter":5}}), json!({}), false).unwrap();
|
||||
let response = transform_format::<MistralOcrFormat>("model", json!({"pages":[page.clone()],"model":"returned-model","usage_info":{"pages_processed":1,"future_counter":5}}), json!({}), false).unwrap();
|
||||
for (name, value) in page.as_object().unwrap() {
|
||||
if name != "images" {
|
||||
assert_eq!(&response["pages"][0][name], value);
|
||||
|
|
@ -59,8 +57,7 @@ fn transform_ocr_response_preserves_ocr4_page_fields() {
|
|||
|
||||
#[test]
|
||||
fn transform_ocr_response_normalizes_mistral_json() {
|
||||
let result = transform_format(
|
||||
&MistralOcrFormat,
|
||||
let result = transform_format::<MistralOcrFormat>(
|
||||
"model",
|
||||
json!({"pages":[{"index":"0","markdown":"hello"}],"usage_info":{"pages_processed":"1"}}),
|
||||
json!({}),
|
||||
|
|
@ -72,14 +69,8 @@ fn transform_ocr_response_normalizes_mistral_json() {
|
|||
assert_eq!(result["pages"][0]["images"], serde_json::Value::Null);
|
||||
assert!(result.get("provider_native_response").is_none());
|
||||
assert!(
|
||||
transform_format(
|
||||
&MistralOcrFormat,
|
||||
"model",
|
||||
json!({"pages":null}),
|
||||
json!({}),
|
||||
false
|
||||
)
|
||||
.is_err()
|
||||
transform_format::<MistralOcrFormat>("model", json!({"pages":null}), json!({}), false)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -27,9 +27,7 @@ pub(crate) fn params<I>(_integration: &I, value: Value) -> MappedParams<I>
|
|||
where
|
||||
I: OcrIntegration,
|
||||
{
|
||||
I::FORMAT
|
||||
.map_params(serde_json::from_value(value).unwrap())
|
||||
.unwrap()
|
||||
I::Format::map_params(serde_json::from_value(value).unwrap()).unwrap()
|
||||
}
|
||||
|
||||
pub(crate) fn transform<I>(
|
||||
|
|
@ -41,13 +39,12 @@ pub(crate) fn transform<I>(
|
|||
where
|
||||
I: OcrIntegration,
|
||||
{
|
||||
let params = I::FORMAT.map_params(serde_json::from_value(options.clone()).unwrap())?;
|
||||
let params = I::Format::map_params(serde_json::from_value(options.clone()).unwrap())?;
|
||||
let preserve_native_response = integration.preserve_native_response(¶ms);
|
||||
transform_format(&I::FORMAT, model, value, options, preserve_native_response)
|
||||
transform_format::<I::Format>(model, value, options, preserve_native_response)
|
||||
}
|
||||
|
||||
pub(crate) fn transform_format<F>(
|
||||
format: &F,
|
||||
model: &str,
|
||||
value: Value,
|
||||
options: Value,
|
||||
|
|
@ -56,12 +53,12 @@ pub(crate) fn transform_format<F>(
|
|||
where
|
||||
F: OcrFormat,
|
||||
{
|
||||
let params = format.map_params(serde_json::from_value(options).unwrap())?;
|
||||
let params = F::map_params(serde_json::from_value(options).unwrap())?;
|
||||
let decoded = decode_response(
|
||||
&serde_json::to_vec(&value).unwrap(),
|
||||
preserve_native_response,
|
||||
)?;
|
||||
let mut response = format.transform_response(model, decoded.data, ¶ms)?;
|
||||
let mut response = F::transform_response(model, decoded.data, ¶ms)?;
|
||||
response.provider_native_response = decoded.native;
|
||||
Ok(response.into_json())
|
||||
}
|
||||
|
|
@ -84,11 +81,10 @@ where
|
|||
&[],
|
||||
)
|
||||
.await?;
|
||||
format_body(&I::FORMAT, model, document.into(), options)
|
||||
format_body::<I::Format>(model, document.into(), options)
|
||||
}
|
||||
|
||||
pub(crate) fn format_body<F>(
|
||||
format: &F,
|
||||
model: &str,
|
||||
document: F::PreparedDocument,
|
||||
options: Value,
|
||||
|
|
@ -96,8 +92,8 @@ pub(crate) fn format_body<F>(
|
|||
where
|
||||
F: OcrFormat,
|
||||
{
|
||||
let params = format.map_params(serde_json::from_value(options).unwrap())?;
|
||||
Ok(serde_json::to_value(format.transform_request(model, document, ¶ms)?).unwrap())
|
||||
let params = F::map_params(serde_json::from_value(options).unwrap())?;
|
||||
Ok(serde_json::to_value(F::transform_request(model, document, ¶ms)?).unwrap())
|
||||
}
|
||||
|
||||
pub(crate) fn wire_request(model: &str, base: &str, options: Value) -> OcrRequest {
|
||||
|
|
|
|||
|
|
@ -8,8 +8,7 @@ use serde_json::json;
|
|||
#[case("deepseek-ocr-maas")]
|
||||
#[case("deepseek-ai/deepseek-ocr-maas")]
|
||||
fn deepseek_request_uses_single_provider_namespace(#[case] model: &str) {
|
||||
let result = format_body(
|
||||
&DeepSeekOcrFormat,
|
||||
let result = format_body::<DeepSeekOcrFormat>(
|
||||
model,
|
||||
serde_json::from_value::<OcrDocument>(
|
||||
json!({"type":"image_url","image_url":"gs://bucket/a.png"}),
|
||||
|
|
@ -39,8 +38,7 @@ fn vertex_deepseek_response_wraps_markdown_content(
|
|||
#[case] content: serde_json::Value,
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
let result = transform_format(
|
||||
&DeepSeekOcrFormat,
|
||||
let result = transform_format::<DeepSeekOcrFormat>(
|
||||
"model",
|
||||
json!({"choices":[{"message":{"content":content}}],"usage":{"prompt_tokens":1}}),
|
||||
json!({}),
|
||||
|
|
@ -59,6 +57,8 @@ fn deepseek_rejects_missing_content_and_invalid_structured_pages() {
|
|||
json!({"choices":[{"message":{"content":""}}]}),
|
||||
json!({"choices":[{"message":{"content":{"pages":[{"markdown":42}]}}}]}),
|
||||
] {
|
||||
assert!(transform_format(&DeepSeekOcrFormat, "model", response, json!({}), false).is_err());
|
||||
assert!(
|
||||
transform_format::<DeepSeekOcrFormat>("model", response, json!({}), false).is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue