From 1a9b533e71c71db31c8dbd1dbba41b4c0bb0f5aa Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 07:17:24 -0700 Subject: [PATCH] refactor(rust): rename litellm-core to litellm-inference (#44802) * refactor(rust): rename litellm-core to litellm-inference Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(rust): expose inference base API for format crates Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(rust): move shared inference test helpers behind test-support Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Yujong Lee Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm-rust/Cargo.lock | 97 +++++++------- litellm-rust/Cargo.toml | 2 +- .../crates/gateway-inference/Cargo.toml | 2 +- .../src/audio_transcription.rs | 2 +- .../crates/gateway-inference/src/caching.rs | 8 +- .../gateway-inference/src/chat_completions.rs | 2 +- .../crates/gateway-inference/src/error.rs | 2 +- .../crates/gateway-inference/src/lib.rs | 4 +- .../crates/gateway-inference/src/messages.rs | 2 +- .../crates/gateway-inference/src/ocr.rs | 2 +- .../crates/gateway-inference/src/responses.rs | 2 +- .../crates/gateway-inference/tests/routes.rs | 8 +- .../gateway-inference/tests/support/mod.rs | 2 +- litellm-rust/crates/gateway/Cargo.toml | 2 +- litellm-rust/crates/gateway/src/lib.rs | 2 +- .../crates/{core => inference}/AGENTS.md | 8 +- .../crates/{core => inference}/Cargo.toml | 6 +- .../src/audio_transcription/handler.rs | 0 .../src/audio_transcription/mod.rs | 0 .../src/audio_transcription/prepare.rs | 3 +- .../src/audio_transcription/types.rs | 0 .../crates/{core => inference}/src/caching.rs | 8 +- .../src/chat_completions/common_utils.rs | 2 +- .../src/chat_completions/handler.rs | 0 .../src/chat_completions/mod.rs | 0 .../src/chat_completions/prepare.rs | 0 .../src/chat_completions/route.rs | 0 .../src/chat_completions/types.rs | 0 .../{core => inference}/src/constants.rs | 0 .../crates/{core => inference}/src/context.rs | 2 +- .../{core => inference}/src/diagnostic.rs | 8 +- .../crates/{core => inference}/src/error.rs | 0 .../crates/{core => inference}/src/lib.rs | 10 +- .../src/messages/AGENTS.md | 0 .../src/messages/common_utils.rs | 4 +- .../src/messages/handler.rs | 0 .../{core => inference}/src/messages/mod.rs | 0 .../src/messages/prepare.rs | 0 .../{core => inference}/src/messages/route.rs | 0 .../{core => inference}/src/messages/types.rs | 0 .../{core => inference}/src/ocr/arguments.rs | 0 .../{core => inference}/src/ocr/client.rs | 0 .../{core => inference}/src/ocr/document.rs | 0 .../{core => inference}/src/ocr/handler.rs | 0 .../crates/{core => inference}/src/ocr/mod.rs | 0 .../{core => inference}/src/ocr/prepare.rs | 2 +- .../src/ocr/provider_config.rs | 2 +- .../{core => inference}/src/ocr/route.rs | 0 .../{core => inference}/src/ocr/types.rs | 0 .../{core => inference}/src/ocr/wire.rs | 0 .../{core => inference}/src/outbound.rs | 4 +- .../{core => inference}/src/provider.rs | 13 +- .../{core => inference}/src/resources.rs | 0 .../src/responses/handler.rs | 0 .../{core => inference}/src/responses/mod.rs | 0 .../src/responses/prepare.rs | 0 .../src/responses/route.rs | 0 .../src/responses/types.rs | 0 .../src/responses/websocket.rs | 0 .../crates/inference/src/test_support.rs | 87 +++++++++++++ .../tests/audio_transcription.rs | 2 +- .../{core => inference}/tests/caching.rs | 20 +-- .../tests/chat_completions.rs | 4 +- .../tests/messages/host.rs | 3 +- .../tests/messages/main.rs | 5 +- .../tests/messages/request.rs | 0 .../tests/messages/response.rs | 11 +- .../tests/messages/secrets.rs | 1 + .../tests/messages/stream.rs | 5 +- .../tests/ocr/aws_textract.rs | 0 .../{core => inference}/tests/ocr/azure_ai.rs | 0 .../tests/ocr/azure_document_intelligence.rs | 0 .../{core => inference}/tests/ocr/cohere.rs | 0 .../tests/ocr/documents.rs | 3 +- .../tests/ocr/lifecycle.rs | 8 +- .../{core => inference}/tests/ocr/machine.rs | 8 +- .../{core => inference}/tests/ocr/main.rs | 11 +- .../{core => inference}/tests/ocr/mistral.rs | 1 + .../{core => inference}/tests/ocr/reducto.rs | 0 .../tests/ocr/vertex_ai.rs | 2 +- .../{core => inference}/tests/resources.rs | 7 +- .../{core => inference}/tests/responses.rs | 7 +- .../{core => inference}/tests/support/mod.rs | 118 +++--------------- litellm-rust/crates/llms/AGENTS.md | 2 +- litellm-rust/crates/python-bridge/AGENTS.md | 6 +- litellm-rust/crates/python-bridge/Cargo.toml | 2 +- .../python-bridge/src/cache/python/service.rs | 2 +- .../crates/python-bridge/src/errors.rs | 2 +- litellm-rust/crates/python-bridge/src/http.rs | 6 +- .../src/routes/audio_transcription.rs | 4 +- .../src/routes/chat_completions.rs | 4 +- .../src/routes/chat_completions/host.rs | 4 +- .../python-bridge/src/routes/inference.rs | 2 +- .../python-bridge/src/routes/messages/host.rs | 6 +- .../python-bridge/src/routes/messages/mod.rs | 4 +- .../python-bridge/src/routes/ocr/document.rs | 2 +- .../python-bridge/src/routes/ocr/host.rs | 2 +- .../python-bridge/src/routes/ocr/mod.rs | 4 +- .../python-bridge/src/routes/ocr/project.rs | 4 +- .../python-bridge/src/routes/responses.rs | 4 +- .../src/routes/responses/host.rs | 2 +- litellm-rust/crates/router/Cargo.toml | 2 +- litellm-rust/crates/router/src/deployment.rs | 2 +- litellm-rust/crates/router/tests/router.rs | 2 +- tests/unit/test_circleci_path_filter.py | 4 +- 105 files changed, 310 insertions(+), 276 deletions(-) rename litellm-rust/crates/{core => inference}/AGENTS.md (80%) rename litellm-rust/crates/{core => inference}/Cargo.toml (92%) rename litellm-rust/crates/{core => inference}/src/audio_transcription/handler.rs (100%) rename litellm-rust/crates/{core => inference}/src/audio_transcription/mod.rs (100%) rename litellm-rust/crates/{core => inference}/src/audio_transcription/prepare.rs (96%) rename litellm-rust/crates/{core => inference}/src/audio_transcription/types.rs (100%) rename litellm-rust/crates/{core => inference}/src/caching.rs (97%) rename litellm-rust/crates/{core => inference}/src/chat_completions/common_utils.rs (96%) rename litellm-rust/crates/{core => inference}/src/chat_completions/handler.rs (100%) rename litellm-rust/crates/{core => inference}/src/chat_completions/mod.rs (100%) rename litellm-rust/crates/{core => inference}/src/chat_completions/prepare.rs (100%) rename litellm-rust/crates/{core => inference}/src/chat_completions/route.rs (100%) rename litellm-rust/crates/{core => inference}/src/chat_completions/types.rs (100%) rename litellm-rust/crates/{core => inference}/src/constants.rs (100%) rename litellm-rust/crates/{core => inference}/src/context.rs (97%) rename litellm-rust/crates/{core => inference}/src/diagnostic.rs (97%) rename litellm-rust/crates/{core => inference}/src/error.rs (100%) rename litellm-rust/crates/{core => inference}/src/lib.rs (87%) rename litellm-rust/crates/{core => inference}/src/messages/AGENTS.md (100%) rename litellm-rust/crates/{core => inference}/src/messages/common_utils.rs (96%) rename litellm-rust/crates/{core => inference}/src/messages/handler.rs (100%) rename litellm-rust/crates/{core => inference}/src/messages/mod.rs (100%) rename litellm-rust/crates/{core => inference}/src/messages/prepare.rs (100%) rename litellm-rust/crates/{core => inference}/src/messages/route.rs (100%) rename litellm-rust/crates/{core => inference}/src/messages/types.rs (100%) rename litellm-rust/crates/{core => inference}/src/ocr/arguments.rs (100%) rename litellm-rust/crates/{core => inference}/src/ocr/client.rs (100%) rename litellm-rust/crates/{core => inference}/src/ocr/document.rs (100%) rename litellm-rust/crates/{core => inference}/src/ocr/handler.rs (100%) rename litellm-rust/crates/{core => inference}/src/ocr/mod.rs (100%) rename litellm-rust/crates/{core => inference}/src/ocr/prepare.rs (99%) rename litellm-rust/crates/{core => inference}/src/ocr/provider_config.rs (99%) rename litellm-rust/crates/{core => inference}/src/ocr/route.rs (100%) rename litellm-rust/crates/{core => inference}/src/ocr/types.rs (100%) rename litellm-rust/crates/{core => inference}/src/ocr/wire.rs (100%) rename litellm-rust/crates/{core => inference}/src/outbound.rs (94%) rename litellm-rust/crates/{core => inference}/src/provider.rs (73%) rename litellm-rust/crates/{core => inference}/src/resources.rs (100%) rename litellm-rust/crates/{core => inference}/src/responses/handler.rs (100%) rename litellm-rust/crates/{core => inference}/src/responses/mod.rs (100%) rename litellm-rust/crates/{core => inference}/src/responses/prepare.rs (100%) rename litellm-rust/crates/{core => inference}/src/responses/route.rs (100%) rename litellm-rust/crates/{core => inference}/src/responses/types.rs (100%) rename litellm-rust/crates/{core => inference}/src/responses/websocket.rs (100%) create mode 100644 litellm-rust/crates/inference/src/test_support.rs rename litellm-rust/crates/{core => inference}/tests/audio_transcription.rs (98%) rename litellm-rust/crates/{core => inference}/tests/caching.rs (99%) rename litellm-rust/crates/{core => inference}/tests/chat_completions.rs (98%) rename litellm-rust/crates/{core => inference}/tests/messages/host.rs (98%) rename litellm-rust/crates/{core => inference}/tests/messages/main.rs (98%) rename litellm-rust/crates/{core => inference}/tests/messages/request.rs (100%) rename litellm-rust/crates/{core => inference}/tests/messages/response.rs (97%) rename litellm-rust/crates/{core => inference}/tests/messages/secrets.rs (99%) rename litellm-rust/crates/{core => inference}/tests/messages/stream.rs (98%) rename litellm-rust/crates/{core => inference}/tests/ocr/aws_textract.rs (100%) rename litellm-rust/crates/{core => inference}/tests/ocr/azure_ai.rs (100%) rename litellm-rust/crates/{core => inference}/tests/ocr/azure_document_intelligence.rs (100%) rename litellm-rust/crates/{core => inference}/tests/ocr/cohere.rs (100%) rename litellm-rust/crates/{core => inference}/tests/ocr/documents.rs (98%) rename litellm-rust/crates/{core => inference}/tests/ocr/lifecycle.rs (99%) rename litellm-rust/crates/{core => inference}/tests/ocr/machine.rs (99%) rename litellm-rust/crates/{core => inference}/tests/ocr/main.rs (98%) rename litellm-rust/crates/{core => inference}/tests/ocr/mistral.rs (98%) rename litellm-rust/crates/{core => inference}/tests/ocr/reducto.rs (100%) rename litellm-rust/crates/{core => inference}/tests/ocr/vertex_ai.rs (98%) rename litellm-rust/crates/{core => inference}/tests/resources.rs (96%) rename litellm-rust/crates/{core => inference}/tests/responses.rs (98%) rename litellm-rust/crates/{core => inference}/tests/support/mod.rs (74%) diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 83d3a289c42..d912f5ceeee 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3697,50 +3697,6 @@ dependencies = [ "thiserror 2.0.19", ] -[[package]] -name = "litellm-core" -version = "0.1.0" -dependencies = [ - "base64 0.22.1", - "bytes", - "futures-util", - "litellm-auth", - "litellm-auth-aws", - "litellm-auth-gcp", - "litellm-cache", - "litellm-cache-memory", - "litellm-cache-response", - "litellm-core-utils", - "litellm-framer", - "litellm-host", - "litellm-host-native", - "litellm-http", - "litellm-llms", - "litellm-llms-types", - "litellm-secrets", - "litellm-tracing", - "mime_guess", - "moka", - "rand 0.8.7", - "reqwest 0.12.28", - "rstest", - "rstest_reuse", - "serde", - "serde_json", - "sha2 0.10.9", - "strum", - "subtle", - "thiserror 2.0.19", - "time", - "tokio", - "tokio-tungstenite", - "tokio-util", - "tracing", - "url", - "veil", - "wiremock", -] - [[package]] name = "litellm-core-utils" version = "0.1.0" @@ -3821,12 +3777,12 @@ dependencies = [ "http-body-util", "litellm-auth-types", "litellm-config", - "litellm-core", "litellm-gateway-auth", "litellm-gateway-inference", "litellm-gateway-mcp", "litellm-gateway-ui", "litellm-http", + "litellm-inference", "litellm-llms", "litellm-secrets", "litellm-tracing", @@ -3875,11 +3831,11 @@ dependencies = [ "litellm-auth", "litellm-cache-memory", "litellm-cache-response", - "litellm-core", "litellm-gateway-auth", "litellm-host", "litellm-host-http", "litellm-http", + "litellm-inference", "litellm-llms", "litellm-llms-types", "litellm-router", @@ -4038,6 +3994,51 @@ dependencies = [ "webpki-roots", ] +[[package]] +name = "litellm-inference" +version = "0.1.0" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-util", + "litellm-auth", + "litellm-auth-aws", + "litellm-auth-gcp", + "litellm-cache", + "litellm-cache-memory", + "litellm-cache-response", + "litellm-core-utils", + "litellm-framer", + "litellm-host", + "litellm-host-native", + "litellm-http", + "litellm-inference", + "litellm-llms", + "litellm-llms-types", + "litellm-secrets", + "litellm-tracing", + "mime_guess", + "moka", + "rand 0.8.7", + "reqwest 0.12.28", + "rstest", + "rstest_reuse", + "serde", + "serde_json", + "sha2 0.10.9", + "strum", + "subtle", + "thiserror 2.0.19", + "time", + "tokio", + "tokio-tungstenite", + "tokio-util", + "tracing", + "url", + "veil", + "wiremock", +] + [[package]] name = "litellm-llms" version = "0.1.0" @@ -4116,11 +4117,11 @@ dependencies = [ "litellm-cache-redis", "litellm-cache-response", "litellm-callbacks-legacy-python", - "litellm-core", "litellm-core-utils", "litellm-host", "litellm-host-python", "litellm-http", + "litellm-inference", "litellm-llms", "litellm-llms-types", "litellm-secrets", @@ -4171,7 +4172,7 @@ name = "litellm-router" version = "0.1.0" dependencies = [ "litellm-config", - "litellm-core", + "litellm-inference", "rstest", ] diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 18724ca5c5f..30d91b99b66 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -16,7 +16,7 @@ litellm-traces = { path = "crates/traces" } litellm-traces-cache = { path = "crates/traces-cache" } litellm-traces-clickhouse = { path = "crates/traces-clickhouse" } litellm-storage-clickhouse = { path = "crates/storage-clickhouse" } -litellm-core = { path = "crates/core" } +litellm-inference = { path = "crates/inference" } litellm-gateway-mcp = { path = "crates/gateway-mcp" } litellm-gateway = { path = "crates/gateway" } litellm-gateway-inference = { path = "crates/gateway-inference" } diff --git a/litellm-rust/crates/gateway-inference/Cargo.toml b/litellm-rust/crates/gateway-inference/Cargo.toml index 4544152d059..b3c255c8830 100644 --- a/litellm-rust/crates/gateway-inference/Cargo.toml +++ b/litellm-rust/crates/gateway-inference/Cargo.toml @@ -12,7 +12,7 @@ base64.workspace = true bytes.workspace = true litellm-auth.workspace = true litellm-gateway-auth.workspace = true -litellm-core.workspace = true +litellm-inference.workspace = true litellm-host-http.workspace = true litellm-host.workspace = true litellm-http.workspace = true diff --git a/litellm-rust/crates/gateway-inference/src/audio_transcription.rs b/litellm-rust/crates/gateway-inference/src/audio_transcription.rs index dc762fb8643..e7bfbb044af 100644 --- a/litellm-rust/crates/gateway-inference/src/audio_transcription.rs +++ b/litellm-rust/crates/gateway-inference/src/audio_transcription.rs @@ -3,7 +3,7 @@ use std::{path::Path, sync::Arc}; use axum::{Json, extract::State, response::IntoResponse}; use base64::{Engine, engine::general_purpose::STANDARD}; -use litellm_core::audio_transcription::types::AudioTranscriptionRequest; +use litellm_inference::audio_transcription::types::AudioTranscriptionRequest; use serde_json::{Value, json}; use crate::{ diff --git a/litellm-rust/crates/gateway-inference/src/caching.rs b/litellm-rust/crates/gateway-inference/src/caching.rs index 5472baf115f..1abea61b164 100644 --- a/litellm-rust/crates/gateway-inference/src/caching.rs +++ b/litellm-rust/crates/gateway-inference/src/caching.rs @@ -72,26 +72,26 @@ fn duration(seconds: f64) -> Result { #[derive(Clone, Default)] pub(crate) struct CacheHeaders(std::sync::Arc>); -impl litellm_host::interceptors::Interceptors for CacheHeaders { +impl litellm_host::interceptors::Interceptors for CacheHeaders { async fn before_provider_request( &self, wire: litellm_host::interceptors::WireRequest, _: litellm_host::interceptors::RequestContext, - ) -> Result { + ) -> Result { Ok(wire) } async fn after_provider_response( &self, _: litellm_host::interceptors::RawResponse, - ) -> Result<(), litellm_core::RouteError> { + ) -> Result<(), litellm_inference::RouteError> { Ok(()) } async fn result_ready( &self, facts: litellm_host::interceptors::ExecutionFacts, - ) -> Result<(), litellm_core::RouteError> { + ) -> Result<(), litellm_inference::RouteError> { if let litellm_host::interceptors::ResultSource::Cache { key } = facts.source { let _ = self.0.set(key); } diff --git a/litellm-rust/crates/gateway-inference/src/chat_completions.rs b/litellm-rust/crates/gateway-inference/src/chat_completions.rs index c9fabca2522..2d26437a3e3 100644 --- a/litellm-rust/crates/gateway-inference/src/chat_completions.rs +++ b/litellm-rust/crates/gateway-inference/src/chat_completions.rs @@ -6,7 +6,7 @@ use axum::{ extract::{Path, State}, response::{IntoResponse, Response}, }; -use litellm_core::chat_completions::types::ChatCompletionsCall; +use litellm_inference::chat_completions::types::ChatCompletionsCall; use serde_json::{Map, Value}; use crate::{Error, Gateway, JsonObject, request}; diff --git a/litellm-rust/crates/gateway-inference/src/error.rs b/litellm-rust/crates/gateway-inference/src/error.rs index 6b99069dc9d..39656b933eb 100644 --- a/litellm-rust/crates/gateway-inference/src/error.rs +++ b/litellm-rust/crates/gateway-inference/src/error.rs @@ -3,8 +3,8 @@ use axum::{ Json, response::{IntoResponse, Response}, }; -use litellm_core::RouteError; use litellm_http::transport::Error as TransportError; +use litellm_inference::RouteError; use litellm_llms::base_llm::ocr::error::Error as OcrError; use serde_json::{Map, Value, json}; diff --git a/litellm-rust/crates/gateway-inference/src/lib.rs b/litellm-rust/crates/gateway-inference/src/lib.rs index a8a70ffcefc..9bb8938389b 100644 --- a/litellm-rust/crates/gateway-inference/src/lib.rs +++ b/litellm-rust/crates/gateway-inference/src/lib.rs @@ -15,11 +15,11 @@ mod responses; use std::sync::Arc; use axum::{Router, routing::post}; -use litellm_core::{ +use litellm_http::{ClientVariant, HttpClientConfig, media::UrlPolicy}; +use litellm_inference::{ audio_transcription::AudioTranscriptionRoute, chat_completions::ChatCompletionsRoute, messages::MessagesRoute, ocr::OcrRoute, resources::CoreResources, responses::ResponsesRoute, }; -use litellm_http::{ClientVariant, HttpClientConfig, media::UrlPolicy}; use litellm_llms::base_llm::ocr::{handler::OcrClient, settings::OcrSettings}; use litellm_secrets::source::SecretSource; diff --git a/litellm-rust/crates/gateway-inference/src/messages.rs b/litellm-rust/crates/gateway-inference/src/messages.rs index 1a08946044b..3957aed3a8a 100644 --- a/litellm-rust/crates/gateway-inference/src/messages.rs +++ b/litellm-rust/crates/gateway-inference/src/messages.rs @@ -10,8 +10,8 @@ use axum::{ http::HeaderMap, response::{IntoResponse, Response}, }; -use litellm_core::messages::{MessagesCall, messages_body, route::Messages}; use litellm_host_http::Sse; +use litellm_inference::messages::{MessagesCall, messages_body, route::Messages}; use litellm_llms_types::headers::{ProviderSpecificHeader, ProviderSpecificHeaders}; use serde_json::{Map, Value}; diff --git a/litellm-rust/crates/gateway-inference/src/ocr.rs b/litellm-rust/crates/gateway-inference/src/ocr.rs index 2d8f62887e5..e3bfc43089f 100644 --- a/litellm-rust/crates/gateway-inference/src/ocr.rs +++ b/litellm-rust/crates/gateway-inference/src/ocr.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use axum::{Json, extract::State, http::HeaderMap, response::IntoResponse}; use litellm_auth::SecretValue; -use litellm_core::ocr::types::{LiteLLMOcrRequest, OcrConnectionInputs, OcrDocumentInput}; +use litellm_inference::ocr::types::{LiteLLMOcrRequest, OcrConnectionInputs, OcrDocumentInput}; use litellm_llms::base_llm::ocr::transformation::decode_request_value; use litellm_llms_types::formats::ocr::OcrDocument; use serde_json::Value; diff --git a/litellm-rust/crates/gateway-inference/src/responses.rs b/litellm-rust/crates/gateway-inference/src/responses.rs index 7a690e4e4c0..67e409790fd 100644 --- a/litellm-rust/crates/gateway-inference/src/responses.rs +++ b/litellm-rust/crates/gateway-inference/src/responses.rs @@ -1,9 +1,9 @@ use std::sync::Arc; use axum::{Json, body::Bytes, extract::State, response::Response}; -use litellm_core::responses::{route::Responses, types::ResponsesCall}; use litellm_gateway_auth::AuthenticatedRequest; use litellm_host_http::Sse; +use litellm_inference::responses::{route::Responses, types::ResponsesCall}; use serde_json::json; use crate::{Error, Gateway, JsonObject, request}; diff --git a/litellm-rust/crates/gateway-inference/tests/routes.rs b/litellm-rust/crates/gateway-inference/tests/routes.rs index d5233cadad8..9ee2204b0c7 100644 --- a/litellm-rust/crates/gateway-inference/tests/routes.rs +++ b/litellm-rust/crates/gateway-inference/tests/routes.rs @@ -6,13 +6,13 @@ use axum::{ body::{Body, Bytes}, http::Request, }; -use litellm_core::{ - chat_completions::{ChatCompletionsRoute, types::ChatCompletionsRequest}, - resources::CoreResources, -}; use litellm_http::{ ClientVariant, HttpClientPool, HttpSettings, Resolution, media::PublicDnsResolver, }; +use litellm_inference::{ + chat_completions::{ChatCompletionsRoute, types::ChatCompletionsRequest}, + resources::CoreResources, +}; use rstest::rstest; use serde_json::{Value, json}; use tower::ServiceExt; diff --git a/litellm-rust/crates/gateway-inference/tests/support/mod.rs b/litellm-rust/crates/gateway-inference/tests/support/mod.rs index 30c009e86f8..0e37bb0aff9 100644 --- a/litellm-rust/crates/gateway-inference/tests/support/mod.rs +++ b/litellm-rust/crates/gateway-inference/tests/support/mod.rs @@ -10,9 +10,9 @@ use axum::{ response::Response, }; use futures_util::future::BoxFuture; -use litellm_core::resources::CoreResources; use litellm_gateway_inference::{Deployment, Gateway, router}; use litellm_http::{HttpClientPool, HttpSettings, Resolution, media::PublicDnsResolver}; +use litellm_inference::resources::CoreResources; use litellm_secrets::{SecretValue, source::SecretSource}; use serde_json::Value; use tower::ServiceExt; diff --git a/litellm-rust/crates/gateway/Cargo.toml b/litellm-rust/crates/gateway/Cargo.toml index 56e14d3a80f..387bcd43231 100644 --- a/litellm-rust/crates/gateway/Cargo.toml +++ b/litellm-rust/crates/gateway/Cargo.toml @@ -9,7 +9,7 @@ repository.workspace = true axum.workspace = true envy = "0.4.2" http-body-util = "0.1" -litellm-core.workspace = true +litellm-inference.workspace = true litellm-gateway-inference.workspace = true litellm-gateway-mcp.workspace = true tokio-util = "0.7" diff --git a/litellm-rust/crates/gateway/src/lib.rs b/litellm-rust/crates/gateway/src/lib.rs index 91335272ae4..d711eb78caa 100644 --- a/litellm-rust/crates/gateway/src/lib.rs +++ b/litellm-rust/crates/gateway/src/lib.rs @@ -15,13 +15,13 @@ use axum::{ use http_body_util::BodyExt; use litellm_config::Config; -use litellm_core::resources::CoreResources; use litellm_gateway_auth::Auth; use litellm_gateway_inference::{Gateway, ModelRouter}; use litellm_http::{ ClientVariant, HttpClientPool, HttpSettings, HttpSettingsLayer, Resolution, SslVerify, media::PublicDnsResolver, }; +use litellm_inference::resources::CoreResources; use litellm_secrets::source::EnvironmentSecrets; use litellm_tracing::ByteChunk; use uuid::Uuid; diff --git a/litellm-rust/crates/core/AGENTS.md b/litellm-rust/crates/inference/AGENTS.md similarity index 80% rename from litellm-rust/crates/core/AGENTS.md rename to litellm-rust/crates/inference/AGENTS.md index c6beef859f7..6baae6d5a48 100644 --- a/litellm-rust/crates/core/AGENTS.md +++ b/litellm-rust/crates/inference/AGENTS.md @@ -1,4 +1,4 @@ -litellm-core owns route orchestration. Messages and HTTP Responses return `litellm_host::call::CallOutput`, containing either a completed response or a stream head and chunks. OCR and currently non-streaming Chat Completions return their completed response directly +litellm-inference owns route orchestration. Messages and HTTP Responses return `litellm_host::call::CallOutput`, containing either a completed response or a stream head and chunks. OCR and currently non-streaming Chat Completions return their completed response directly Hosts assemble route objects from shared `CoreResources`, HTTP settings, and secret sources. Each route owns its provider client and authentication dependencies. Gateway routes live for the gateway lifetime; Python assembles routes per call from its settings snapshot @@ -10,7 +10,7 @@ Responses WebSocket sessions remain separate from the HTTP call driver because a ## Crate layering -For Messages, Responses, Chat Completions, OCR, and other API formats, `core/src//` owns orchestration. Shared API data contracts belong in `litellm-llms-types`, adapter contracts and shared transformation machinery in `llms/src/base_llm//`, and provider policy in `llms/src///`. A repeated format directory name does not imply interchangeable responsibilities. Select concrete adapters here, then invoke their contracts instead of applying one provider's policy to every call. Route types describe call envelopes and execution state, not duplicate public payload schemas +For Messages, Responses, Chat Completions, OCR, and other API formats, `inference/src//` owns orchestration. Shared API data contracts belong in `litellm-llms-types`, adapter contracts and shared transformation machinery in `llms/src/base_llm//`, and provider policy in `llms/src///`. A repeated format directory name does not imply interchangeable responsibilities. Select concrete adapters here, then invoke their contracts instead of applying one provider's policy to every call. Route types describe call envelopes and execution state, not duplicate public payload schemas Crates separate API data, transformations, transport, and orchestration. Python package names identify counterparts, not ownership. Dependencies only point down: @@ -18,9 +18,9 @@ Crates separate API data, transformations, transport, and orchestration. Python - `litellm-core-utils` mirrors `litellm/litellm_core_utils/`: pure helpers (provider resolution, prompt factory, call arguments, settings lookup and layer merge), no network I/O - `litellm-http` is Rust-only and route-neutral: settings resolution, the pooled `reqwest` clients, TLS, proxies, the SSRF-safe media fetcher, request and header helpers, and transport errors. Python's `litellm/llms/custom_httpx/` is split by responsibility instead of mirrored: its transport half lives here, its OCR handler in `litellm-llms` - `litellm-llms` mirrors `litellm/llms/`: `base_llm//transformation.rs`, `//transformation.rs`, and `base_llm/ocr/handler.rs` (the OCR request handler) -- `litellm-core` mirrors the route packages (`litellm/ocr/`, `litellm/messages/`, ...): entrypoints, route request types, provider dispatch, the route machine, and hooks +- `litellm-inference` mirrors the route packages (`litellm/ocr/`, `litellm/messages/`, ...): entrypoints, route request types, provider dispatch, the route machine, and hooks -A route module owns the call entrypoint, route request types (`*Request<'a>`), credential fallback, provider dispatch, and the handler glue that runs a provider config. Provider code never imports from core; when it needs the caller's hooks mid-call it goes through `litellm_llms::base_llm::ocr::handler::CallHooks`, the provider-level hooks OCR implements over its host until it folds into `litellm_host::interceptors::Interceptors`. Import every item from its canonical path. Never re-export another crate's items or give an item a second public path; the only re-export allowed is a private submodule surfacing its item at its module root (`mod error; pub use error::Error;`). Handlers belong in core or llms, never in a host crate +A route module owns the call entrypoint, route request types (`*Request<'a>`), credential fallback, provider dispatch, and the handler glue that runs a provider config. Provider code never imports from inference; when it needs the caller's hooks mid-call it goes through `litellm_llms::base_llm::ocr::handler::CallHooks`, the provider-level hooks OCR implements over its host until it folds into `litellm_host::interceptors::Interceptors`. Import every item from its canonical path. Never re-export another crate's items or give an item a second public path; the only re-export allowed is a private submodule surfacing its item at its module root (`mod error; pub use error::Error;`). Handlers belong in core or llms, never in a host crate ## Error placement diff --git a/litellm-rust/crates/core/Cargo.toml b/litellm-rust/crates/inference/Cargo.toml similarity index 92% rename from litellm-rust/crates/core/Cargo.toml rename to litellm-rust/crates/inference/Cargo.toml index 707297d7161..e89ed8d94e7 100644 --- a/litellm-rust/crates/core/Cargo.toml +++ b/litellm-rust/crates/inference/Cargo.toml @@ -1,10 +1,13 @@ [package] -name = "litellm-core" +name = "litellm-inference" version = "0.1.0" edition.workspace = true license.workspace = true repository.workspace = true +[features] +test-support = [] + [dependencies] litellm-cache.workspace = true litellm-cache-response.workspace = true @@ -40,6 +43,7 @@ url.workspace = true veil.workspace = true [dev-dependencies] +litellm-inference = { workspace = true, features = ["test-support"] } litellm-cache-memory.workspace = true litellm-http = { workspace = true, features = ["test-support"] } litellm-auth-gcp.workspace = true diff --git a/litellm-rust/crates/core/src/audio_transcription/handler.rs b/litellm-rust/crates/inference/src/audio_transcription/handler.rs similarity index 100% rename from litellm-rust/crates/core/src/audio_transcription/handler.rs rename to litellm-rust/crates/inference/src/audio_transcription/handler.rs diff --git a/litellm-rust/crates/core/src/audio_transcription/mod.rs b/litellm-rust/crates/inference/src/audio_transcription/mod.rs similarity index 100% rename from litellm-rust/crates/core/src/audio_transcription/mod.rs rename to litellm-rust/crates/inference/src/audio_transcription/mod.rs diff --git a/litellm-rust/crates/core/src/audio_transcription/prepare.rs b/litellm-rust/crates/inference/src/audio_transcription/prepare.rs similarity index 96% rename from litellm-rust/crates/core/src/audio_transcription/prepare.rs rename to litellm-rust/crates/inference/src/audio_transcription/prepare.rs index 3c2be913b0b..9426384fa97 100644 --- a/litellm-rust/crates/core/src/audio_transcription/prepare.rs +++ b/litellm-rust/crates/inference/src/audio_transcription/prepare.rs @@ -1,3 +1,4 @@ +use litellm_core_utils::get_llm_provider_logic::LlmProviders; use litellm_http::request::string_headers; use litellm_http::request::with_default_headers; use litellm_llms::{ @@ -13,7 +14,7 @@ use super::Error; use crate::audio_transcription::types::{ AudioTranscriptionRequest, ProviderAudioTranscriptionRequest, }; -use crate::provider::{LlmProviders, resolve_llm_provider}; +use crate::provider::resolve_llm_provider; fn provider_config(provider: LlmProviders) -> Option<&'static dyn BaseAudioTranscriptionConfig> { match provider { diff --git a/litellm-rust/crates/core/src/audio_transcription/types.rs b/litellm-rust/crates/inference/src/audio_transcription/types.rs similarity index 100% rename from litellm-rust/crates/core/src/audio_transcription/types.rs rename to litellm-rust/crates/inference/src/audio_transcription/types.rs diff --git a/litellm-rust/crates/core/src/caching.rs b/litellm-rust/crates/inference/src/caching.rs similarity index 97% rename from litellm-rust/crates/core/src/caching.rs rename to litellm-rust/crates/inference/src/caching.rs index 99e21f81f2a..d2f4198b910 100644 --- a/litellm-rust/crates/core/src/caching.rs +++ b/litellm-rust/crates/inference/src/caching.rs @@ -221,13 +221,13 @@ where Ok(cache.finish(output, &source).await) } -pub(crate) struct CallCache

{ +pub struct CallCache

{ session: Option, protocol: PhantomData

, } impl CallCache

{ - pub(crate) fn from_wire( + pub fn from_wire( cache: Option<&ScopedCache>, policy: CachePolicy, identity: &ProviderIdentity, @@ -254,7 +254,7 @@ impl CallCache

{ } } - pub(crate) async fn lookup(&self) -> Option<(OutputOf

, ResultSource)> + pub async fn lookup(&self) -> Option<(OutputOf

, ResultSource)> where P::Response: DeserializeOwned, { @@ -271,7 +271,7 @@ impl CallCache

{ )) } - pub(crate) async fn finish(self, output: OutputOf

, source: &ResultSource) -> OutputOf

+ pub async fn finish(self, output: OutputOf

, source: &ResultSource) -> OutputOf

where P::Response: Serialize, { diff --git a/litellm-rust/crates/core/src/chat_completions/common_utils.rs b/litellm-rust/crates/inference/src/chat_completions/common_utils.rs similarity index 96% rename from litellm-rust/crates/core/src/chat_completions/common_utils.rs rename to litellm-rust/crates/inference/src/chat_completions/common_utils.rs index c8001b52891..14abd42f6b7 100644 --- a/litellm-rust/crates/core/src/chat_completions/common_utils.rs +++ b/litellm-rust/crates/inference/src/chat_completions/common_utils.rs @@ -1,3 +1,4 @@ +use litellm_core_utils::get_llm_provider_logic::LlmProviders; use litellm_http::request::string_headers as shared_string_headers; use litellm_llms::{ anthropic::chat::transformation::ANTHROPIC_CHAT_COMPLETIONS_CONFIG, @@ -8,7 +9,6 @@ use litellm_llms::{ use serde_json::{Map, Value}; use super::Error; -use crate::provider::LlmProviders; const HEADER_CONTEXT: &str = "chat completions"; diff --git a/litellm-rust/crates/core/src/chat_completions/handler.rs b/litellm-rust/crates/inference/src/chat_completions/handler.rs similarity index 100% rename from litellm-rust/crates/core/src/chat_completions/handler.rs rename to litellm-rust/crates/inference/src/chat_completions/handler.rs diff --git a/litellm-rust/crates/core/src/chat_completions/mod.rs b/litellm-rust/crates/inference/src/chat_completions/mod.rs similarity index 100% rename from litellm-rust/crates/core/src/chat_completions/mod.rs rename to litellm-rust/crates/inference/src/chat_completions/mod.rs diff --git a/litellm-rust/crates/core/src/chat_completions/prepare.rs b/litellm-rust/crates/inference/src/chat_completions/prepare.rs similarity index 100% rename from litellm-rust/crates/core/src/chat_completions/prepare.rs rename to litellm-rust/crates/inference/src/chat_completions/prepare.rs diff --git a/litellm-rust/crates/core/src/chat_completions/route.rs b/litellm-rust/crates/inference/src/chat_completions/route.rs similarity index 100% rename from litellm-rust/crates/core/src/chat_completions/route.rs rename to litellm-rust/crates/inference/src/chat_completions/route.rs diff --git a/litellm-rust/crates/core/src/chat_completions/types.rs b/litellm-rust/crates/inference/src/chat_completions/types.rs similarity index 100% rename from litellm-rust/crates/core/src/chat_completions/types.rs rename to litellm-rust/crates/inference/src/chat_completions/types.rs diff --git a/litellm-rust/crates/core/src/constants.rs b/litellm-rust/crates/inference/src/constants.rs similarity index 100% rename from litellm-rust/crates/core/src/constants.rs rename to litellm-rust/crates/inference/src/constants.rs diff --git a/litellm-rust/crates/core/src/context.rs b/litellm-rust/crates/inference/src/context.rs similarity index 97% rename from litellm-rust/crates/core/src/context.rs rename to litellm-rust/crates/inference/src/context.rs index caadf66cfb6..6fcbb3051b4 100644 --- a/litellm-rust/crates/core/src/context.rs +++ b/litellm-rust/crates/inference/src/context.rs @@ -7,7 +7,7 @@ use litellm_host::{ use crate::{CallOptions, RouteError}; -pub(crate) struct CallContext<'a, I> { +pub struct CallContext<'a, I> { pub interceptors: &'a I, pub observers: Option, pub cache: CachePolicy, diff --git a/litellm-rust/crates/core/src/diagnostic.rs b/litellm-rust/crates/inference/src/diagnostic.rs similarity index 97% rename from litellm-rust/crates/core/src/diagnostic.rs rename to litellm-rust/crates/inference/src/diagnostic.rs index c1ac4451c81..d7e6067cf53 100644 --- a/litellm-rust/crates/core/src/diagnostic.rs +++ b/litellm-rust/crates/inference/src/diagnostic.rs @@ -41,17 +41,17 @@ impl Drop for Completion { } } -pub(crate) fn provider(model: &str, provider: &str) { +pub fn provider(model: &str, provider: &str) { let span = Span::current(); span.record("resolved_model", model); span.record("provider", provider); } -pub(crate) async fn unary(execute: impl Future>) -> Result { +pub async fn unary(execute: impl Future>) -> Result { operation("litellm.route", execute).await } -pub(crate) async fn operation( +pub async fn operation( name: &str, execute: impl Future>, ) -> Result { @@ -61,7 +61,7 @@ pub(crate) async fn operation( result } -pub(crate) async fn call( +pub async fn call( execute: impl Future, E>>, ) -> Result, E> where diff --git a/litellm-rust/crates/core/src/error.rs b/litellm-rust/crates/inference/src/error.rs similarity index 100% rename from litellm-rust/crates/core/src/error.rs rename to litellm-rust/crates/inference/src/error.rs diff --git a/litellm-rust/crates/core/src/lib.rs b/litellm-rust/crates/inference/src/lib.rs similarity index 87% rename from litellm-rust/crates/core/src/lib.rs rename to litellm-rust/crates/inference/src/lib.rs index 1fd38df191f..e148d794c13 100644 --- a/litellm-rust/crates/core/src/lib.rs +++ b/litellm-rust/crates/inference/src/lib.rs @@ -1,5 +1,5 @@ -mod context; -mod diagnostic; +pub mod context; +pub mod diagnostic; pub mod audio_transcription; pub mod caching; @@ -8,10 +8,12 @@ pub mod constants; pub mod error; pub mod messages; pub mod ocr; -mod outbound; -mod provider; +pub mod outbound; +pub mod provider; pub mod resources; pub mod responses; +#[cfg(feature = "test-support")] +pub mod test_support; pub use error::RouteError; diff --git a/litellm-rust/crates/core/src/messages/AGENTS.md b/litellm-rust/crates/inference/src/messages/AGENTS.md similarity index 100% rename from litellm-rust/crates/core/src/messages/AGENTS.md rename to litellm-rust/crates/inference/src/messages/AGENTS.md diff --git a/litellm-rust/crates/core/src/messages/common_utils.rs b/litellm-rust/crates/inference/src/messages/common_utils.rs similarity index 96% rename from litellm-rust/crates/core/src/messages/common_utils.rs rename to litellm-rust/crates/inference/src/messages/common_utils.rs index 98a92c90dba..6e7d6b4a45f 100644 --- a/litellm-rust/crates/core/src/messages/common_utils.rs +++ b/litellm-rust/crates/inference/src/messages/common_utils.rs @@ -1,3 +1,4 @@ +use litellm_core_utils::get_llm_provider_logic::LlmProviders; use litellm_http::request::string_headers as shared_string_headers; pub(super) use litellm_http::request::truncate_error_body; use litellm_llms::{ @@ -9,7 +10,6 @@ use litellm_llms::{ use serde_json::{Map, Value}; use super::Error; -use crate::provider::LlmProviders; const HEADER_CONTEXT: &str = "messages"; @@ -68,7 +68,7 @@ mod tests { use super::{MessagesProvider, messages_provider, string_headers, truncate_error_body}; use crate::messages::Error; - use crate::provider::LlmProviders; + use litellm_core_utils::get_llm_provider_logic::LlmProviders; #[rstest] #[case::anthropic("anthropic", MessagesProvider::Anthropic)] diff --git a/litellm-rust/crates/core/src/messages/handler.rs b/litellm-rust/crates/inference/src/messages/handler.rs similarity index 100% rename from litellm-rust/crates/core/src/messages/handler.rs rename to litellm-rust/crates/inference/src/messages/handler.rs diff --git a/litellm-rust/crates/core/src/messages/mod.rs b/litellm-rust/crates/inference/src/messages/mod.rs similarity index 100% rename from litellm-rust/crates/core/src/messages/mod.rs rename to litellm-rust/crates/inference/src/messages/mod.rs diff --git a/litellm-rust/crates/core/src/messages/prepare.rs b/litellm-rust/crates/inference/src/messages/prepare.rs similarity index 100% rename from litellm-rust/crates/core/src/messages/prepare.rs rename to litellm-rust/crates/inference/src/messages/prepare.rs diff --git a/litellm-rust/crates/core/src/messages/route.rs b/litellm-rust/crates/inference/src/messages/route.rs similarity index 100% rename from litellm-rust/crates/core/src/messages/route.rs rename to litellm-rust/crates/inference/src/messages/route.rs diff --git a/litellm-rust/crates/core/src/messages/types.rs b/litellm-rust/crates/inference/src/messages/types.rs similarity index 100% rename from litellm-rust/crates/core/src/messages/types.rs rename to litellm-rust/crates/inference/src/messages/types.rs diff --git a/litellm-rust/crates/core/src/ocr/arguments.rs b/litellm-rust/crates/inference/src/ocr/arguments.rs similarity index 100% rename from litellm-rust/crates/core/src/ocr/arguments.rs rename to litellm-rust/crates/inference/src/ocr/arguments.rs diff --git a/litellm-rust/crates/core/src/ocr/client.rs b/litellm-rust/crates/inference/src/ocr/client.rs similarity index 100% rename from litellm-rust/crates/core/src/ocr/client.rs rename to litellm-rust/crates/inference/src/ocr/client.rs diff --git a/litellm-rust/crates/core/src/ocr/document.rs b/litellm-rust/crates/inference/src/ocr/document.rs similarity index 100% rename from litellm-rust/crates/core/src/ocr/document.rs rename to litellm-rust/crates/inference/src/ocr/document.rs diff --git a/litellm-rust/crates/core/src/ocr/handler.rs b/litellm-rust/crates/inference/src/ocr/handler.rs similarity index 100% rename from litellm-rust/crates/core/src/ocr/handler.rs rename to litellm-rust/crates/inference/src/ocr/handler.rs diff --git a/litellm-rust/crates/core/src/ocr/mod.rs b/litellm-rust/crates/inference/src/ocr/mod.rs similarity index 100% rename from litellm-rust/crates/core/src/ocr/mod.rs rename to litellm-rust/crates/inference/src/ocr/mod.rs diff --git a/litellm-rust/crates/core/src/ocr/prepare.rs b/litellm-rust/crates/inference/src/ocr/prepare.rs similarity index 99% rename from litellm-rust/crates/core/src/ocr/prepare.rs rename to litellm-rust/crates/inference/src/ocr/prepare.rs index 0f4e217c074..963ac81dd83 100644 --- a/litellm-rust/crates/core/src/ocr/prepare.rs +++ b/litellm-rust/crates/inference/src/ocr/prepare.rs @@ -1,4 +1,5 @@ use litellm_auth::{InputSource, SecretValue, Sourced}; +use litellm_core_utils::get_llm_provider_logic::LlmProviders; use litellm_llms::base_llm::ocr::{ handler::OcrClient, transformation::{OcrConnection, OcrCredentialInputs, PreparedOcrRequest}, @@ -6,7 +7,6 @@ use litellm_llms::base_llm::ocr::{ use litellm_secrets::source::Secrets; use crate::ocr::types::{LiteLLMOcrRequest, ResolvedOcrRequest}; -use crate::provider::LlmProviders; pub(crate) fn prepare_request( request: ResolvedOcrRequest, diff --git a/litellm-rust/crates/core/src/ocr/provider_config.rs b/litellm-rust/crates/inference/src/ocr/provider_config.rs similarity index 99% rename from litellm-rust/crates/core/src/ocr/provider_config.rs rename to litellm-rust/crates/inference/src/ocr/provider_config.rs index 6e934b53e82..28cb96238a3 100644 --- a/litellm-rust/crates/core/src/ocr/provider_config.rs +++ b/litellm-rust/crates/inference/src/ocr/provider_config.rs @@ -1,4 +1,4 @@ -use crate::provider::LlmProviders; +use litellm_core_utils::get_llm_provider_logic::LlmProviders; use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider}; use litellm_llms::{ aws_textract::ocr::{ diff --git a/litellm-rust/crates/core/src/ocr/route.rs b/litellm-rust/crates/inference/src/ocr/route.rs similarity index 100% rename from litellm-rust/crates/core/src/ocr/route.rs rename to litellm-rust/crates/inference/src/ocr/route.rs diff --git a/litellm-rust/crates/core/src/ocr/types.rs b/litellm-rust/crates/inference/src/ocr/types.rs similarity index 100% rename from litellm-rust/crates/core/src/ocr/types.rs rename to litellm-rust/crates/inference/src/ocr/types.rs diff --git a/litellm-rust/crates/core/src/ocr/wire.rs b/litellm-rust/crates/inference/src/ocr/wire.rs similarity index 100% rename from litellm-rust/crates/core/src/ocr/wire.rs rename to litellm-rust/crates/inference/src/ocr/wire.rs diff --git a/litellm-rust/crates/core/src/outbound.rs b/litellm-rust/crates/inference/src/outbound.rs similarity index 94% rename from litellm-rust/crates/core/src/outbound.rs rename to litellm-rust/crates/inference/src/outbound.rs index 4d038e0eb00..41fe3aa62b8 100644 --- a/litellm-rust/crates/core/src/outbound.rs +++ b/litellm-rust/crates/inference/src/outbound.rs @@ -10,7 +10,7 @@ use serde_json::Value; skip_all, fields(status) )] -pub(crate) async fn send( +pub async fn send( request: OutboundRequest, client: &litellm_http::Client, ) -> Result { @@ -21,7 +21,7 @@ pub(crate) async fn send( /// Header credentials are already in `headers`; SigV4 is applied here, over the /// bytes that are sent. -pub(crate) fn outbound_request( +pub fn outbound_request( authenticated: Authenticated, url: String, body: &Value, diff --git a/litellm-rust/crates/core/src/provider.rs b/litellm-rust/crates/inference/src/provider.rs similarity index 73% rename from litellm-rust/crates/core/src/provider.rs rename to litellm-rust/crates/inference/src/provider.rs index e071aee8084..7e1269db1cf 100644 --- a/litellm-rust/crates/core/src/provider.rs +++ b/litellm-rust/crates/inference/src/provider.rs @@ -1,15 +1,16 @@ -pub use litellm_core_utils::get_llm_provider_logic::LlmProviders; -use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider}; +use litellm_core_utils::get_llm_provider_logic::{ + CustomLlmProvider, LlmProviders, get_custom_llm_provider, +}; use crate::error::RouteError as Error; #[derive(Debug)] -pub(crate) struct ResolvedProvider<'a> { - pub(crate) model: &'a str, - pub(crate) provider: LlmProviders, +pub struct ResolvedProvider<'a> { + pub model: &'a str, + pub provider: LlmProviders, } -pub(crate) fn resolve_llm_provider<'a>( +pub fn resolve_llm_provider<'a>( model: &'a str, custom_llm_provider: Option<&'a str>, route: &'static str, diff --git a/litellm-rust/crates/core/src/resources.rs b/litellm-rust/crates/inference/src/resources.rs similarity index 100% rename from litellm-rust/crates/core/src/resources.rs rename to litellm-rust/crates/inference/src/resources.rs diff --git a/litellm-rust/crates/core/src/responses/handler.rs b/litellm-rust/crates/inference/src/responses/handler.rs similarity index 100% rename from litellm-rust/crates/core/src/responses/handler.rs rename to litellm-rust/crates/inference/src/responses/handler.rs diff --git a/litellm-rust/crates/core/src/responses/mod.rs b/litellm-rust/crates/inference/src/responses/mod.rs similarity index 100% rename from litellm-rust/crates/core/src/responses/mod.rs rename to litellm-rust/crates/inference/src/responses/mod.rs diff --git a/litellm-rust/crates/core/src/responses/prepare.rs b/litellm-rust/crates/inference/src/responses/prepare.rs similarity index 100% rename from litellm-rust/crates/core/src/responses/prepare.rs rename to litellm-rust/crates/inference/src/responses/prepare.rs diff --git a/litellm-rust/crates/core/src/responses/route.rs b/litellm-rust/crates/inference/src/responses/route.rs similarity index 100% rename from litellm-rust/crates/core/src/responses/route.rs rename to litellm-rust/crates/inference/src/responses/route.rs diff --git a/litellm-rust/crates/core/src/responses/types.rs b/litellm-rust/crates/inference/src/responses/types.rs similarity index 100% rename from litellm-rust/crates/core/src/responses/types.rs rename to litellm-rust/crates/inference/src/responses/types.rs diff --git a/litellm-rust/crates/core/src/responses/websocket.rs b/litellm-rust/crates/inference/src/responses/websocket.rs similarity index 100% rename from litellm-rust/crates/core/src/responses/websocket.rs rename to litellm-rust/crates/inference/src/responses/websocket.rs diff --git a/litellm-rust/crates/inference/src/test_support.rs b/litellm-rust/crates/inference/src/test_support.rs new file mode 100644 index 00000000000..65f5eb1fe0e --- /dev/null +++ b/litellm-rust/crates/inference/src/test_support.rs @@ -0,0 +1,87 @@ +use std::sync::{Arc, Mutex}; + +use futures_util::future::BoxFuture; +use litellm_http::{ + ClientVariant, HttpClientConfig, HttpClientPool, HttpSettings, Resolution, + media::PublicDnsResolver, +}; +use litellm_secrets::{SecretValue, source::SecretSource}; + +use crate::resources::CoreResources; + +pub fn http_pool() -> HttpClientPool { + HttpClientPool::new(Arc::new(PublicDnsResolver)) +} + +pub fn resources() -> CoreResources { + CoreResources::new(Arc::new(http_pool())) +} + +pub fn no_secrets() -> Arc { + Arc::new(RecordingSecrets::empty()) +} + +pub fn provider_http(resources: &CoreResources, config: &HttpClientConfig) -> litellm_http::Client { + resources + .pool + .client(config, ClientVariant::Provider) + .unwrap() +} + +pub fn http_config() -> HttpClientConfig { + Resolution::from(&HttpSettings::default()).config +} + +/// A secret source that answers from a fixed table and records every name it was asked for. +pub struct RecordingSecrets { + values: Vec<(String, String)>, + fails: bool, + requested: Mutex>, +} + +impl RecordingSecrets { + pub fn new<'a>(values: impl IntoIterator) -> Self { + Self { + values: values + .into_iter() + .map(|(name, value)| (name.to_string(), value.to_string())) + .collect(), + fails: false, + requested: Mutex::new(Vec::new()), + } + } + + pub fn empty() -> Self { + Self::new([]) + } + + pub fn failing() -> Self { + Self { + fails: true, + ..Self::empty() + } + } + + pub fn requested(&self) -> Vec { + self.requested.lock().unwrap().clone() + } +} + +impl SecretSource for RecordingSecrets { + fn get_secret_str<'a>( + &'a self, + name: &'a str, + ) -> BoxFuture<'a, Result, litellm_secrets::Error>> { + Box::pin(async move { + self.requested.lock().unwrap().push(name.to_string()); + if self.fails { + return Err(litellm_secrets::Error::ManagedSecretMissing); + } + Ok(self + .values + .iter() + .find(|(key, _)| key == name) + .map(|(_, value)| SecretValue::new(value.clone()))) + }) + } +} diff --git a/litellm-rust/crates/core/tests/audio_transcription.rs b/litellm-rust/crates/inference/tests/audio_transcription.rs similarity index 98% rename from litellm-rust/crates/core/tests/audio_transcription.rs rename to litellm-rust/crates/inference/tests/audio_transcription.rs index 02dbbc50bac..cd9b3f71d8c 100644 --- a/litellm-rust/crates/core/tests/audio_transcription.rs +++ b/litellm-rust/crates/inference/tests/audio_transcription.rs @@ -1,4 +1,4 @@ -use litellm_core::audio_transcription::{Error, types::AudioTranscriptionRequest}; +use litellm_inference::audio_transcription::{Error, types::AudioTranscriptionRequest}; use rstest::{fixture, rstest}; use serde_json::{Map, Value, json}; use wiremock::ResponseTemplate; diff --git a/litellm-rust/crates/core/tests/caching.rs b/litellm-rust/crates/inference/tests/caching.rs similarity index 99% rename from litellm-rust/crates/core/tests/caching.rs rename to litellm-rust/crates/inference/tests/caching.rs index 4f4f6e20ad6..ba356a62533 100644 --- a/litellm-rust/crates/core/tests/caching.rs +++ b/litellm-rust/crates/inference/tests/caching.rs @@ -15,10 +15,6 @@ use litellm_cache_response::{ CacheOptions, CachePolicy, CacheScope, ResponseCache, ResponseCacheConfig, ResponseCacheService, ResponseEnvelope, }; -use litellm_core::{ - RouteError, - caching::{Cachable, CacheRequest, StreamCachable, execute_streaming, execute_unary}, -}; use litellm_host::{ call::{CallOutput, OutputOf}, interceptors::{ @@ -29,6 +25,10 @@ use litellm_host::{ observation::observation_channel, protocol::Protocol, }; +use litellm_inference::{ + RouteError, + caching::{Cachable, CacheRequest, StreamCachable, execute_streaming, execute_unary}, +}; use rstest::{fixture, rstest}; use serde_json::{Value, json}; @@ -410,7 +410,7 @@ async fn an_invalid_cached_envelope_is_replaced_by_a_provider_result(#[case] poi async fn responses_refetches_instead_of_deserializing_another_api_response( #[case] poisoned: Value, ) { - use litellm_core::responses::route::Responses; + use litellm_inference::responses::route::Responses; use litellm_llms_types::formats::responses::ResponsesApiResponse; let cache: Arc = Arc::new(InvalidEntryCache( @@ -460,7 +460,7 @@ async fn messages_cache_identity_includes_provider_native_parameters( #[case] original: Value, #[case] changed: Value, ) { - use litellm_core::messages::route::Messages; + use litellm_inference::messages::route::Messages; use litellm_llms_types::formats::messages::MessagesResponse; let calls = AtomicUsize::new(0); @@ -842,7 +842,7 @@ async fn responses_cache_only_reuses_completed_responses( #[case] status: &str, #[case] expected_calls: usize, ) { - use litellm_core::responses::route::Responses; + use litellm_inference::responses::route::Responses; use litellm_llms_types::formats::responses::ResponsesApiResponse; let calls = AtomicUsize::new(0); @@ -883,7 +883,7 @@ async fn the_same_route_entrypoint_reports_facts_with_or_without_caching( traces: support::TraceCapture, ) { use litellm_cache_response::ScopedCache; - use litellm_core::chat_completions::types::ChatCompletionsRequest; + use litellm_inference::chat_completions::types::ChatCompletionsRequest; use wiremock::{Mock, MockServer, ResponseTemplate, matchers::method}; let upstream = MockServer::start().await; @@ -1038,7 +1038,7 @@ async fn cache_identity_follows_resolved_configuration_and_request_callbacks( #[case] change: &str, ) { use litellm_cache_response::ScopedCache; - use litellm_core::{ + use litellm_inference::{ chat_completions::{ChatCompletionsRoute, types::ChatCompletionsRequest}, messages::MessagesCall, responses::types::ResponsesCall, @@ -1178,7 +1178,7 @@ async fn cache_identity_follows_resolved_configuration_and_request_callbacks( #[tokio::test] async fn signed_requests_bypass_response_caching(cache: Arc) { use litellm_cache_response::ScopedCache; - use litellm_core::chat_completions::types::ChatCompletionsRequest; + use litellm_inference::chat_completions::types::ChatCompletionsRequest; use wiremock::{Mock, MockServer, ResponseTemplate, matchers::method}; let upstream = MockServer::start().await; diff --git a/litellm-rust/crates/core/tests/chat_completions.rs b/litellm-rust/crates/inference/tests/chat_completions.rs similarity index 98% rename from litellm-rust/crates/core/tests/chat_completions.rs rename to litellm-rust/crates/inference/tests/chat_completions.rs index b08fec41d3a..998e4b944ec 100644 --- a/litellm-rust/crates/core/tests/chat_completions.rs +++ b/litellm-rust/crates/inference/tests/chat_completions.rs @@ -5,8 +5,8 @@ use litellm_host::{ }; use std::time::Duration; -use litellm_core::chat_completions::{Error, types::ChatCompletionsRequest}; use litellm_http::transport::Error as TransportError; +use litellm_inference::chat_completions::{Error, types::ChatCompletionsRequest}; use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse; use rstest::{fixture, rstest}; use serde_json::{Map, Value, json}; @@ -257,8 +257,8 @@ async fn direct_and_hosted_calls_share_hooks_and_lifecycle( request: ChatCompletionsRequest<'static>, #[case] hosted: bool, ) { - use litellm_core::chat_completions::route::ChatCompletions; use litellm_host::{call::HostedCompletion, lifecycle::CallEvent}; + use litellm_inference::chat_completions::route::ChatCompletions; let upstream = upstream([anthropic_response(ANTHROPIC_MESSAGE)]).await; let base = upstream.uri(); diff --git a/litellm-rust/crates/core/tests/messages/host.rs b/litellm-rust/crates/inference/tests/messages/host.rs similarity index 98% rename from litellm-rust/crates/core/tests/messages/host.rs rename to litellm-rust/crates/inference/tests/messages/host.rs index 46ac2a634c5..04b6223bf1b 100644 --- a/litellm-rust/crates/core/tests/messages/host.rs +++ b/litellm-rust/crates/inference/tests/messages/host.rs @@ -1,11 +1,12 @@ use litellm_host::lifecycle::ExecutionEvent; use std::sync::Mutex; -use litellm_core::messages::{MessagesCallResponse, route::Messages}; use litellm_host::{ interceptors::{ExecutionFacts, RequestContext, ResultSource, WireRequest}, lifecycle::CallEvent, }; +use litellm_inference::messages::{MessagesCallResponse, route::Messages}; +use litellm_inference::test_support::{RecordingSecrets, no_secrets}; use litellm_llms::base_llm::messages::context::MessagesModelCapabilities as AnthropicModelCapabilities; use rstest::rstest; diff --git a/litellm-rust/crates/core/tests/messages/main.rs b/litellm-rust/crates/inference/tests/messages/main.rs similarity index 98% rename from litellm-rust/crates/core/tests/messages/main.rs rename to litellm-rust/crates/inference/tests/messages/main.rs index dd9689cf673..3ecd89dcf4b 100644 --- a/litellm-rust/crates/core/tests/messages/main.rs +++ b/litellm-rust/crates/inference/tests/messages/main.rs @@ -3,11 +3,12 @@ use std::{ time::Duration, }; -use litellm_core::messages::{ +use litellm_http::{HttpSettings, Resolution}; +use litellm_inference::messages::{ Error, MessagesCall, MessagesShaping, route::{Messages, MessagesMachine, MessagesOutput}, }; -use litellm_http::{HttpSettings, Resolution}; +use litellm_inference::test_support::RecordingSecrets; use litellm_llms_types::formats::messages::{MessagesRequest, MessagesResponse}; use litellm_secrets::source::SecretSource; use rstest::fixture; diff --git a/litellm-rust/crates/core/tests/messages/request.rs b/litellm-rust/crates/inference/tests/messages/request.rs similarity index 100% rename from litellm-rust/crates/core/tests/messages/request.rs rename to litellm-rust/crates/inference/tests/messages/request.rs diff --git a/litellm-rust/crates/core/tests/messages/response.rs b/litellm-rust/crates/inference/tests/messages/response.rs similarity index 97% rename from litellm-rust/crates/core/tests/messages/response.rs rename to litellm-rust/crates/inference/tests/messages/response.rs index 6d91e5e242d..78d553257c4 100644 --- a/litellm-rust/crates/core/tests/messages/response.rs +++ b/litellm-rust/crates/inference/tests/messages/response.rs @@ -1,9 +1,12 @@ -use litellm_core::messages::{MessagesCallResponse, messages_body}; use litellm_host::{ interceptors::{ExecutionFacts, ResultSource}, lifecycle::ExecutionEvent, }; use litellm_http::transport::Error as TransportError; +use litellm_inference::messages::{MessagesCallResponse, messages_body}; +use litellm_inference::test_support::{ + RecordingSecrets, http_config, no_secrets, provider_http, resources, +}; use rstest::rstest; use super::*; @@ -268,8 +271,8 @@ async fn the_facade_sends_through_the_injected_http_pool_configuration(call: Mes ..HttpSettings::default() }; - let resources = support::resources(); - let response = litellm_core::messages::MessagesRoute::new( + let resources = resources(); + let response = litellm_inference::messages::MessagesRoute::new( provider_http(&resources, &Resolution::from(&settings).config), resources.auth, no_secrets(), @@ -348,7 +351,7 @@ async fn route_uses_injected_dependencies_and_optional_cache( ) { use litellm_cache_memory::InMemoryCache; use litellm_cache_response::{CacheScope, ResponseCache, ScopedCache}; - use litellm_core::messages::MessagesRoute; + use litellm_inference::messages::MessagesRoute; let upstream = upstream([message_response(), message_response()]).await; let resources = resources(); diff --git a/litellm-rust/crates/core/tests/messages/secrets.rs b/litellm-rust/crates/inference/tests/messages/secrets.rs similarity index 99% rename from litellm-rust/crates/core/tests/messages/secrets.rs rename to litellm-rust/crates/inference/tests/messages/secrets.rs index 4b3d0ba705a..6d32fbed595 100644 --- a/litellm-rust/crates/core/tests/messages/secrets.rs +++ b/litellm-rust/crates/inference/tests/messages/secrets.rs @@ -1,3 +1,4 @@ +use litellm_inference::test_support::RecordingSecrets; use rstest::rstest; use super::*; diff --git a/litellm-rust/crates/core/tests/messages/stream.rs b/litellm-rust/crates/inference/tests/messages/stream.rs similarity index 98% rename from litellm-rust/crates/core/tests/messages/stream.rs rename to litellm-rust/crates/inference/tests/messages/stream.rs index ad5ae5a8765..dfe31f66c49 100644 --- a/litellm-rust/crates/core/tests/messages/stream.rs +++ b/litellm-rust/crates/inference/tests/messages/stream.rs @@ -5,10 +5,11 @@ use std::{ use bytes::Bytes; use futures_util::{StreamExt, TryStreamExt}; -use litellm_core::messages::{ +use litellm_inference::messages::{ MessagesCallResponse, route::{Messages, MessagesStreamHead}, }; +use litellm_inference::test_support::{RecordingSecrets, no_secrets}; use litellm_tracing::{Logger, Metadata, Record, Sink}; use rstest::rstest; use tokio::{ @@ -35,7 +36,7 @@ struct TraceSink(mpsc::Sender<(String, Value)>); impl Sink for TraceSink { fn enabled(&self, metadata: &Metadata<'_>) -> bool { - metadata.target().starts_with("litellm_core::messages") + metadata.target().starts_with("litellm_inference::messages") } fn emit(&self, record: &Record) { diff --git a/litellm-rust/crates/core/tests/ocr/aws_textract.rs b/litellm-rust/crates/inference/tests/ocr/aws_textract.rs similarity index 100% rename from litellm-rust/crates/core/tests/ocr/aws_textract.rs rename to litellm-rust/crates/inference/tests/ocr/aws_textract.rs diff --git a/litellm-rust/crates/core/tests/ocr/azure_ai.rs b/litellm-rust/crates/inference/tests/ocr/azure_ai.rs similarity index 100% rename from litellm-rust/crates/core/tests/ocr/azure_ai.rs rename to litellm-rust/crates/inference/tests/ocr/azure_ai.rs diff --git a/litellm-rust/crates/core/tests/ocr/azure_document_intelligence.rs b/litellm-rust/crates/inference/tests/ocr/azure_document_intelligence.rs similarity index 100% rename from litellm-rust/crates/core/tests/ocr/azure_document_intelligence.rs rename to litellm-rust/crates/inference/tests/ocr/azure_document_intelligence.rs diff --git a/litellm-rust/crates/core/tests/ocr/cohere.rs b/litellm-rust/crates/inference/tests/ocr/cohere.rs similarity index 100% rename from litellm-rust/crates/core/tests/ocr/cohere.rs rename to litellm-rust/crates/inference/tests/ocr/cohere.rs diff --git a/litellm-rust/crates/core/tests/ocr/documents.rs b/litellm-rust/crates/inference/tests/ocr/documents.rs similarity index 98% rename from litellm-rust/crates/core/tests/ocr/documents.rs rename to litellm-rust/crates/inference/tests/ocr/documents.rs index d99d99ead6e..439881d05ae 100644 --- a/litellm-rust/crates/core/tests/ocr/documents.rs +++ b/litellm-rust/crates/inference/tests/ocr/documents.rs @@ -1,6 +1,7 @@ use base64::Engine; -use litellm_core::ocr::types::OcrDocumentInput; use litellm_host::interceptors::WireRequest; +use litellm_inference::ocr::types::OcrDocumentInput; +use litellm_inference::test_support::{http_config, no_secrets, resources}; use rstest::rstest; use wiremock::{Mock, matchers::any}; diff --git a/litellm-rust/crates/core/tests/ocr/lifecycle.rs b/litellm-rust/crates/inference/tests/ocr/lifecycle.rs similarity index 99% rename from litellm-rust/crates/core/tests/ocr/lifecycle.rs rename to litellm-rust/crates/inference/tests/ocr/lifecycle.rs index dd94e44df37..171031e7ee5 100644 --- a/litellm-rust/crates/core/tests/ocr/lifecycle.rs +++ b/litellm-rust/crates/inference/tests/ocr/lifecycle.rs @@ -5,14 +5,14 @@ use std::sync::{ atomic::{AtomicUsize, Ordering}, }; -use litellm_core::ocr::{ - route::{Ocr, OcrCall, OcrOp}, - types::OcrDocumentInput, -}; use litellm_host::{ interceptors::{RequestContext, WireRequest}, lifecycle::CallEvent, }; +use litellm_inference::ocr::{ + route::{Ocr, OcrCall, OcrOp}, + types::OcrDocumentInput, +}; use rstest::rstest; use super::*; diff --git a/litellm-rust/crates/core/tests/ocr/machine.rs b/litellm-rust/crates/inference/tests/ocr/machine.rs similarity index 99% rename from litellm-rust/crates/core/tests/ocr/machine.rs rename to litellm-rust/crates/inference/tests/ocr/machine.rs index 3d7a4140635..8f7e08212ea 100644 --- a/litellm-rust/crates/core/tests/ocr/machine.rs +++ b/litellm-rust/crates/inference/tests/ocr/machine.rs @@ -6,16 +6,16 @@ use std::{ time::Duration, }; -use litellm_core::ocr::{ - route::{OcrCall, OcrMachine, OcrOp}, - types::OcrDocumentInput, -}; use litellm_host::{ interceptors::{Interceptors, WireRequest}, machine::{HostFailure, Machine, MachineStep}, protocol::{HostRequest, InterceptRequest}, }; use litellm_host_native::services::HostCallHandler; +use litellm_inference::ocr::{ + route::{OcrCall, OcrMachine, OcrOp}, + types::OcrDocumentInput, +}; use litellm_llms::base_llm::ocr::transformation::OcrTransportConfig; use rstest::rstest; use tokio::{io::AsyncReadExt, net::TcpListener, sync::Notify}; diff --git a/litellm-rust/crates/core/tests/ocr/main.rs b/litellm-rust/crates/inference/tests/ocr/main.rs similarity index 98% rename from litellm-rust/crates/core/tests/ocr/main.rs rename to litellm-rust/crates/inference/tests/ocr/main.rs index aa18dc8df24..e6c4b7483b5 100644 --- a/litellm-rust/crates/core/tests/ocr/main.rs +++ b/litellm-rust/crates/inference/tests/ocr/main.rs @@ -1,14 +1,15 @@ -use litellm_core::ocr::{ +use litellm_host::{ + interceptors::{RequestContext, WireRequest}, + lifecycle::CallEvent, +}; +use litellm_inference::ocr::{ OcrRoute, document::prepare_document, route::{Ocr, OcrCall, OcrOp}, types::{LiteLLMOcrRequest, OcrDocumentInput}, wire::{OcrWireRequest, decode_request}, }; -use litellm_host::{ - interceptors::{RequestContext, WireRequest}, - lifecycle::CallEvent, -}; +use litellm_inference::test_support::{http_config, no_secrets, resources}; use litellm_llms::base_llm::ocr::{error::Error, settings::OcrSettings}; use litellm_llms_types::formats::ocr::{LiteLLMOcrResponse, OcrDocument}; use serde_json::{Map, Value, json}; diff --git a/litellm-rust/crates/core/tests/ocr/mistral.rs b/litellm-rust/crates/inference/tests/ocr/mistral.rs similarity index 98% rename from litellm-rust/crates/core/tests/ocr/mistral.rs rename to litellm-rust/crates/inference/tests/ocr/mistral.rs index 34dec577c05..39baafffa9b 100644 --- a/litellm-rust/crates/core/tests/ocr/mistral.rs +++ b/litellm-rust/crates/inference/tests/ocr/mistral.rs @@ -1,6 +1,7 @@ use std::sync::Arc; use litellm_http::{HttpSettings, Resolution}; +use litellm_inference::test_support::{RecordingSecrets, http_config, no_secrets, resources}; use litellm_llms::{ base_llm::ocr::transformation::{BaseOcrConfig, OCR_RESPONSE_MAX_BYTES}, mistral::ocr::transformation::MistralOcrConfig, diff --git a/litellm-rust/crates/core/tests/ocr/reducto.rs b/litellm-rust/crates/inference/tests/ocr/reducto.rs similarity index 100% rename from litellm-rust/crates/core/tests/ocr/reducto.rs rename to litellm-rust/crates/inference/tests/ocr/reducto.rs diff --git a/litellm-rust/crates/core/tests/ocr/vertex_ai.rs b/litellm-rust/crates/inference/tests/ocr/vertex_ai.rs similarity index 98% rename from litellm-rust/crates/core/tests/ocr/vertex_ai.rs rename to litellm-rust/crates/inference/tests/ocr/vertex_ai.rs index 9981c5362e2..9eb78377f7f 100644 --- a/litellm-rust/crates/core/tests/ocr/vertex_ai.rs +++ b/litellm-rust/crates/inference/tests/ocr/vertex_ai.rs @@ -1,5 +1,5 @@ use litellm_auth::{InputSource, Sourced}; -use litellm_core::ocr::arguments::is_supported_request; +use litellm_inference::ocr::arguments::is_supported_request; use litellm_llms::base_llm::ocr::settings::OcrSettings; use rstest::rstest; diff --git a/litellm-rust/crates/core/tests/resources.rs b/litellm-rust/crates/inference/tests/resources.rs similarity index 96% rename from litellm-rust/crates/core/tests/resources.rs rename to litellm-rust/crates/inference/tests/resources.rs index d910792ff72..61ff0656da7 100644 --- a/litellm-rust/crates/core/tests/resources.rs +++ b/litellm-rust/crates/inference/tests/resources.rs @@ -9,15 +9,16 @@ use litellm_auth::AuthServices; use litellm_auth_gcp::{ CredentialSource, VertexAuth, VertexAuthFuture, VertexProviderLoader, VertexTokenSource, }; -use litellm_core::{ +use litellm_http::{HttpSettings, Resolution}; +use litellm_inference::test_support::{RecordingSecrets, http_pool}; +use litellm_inference::{ ocr::wire::{OcrWireRequest, decode_request}, resources::CoreResources, }; -use litellm_http::{HttpSettings, Resolution}; use litellm_llms::base_llm::ocr::settings::OcrSettings; use rstest::{fixture, rstest}; use serde_json::json; -use support::{ReceivedRequest, RecordingSecrets, http_pool, json_response, upstream}; +use support::{ReceivedRequest, json_response, upstream}; struct TokenSource(String); diff --git a/litellm-rust/crates/core/tests/responses.rs b/litellm-rust/crates/inference/tests/responses.rs similarity index 98% rename from litellm-rust/crates/core/tests/responses.rs rename to litellm-rust/crates/inference/tests/responses.rs index cd2a0c4734c..0a15be38479 100644 --- a/litellm-rust/crates/core/tests/responses.rs +++ b/litellm-rust/crates/inference/tests/responses.rs @@ -5,11 +5,12 @@ use litellm_host::{ use std::sync::Arc; use futures_util::TryStreamExt; -use litellm_core::responses::{ +use litellm_host::{call::HostedCompletion, lifecycle::CallEvent}; +use litellm_inference::responses::{ route::Responses, types::{ResponsesCall, ResponsesOutput}, }; -use litellm_host::{call::HostedCompletion, lifecycle::CallEvent}; +use litellm_inference::test_support::{RecordingSecrets, no_secrets}; use rstest::{fixture, rstest}; use serde_json::json; use wiremock::ResponseTemplate; @@ -417,7 +418,7 @@ async fn websocket_operations_trace_outcomes_without_capturing_frames_or_credent traces: TraceCapture, ) { use futures_util::{SinkExt, StreamExt}; - use litellm_core::responses::websocket::ResponsesWebSocketConnection; + use litellm_inference::responses::websocket::ResponsesWebSocketConnection; let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); diff --git a/litellm-rust/crates/core/tests/support/mod.rs b/litellm-rust/crates/inference/tests/support/mod.rs similarity index 74% rename from litellm-rust/crates/core/tests/support/mod.rs rename to litellm-rust/crates/inference/tests/support/mod.rs index 1dd53114293..8aad53857dc 100644 --- a/litellm-rust/crates/core/tests/support/mod.rs +++ b/litellm-rust/crates/inference/tests/support/mod.rs @@ -8,70 +8,50 @@ use std::{ sync::{Arc, Mutex}, }; -use futures_util::future::BoxFuture; -use litellm_http::{ - ClientVariant, HttpClientConfig, HttpClientPool, HttpSettings, Resolution, - media::PublicDnsResolver, -}; -use litellm_secrets::{SecretValue, source::SecretSource}; +use litellm_http::HttpClientConfig; +use litellm_inference::test_support::{http_config, no_secrets, provider_http, resources}; +use litellm_secrets::source::SecretSource; use serde_json::Value; use wiremock::{Mock, MockServer, Request, ResponseTemplate, matchers::any}; /// A port nothing listens on, for calls that must fail before any request is sent. pub const UNREACHABLE_BASE: &str = "http://127.0.0.1:1"; -pub fn http_pool() -> HttpClientPool { - HttpClientPool::new(Arc::new(PublicDnsResolver)) -} - -pub fn resources() -> litellm_core::resources::CoreResources { - litellm_core::resources::CoreResources::new(Arc::new(http_pool())) -} - -pub fn no_secrets() -> Arc { - Arc::new(RecordingSecrets::empty()) -} - -pub fn provider_http( - resources: &litellm_core::resources::CoreResources, - config: &HttpClientConfig, -) -> litellm_http::Client { - resources - .pool - .client(config, ClientVariant::Provider) - .unwrap() -} - -pub fn messages_route(secrets: Arc) -> litellm_core::messages::MessagesRoute { +pub fn messages_route( + secrets: Arc, +) -> litellm_inference::messages::MessagesRoute { let resources = resources(); - litellm_core::messages::MessagesRoute::new( + litellm_inference::messages::MessagesRoute::new( provider_http(&resources, &http_config()), resources.auth, secrets, ) } -pub fn chat_completions_route() -> litellm_core::chat_completions::ChatCompletionsRoute { +pub fn chat_completions_route() -> litellm_inference::chat_completions::ChatCompletionsRoute { let resources = resources(); - litellm_core::chat_completions::ChatCompletionsRoute::new( + litellm_inference::chat_completions::ChatCompletionsRoute::new( provider_http(&resources, &http_config()), resources.auth, no_secrets(), ) } -pub fn responses_route(secrets: Arc) -> litellm_core::responses::ResponsesRoute { +pub fn responses_route( + secrets: Arc, +) -> litellm_inference::responses::ResponsesRoute { let resources = resources(); - litellm_core::responses::ResponsesRoute::new( + litellm_inference::responses::ResponsesRoute::new( provider_http(&resources, &http_config()), resources.auth, secrets, ) } -pub fn audio_transcription_route() -> litellm_core::audio_transcription::AudioTranscriptionRoute { +pub fn audio_transcription_route() -> litellm_inference::audio_transcription::AudioTranscriptionRoute +{ let resources = resources(); - litellm_core::audio_transcription::AudioTranscriptionRoute::new( + litellm_inference::audio_transcription::AudioTranscriptionRoute::new( provider_http(&resources, &http_config()), resources.auth, no_secrets(), @@ -79,13 +59,13 @@ pub fn audio_transcription_route() -> litellm_core::audio_transcription::AudioTr } pub fn build_ocr_route( - resources: &litellm_core::resources::CoreResources, + resources: &litellm_inference::resources::CoreResources, config: &HttpClientConfig, url_policy: litellm_http::media::UrlPolicy, settings: litellm_llms::base_llm::ocr::settings::OcrSettings, secrets: Arc, -) -> litellm_core::ocr::OcrRoute { - litellm_core::ocr::OcrRoute::new( +) -> litellm_inference::ocr::OcrRoute { + litellm_inference::ocr::OcrRoute::new( litellm_llms::base_llm::ocr::handler::OcrClient::new( &resources.pool, config, @@ -98,10 +78,6 @@ pub fn build_ocr_route( ) } -pub fn http_config() -> HttpClientConfig { - Resolution::from(&HttpSettings::default()).config -} - /// Starts an upstream that answers its n-th request with the n-th response and 404s after. pub async fn upstream(responses: impl IntoIterator) -> MockServer { let server = MockServer::start().await; @@ -189,60 +165,6 @@ impl ReceivedRequest for Request { } } -/// A secret source that answers from a fixed table and records every name it was asked for. -pub struct RecordingSecrets { - values: Vec<(String, String)>, - fails: bool, - requested: Mutex>, -} - -impl RecordingSecrets { - pub fn new<'a>(values: impl IntoIterator) -> Self { - Self { - values: values - .into_iter() - .map(|(name, value)| (name.to_string(), value.to_string())) - .collect(), - fails: false, - requested: Mutex::new(Vec::new()), - } - } - - pub fn empty() -> Self { - Self::new([]) - } - - pub fn failing() -> Self { - Self { - fails: true, - ..Self::empty() - } - } - - pub fn requested(&self) -> Vec { - self.requested.lock().unwrap().clone() - } -} - -impl SecretSource for RecordingSecrets { - fn get_secret_str<'a>( - &'a self, - name: &'a str, - ) -> BoxFuture<'a, Result, litellm_secrets::Error>> { - Box::pin(async move { - self.requested.lock().unwrap().push(name.to_string()); - if self.fails { - return Err(litellm_secrets::Error::ManagedSecretMissing); - } - Ok(self - .values - .iter() - .find(|(key, _)| key == name) - .map(|(_, value)| SecretValue::new(value.clone()))) - }) - } -} - pub struct RecordingCall { pub request: Mutex>, pub events: Arc, @@ -396,7 +318,7 @@ impl TraceCapture { impl litellm_tracing::Sink for TraceCapture { fn enabled(&self, metadata: &litellm_tracing::Metadata<'_>) -> bool { - metadata.target().starts_with("litellm_core") + metadata.target().starts_with("litellm_inference") } fn emit(&self, record: &litellm_tracing::Record) { diff --git a/litellm-rust/crates/llms/AGENTS.md b/litellm-rust/crates/llms/AGENTS.md index c48dc7962c8..91b81536fe7 100644 --- a/litellm-rust/crates/llms/AGENTS.md +++ b/litellm-rust/crates/llms/AGENTS.md @@ -16,7 +16,7 @@ Shared OCR document and response contracts live in `litellm-llms-types::formats: For Mistral, `async_transform_ocr_request` uses the base default in both languages. `resolve_headers` and `build_ocr_url` implement the respective environment and URL operations, and `normalize_response` implements the typed part of response transformation. Existing auth key/header handling and top-level response-extra preservation differ between languages; layout refactors must preserve those behaviors and verify them with the existing tests -For non-OCR pairs, order corresponding methods as parameter support/mapping, environment validation, URL construction, request transformation, and response transformation, followed by Rust-only runtime hooks. Auth resolution remains split between configs and route preparation in litellm-core. Chat `supported_openai_param_mappings` describes accepted OpenAI/provider name pairs, unlike Python's `get_supported_openai_params` name list. Audio `map_transcription_params` remains a Rust filtering helper +For non-OCR pairs, order corresponding methods as parameter support/mapping, environment validation, URL construction, request transformation, and response transformation, followed by Rust-only runtime hooks. Auth resolution remains split between configs and route preparation in litellm-inference. Chat `supported_openai_param_mappings` describes accepted OpenAI/provider name pairs, unlike Python's `get_supported_openai_params` name list. Audio `map_transcription_params` remains a Rust filtering helper Azure Messages maps to `llms/azure_ai/anthropic/messages_transformation.py`; Bedrock Converse maps to `llms/bedrock/chat/converse_transformation.py`. `AnthropicConfig`, `AmazonConverseConfig`, and the non-OCR base traits are partial ports. `OpenAiResponsesApiConfig` implements WebSocket transformations and a direct HTTP Responses path. Its HTTP path does not implement Python model-specific parameter rewriting or Responses-to-Chat emulation. Preserve their acceptance gates, passthrough behavior, and host fallback contracts when aligning layout diff --git a/litellm-rust/crates/python-bridge/AGENTS.md b/litellm-rust/crates/python-bridge/AGENTS.md index 162b0bdb07b..a7ba35d9051 100644 --- a/litellm-rust/crates/python-bridge/AGENTS.md +++ b/litellm-rust/crates/python-bridge/AGENTS.md @@ -82,11 +82,11 @@ GIL handling to `litellm-host-python`. ## Bridge Shape - Prefer one stable method per top-level LiteLLM route, for example - `messages(...)`, calling the matching `litellm-core` entrypoint. + `messages(...)`, calling the matching `litellm-inference` entrypoint. - Do not add one exported PyO3 function per provider helper unless there is a measured reason. -- Provider dispatch belongs in the `litellm-core` route module (e.g. - `litellm_core::messages`), not in this PyO3 crate. +- Provider dispatch belongs in the `litellm-inference` route module (e.g. + `litellm_inference::messages`), not in this PyO3 crate. - Python owns rollout state and fallback. Rust should return errors; Python decides whether to raise or fall back. For a rust-only provider/route (no Python reference), the Python side is a thin dispatch that calls Rust and diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index 105241e1440..cc9d85aca1f 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -36,7 +36,7 @@ serde.workspace = true litellm-auth.workspace = true litellm-auth-aws.workspace = true litellm-callbacks-legacy-python.workspace = true -litellm-core.workspace = true +litellm-inference.workspace = true litellm-core-utils.workspace = true litellm-http.workspace = true litellm-llms.workspace = true diff --git a/litellm-rust/crates/python-bridge/src/cache/python/service.rs b/litellm-rust/crates/python-bridge/src/cache/python/service.rs index 14310d952b2..f215e5e97f5 100644 --- a/litellm-rust/crates/python-bridge/src/cache/python/service.rs +++ b/litellm-rust/crates/python-bridge/src/cache/python/service.rs @@ -4,11 +4,11 @@ use litellm_cache::Error; use litellm_cache_response::{ ResponseCacheConfig, ResponseCacheRequest, ResponseCacheService, ResponseEnvelope, }; -use litellm_core::caching::CachedOutput; use litellm_host::{ machine::{HostServices, MachineFault}, protocol::{Protocol, Reply}, }; +use litellm_inference::caching::CachedOutput; use serde_json::Value; const STREAM_EVENTS_KEY: &str = "litellm_cached_anthropic_sse_events"; diff --git a/litellm-rust/crates/python-bridge/src/errors.rs b/litellm-rust/crates/python-bridge/src/errors.rs index eaeef7c50ff..dc63f0d6353 100644 --- a/litellm-rust/crates/python-bridge/src/errors.rs +++ b/litellm-rust/crates/python-bridge/src/errors.rs @@ -1,5 +1,5 @@ -use litellm_core::RouteError; use litellm_http::transport::Error as TransportError; +use litellm_inference::RouteError; use pyo3::{ exceptions::{PyRuntimeError, PyValueError}, prelude::*, diff --git a/litellm-rust/crates/python-bridge/src/http.rs b/litellm-rust/crates/python-bridge/src/http.rs index a5c2166c18e..57bf8aafc4e 100644 --- a/litellm-rust/crates/python-bridge/src/http.rs +++ b/litellm-rust/crates/python-bridge/src/http.rs @@ -80,13 +80,13 @@ fn decode_ssl_verify(field: &Field<'_>) -> Result, ProjectionE Err(field.invalid("a Boolean, Boolean string, CA path, or None")) } -static RESOURCES: LazyLock = LazyLock::new(|| { - litellm_core::resources::CoreResources::new(Arc::new(HttpClientPool::new(Arc::new( +static RESOURCES: LazyLock = LazyLock::new(|| { + litellm_inference::resources::CoreResources::new(Arc::new(HttpClientPool::new(Arc::new( PublicDnsResolver, )))) }); -pub(crate) fn resources() -> &'static litellm_core::resources::CoreResources { +pub(crate) fn resources() -> &'static litellm_inference::resources::CoreResources { &RESOURCES } diff --git a/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs b/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs index 8d434dbbc74..0faf162eb90 100644 --- a/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs +++ b/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs @@ -1,8 +1,8 @@ use crate::execution::{run_async, run_sync}; -use litellm_core::audio_transcription::{ +use litellm_host_python::from_py_argument; +use litellm_inference::audio_transcription::{ AudioTranscriptionRoute, Error, types::AudioTranscriptionRequest, }; -use litellm_host_python::from_py_argument; use pyo3::{prelude::*, types::PyDict}; use serde_json::{Map, Value}; diff --git a/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs b/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs index 5955729d6e9..9ea674ad341 100644 --- a/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs +++ b/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs @@ -3,7 +3,9 @@ mod host; use pyo3::types::{PyDict, PyTuple}; use crate::execution::{run_async, run_sync}; -use litellm_core::chat_completions::{ChatCompletionsRoute, Error, types::ChatCompletionsRequest}; +use litellm_inference::chat_completions::{ + ChatCompletionsRoute, Error, types::ChatCompletionsRequest, +}; use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse; use pyo3::prelude::*; use serde_json::{Map, Value}; diff --git a/litellm-rust/crates/python-bridge/src/routes/chat_completions/host.rs b/litellm-rust/crates/python-bridge/src/routes/chat_completions/host.rs index 5011e151b48..a062ffc0aa6 100644 --- a/litellm-rust/crates/python-bridge/src/routes/chat_completions/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/chat_completions/host.rs @@ -1,8 +1,10 @@ use std::convert::Infallible; use super::super::inference::InferenceHost; -use litellm_core::chat_completions::{Error, route::ChatCompletions, types::ChatCompletionsCall}; use litellm_host_python::{InvokeError, PythonBinding, PythonHostCalls, PythonOwned}; +use litellm_inference::chat_completions::{ + Error, route::ChatCompletions, types::ChatCompletionsCall, +}; use pyo3::{ gc::{PyTraverseError, PyVisit}, prelude::*, diff --git a/litellm-rust/crates/python-bridge/src/routes/inference.rs b/litellm-rust/crates/python-bridge/src/routes/inference.rs index 42015802f01..cb1e31c7c85 100644 --- a/litellm-rust/crates/python-bridge/src/routes/inference.rs +++ b/litellm-rust/crates/python-bridge/src/routes/inference.rs @@ -1,7 +1,7 @@ -use litellm_core::RouteError; use litellm_core_utils::get_llm_provider_logic::get_custom_llm_provider; use litellm_host_python::{from_py, lookup, to_py}; use litellm_http::transport::Error as TransportError; +use litellm_inference::RouteError; use pyo3::{exceptions::PyValueError, prelude::*, types::PyDict}; use serde::Serialize; use serde_json::{Map, Value}; diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs index 2f67151374d..127ed81b6b1 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs @@ -2,12 +2,12 @@ use crate::cache::{CacheCall, Cached, PythonCache, Selection}; use litellm_host_python::{PythonHostCalls, PythonOwned}; use bytes::Bytes; -use litellm_core::messages::{ +use litellm_host_python::{InvokeError, PythonBinding, from_py, lookup, to_py}; +use litellm_http::transport::Error as TransportError; +use litellm_inference::messages::{ Error, MessagesCall, MessagesShaping, messages_body, route::{Messages, MessagesStreamHead}, }; -use litellm_host_python::{InvokeError, PythonBinding, from_py, lookup, to_py}; -use litellm_http::transport::Error as TransportError; use litellm_llms_types::headers::ProviderSpecificHeaders; use pyo3::{ exceptions::{PyException, PyValueError}, diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs b/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs index 4838c973a34..75ecbbb128a 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs @@ -26,7 +26,7 @@ fn run_messages( py, arguments, move |py, arguments, request| { - let route = litellm_core::messages::MessagesRoute::new( + let route = litellm_inference::messages::MessagesRoute::new( crate::http::provider_client(py, arguments, asynchronous)? .map_err(crate::http::client_error)?, crate::http::resources().auth.clone(), @@ -48,7 +48,7 @@ fn run_messages( .execute( call, &interceptors, - litellm_core::CallOptions { + litellm_inference::CallOptions { cache: Some(options.policy), observers, }, diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/document.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/document.rs index e6821241c89..866a1510f80 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/document.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/document.rs @@ -1,7 +1,7 @@ use std::path::PathBuf; -use litellm_core::ocr::types::OcrDocumentInput; use litellm_host_python::{PythonFileReader, py_bytes}; +use litellm_inference::ocr::types::OcrDocumentInput; use pyo3::{ exceptions::PyValueError, prelude::*, diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs index 28317317544..a5ba3fc17ac 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs @@ -1,7 +1,7 @@ use litellm_auth::ResolvedCredential; -use litellm_core::ocr::route::{Ocr, OcrCall, OcrOp}; use litellm_host_python::{InvokeError, PythonBinding, missing_state, to_py}; use litellm_host_python::{PythonHostCalls, PythonOwned}; +use litellm_inference::ocr::route::{Ocr, OcrCall, OcrOp}; use litellm_llms::base_llm::ocr::error::Error; use litellm_llms_types::formats::ocr::LiteLLMOcrResponse; use pyo3::{ diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs index 041d5c7d0b3..e77e2edc3b1 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs @@ -5,9 +5,9 @@ mod project; use host::OcrPythonHost; use litellm_callbacks_legacy_python::LoggingOperation; -use litellm_core::ocr::provider_config; use litellm_core_utils::settings::ProcessEnvironment; use litellm_host_python::to_py; +use litellm_inference::ocr::provider_config; use litellm_llms::base_llm::ocr::settings::OcrSettings; use pyo3::{ prelude::*, @@ -58,7 +58,7 @@ fn run_ocr( crate::secrets::source(py)?, ) .map_err(http::client_error)?; - let route = litellm_core::ocr::OcrRoute::new(client); + let route = litellm_inference::ocr::OcrRoute::new(client); Ok(route.machine(request, None)) }, OcrPythonHost::new(request.unbind()), diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs index 959993f5493..c60e0da96da 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs @@ -1,9 +1,9 @@ use litellm_auth::SecretValue; -use litellm_core::ocr::{ +use litellm_host_python::from_py; +use litellm_inference::ocr::{ types::{LiteLLMOcrRequest, OcrDocumentInput}, wire::{OcrWireRequest, consumed_optional_params, decode_document, decode_request_input}, }; -use litellm_host_python::from_py; use litellm_llms::base_llm::ocr::error::Error; use pyo3::{exceptions::PyValueError, prelude::*, types::PyDict}; use serde_json::{Map, Value}; diff --git a/litellm-rust/crates/python-bridge/src/routes/responses.rs b/litellm-rust/crates/python-bridge/src/routes/responses.rs index 4e1426cd298..527bf51feab 100644 --- a/litellm-rust/crates/python-bridge/src/routes/responses.rs +++ b/litellm-rust/crates/python-bridge/src/routes/responses.rs @@ -1,6 +1,6 @@ mod host; -use litellm_core::responses::websocket::ResponsesWebSocketConnection as RustResponsesWebSocketConnection; +use litellm_inference::responses::websocket::ResponsesWebSocketConnection as RustResponsesWebSocketConnection; use pyo3::{ prelude::*, types::{PyDict, PyTuple}, @@ -81,7 +81,7 @@ fn run_public( py, arguments, move |py, arguments, request| { - let route = litellm_core::responses::ResponsesRoute::new( + let route = litellm_inference::responses::ResponsesRoute::new( crate::http::provider_client(py, arguments, asynchronous)? .map_err(crate::http::client_error)?, crate::http::resources().auth.clone(), diff --git a/litellm-rust/crates/python-bridge/src/routes/responses/host.rs b/litellm-rust/crates/python-bridge/src/routes/responses/host.rs index b054cbe5f38..cb2750e9398 100644 --- a/litellm-rust/crates/python-bridge/src/routes/responses/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/responses/host.rs @@ -1,8 +1,8 @@ use std::convert::Infallible; use super::super::inference::InferenceHost; -use litellm_core::responses::{Error, route::Responses, types::ResponsesCall}; use litellm_host_python::{InvokeError, PythonBinding, PythonHostCalls, PythonOwned}; +use litellm_inference::responses::{Error, route::Responses, types::ResponsesCall}; use pyo3::{ gc::{PyTraverseError, PyVisit}, prelude::*, diff --git a/litellm-rust/crates/router/Cargo.toml b/litellm-rust/crates/router/Cargo.toml index cc6972a066e..bf878fe391c 100644 --- a/litellm-rust/crates/router/Cargo.toml +++ b/litellm-rust/crates/router/Cargo.toml @@ -7,7 +7,7 @@ repository.workspace = true [dependencies] litellm-config.workspace = true -litellm-core.workspace = true +litellm-inference.workspace = true [dev-dependencies] rstest.workspace = true diff --git a/litellm-rust/crates/router/src/deployment.rs b/litellm-rust/crates/router/src/deployment.rs index 9234728db28..fbe4a48aaa0 100644 --- a/litellm-rust/crates/router/src/deployment.rs +++ b/litellm-rust/crates/router/src/deployment.rs @@ -1,6 +1,6 @@ use std::time::Duration; -use litellm_core::messages::MessagesShaping; +use litellm_inference::messages::MessagesShaping; #[derive(Clone, Debug, Default)] pub struct Deployment { diff --git a/litellm-rust/crates/router/tests/router.rs b/litellm-rust/crates/router/tests/router.rs index 2d16f102de3..f2cf6609106 100644 --- a/litellm-rust/crates/router/tests/router.rs +++ b/litellm-rust/crates/router/tests/router.rs @@ -1,7 +1,7 @@ use std::time::Duration; use litellm_config::Config; -use litellm_core::messages::MessagesShaping; +use litellm_inference::messages::MessagesShaping; use litellm_router::{Deployment, Router}; use rstest::rstest; diff --git a/tests/unit/test_circleci_path_filter.py b/tests/unit/test_circleci_path_filter.py index 281790ee13e..448dfb8c280 100644 --- a/tests/unit/test_circleci_path_filter.py +++ b/tests/unit/test_circleci_path_filter.py @@ -73,7 +73,7 @@ CI = [".github/workflows/test-litellm-ui-unit.yml"] ("provider-harness", ["tests/e2e/quota_management/test_quota.py"], "skip"), ("provider-harness", ["litellm/main.py"], "skip"), ("provider-harness", ["ui/litellm-dashboard/src/App.tsx"], "skip"), - ("windows-release", ["litellm-rust/crates/core/src/lib.rs"], "run"), + ("windows-release", ["litellm-rust/crates/inference/src/lib.rs"], "run"), ("windows-release", ["litellm/rust_bridge/dispatch.py"], "run"), ("windows-release", ["rust-toolchain.toml"], "run"), ("windows-release", ["pyproject.toml"], "run"), @@ -204,7 +204,7 @@ def _pr_repo(tmp_path: Path, feature_files: dict[str, str]) -> Path: _git(work, "config", "user.email", "t@t") _git(work, "config", "user.name", "t") _git(work, "remote", "add", "origin", str(remote)) - (work / "litellm_core.py").write_text("x\n") + (work / "litellm_inference.py").write_text("x\n") _git(work, "add", ".") _git(work, "commit", "-qm", "base") _git(work, "push", "-q", "origin", "main")