diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 9bffe9f9ec6..3da7d438f50 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -670,6 +670,7 @@ name = "litellm-core" version = "0.1.0" dependencies = [ "rand 0.8.6", + "reqwest", "serde", "serde_json", "thiserror 2.0.18", diff --git a/litellm-rust/crates/ai-gateway/src/constants.rs b/litellm-rust/crates/ai-gateway/src/constants.rs index 109b648f5db..dd41b60de42 100644 --- a/litellm-rust/crates/ai-gateway/src/constants.rs +++ b/litellm-rust/crates/ai-gateway/src/constants.rs @@ -5,26 +5,6 @@ //! modules. Env-overridable tunables keep their `DEFAULT_*` value here; the env //! read + fallback happens at the host/config layer. -/// Default LiteLLM control-plane base URL for request-log egress when -/// `LITELLM_PROXY_BASE_URL` is unset. -pub(crate) const DEFAULT_PROXY_BASE_URL: &str = "http://localhost:4000"; - -/// The logs ingest path appended to the proxy base. Not a tunable; it is the -/// proxy's API contract (the rust-control-plane router on the Python proxy). -pub(crate) const RUST_CONTROL_PLANE_LOGS_PATH: &str = "/v1/rust_control_plane/logs"; - -/// Default bounded channel depth for the log-egress worker. -/// Override: `LITELLM_LOG_CHANNEL_CAPACITY`. -pub(crate) const DEFAULT_CHANNEL_CAPACITY: usize = 4096; - -/// Default max records POSTed per request to the control plane. -/// Override: `LITELLM_LOG_BATCH_SIZE`. -pub(crate) const DEFAULT_MAX_BATCH_SIZE: usize = 256; - -/// Default partial-batch flush cadence, in ms. -/// Override: `LITELLM_LOG_FLUSH_INTERVAL_MS`. -pub(crate) const DEFAULT_FLUSH_INTERVAL_MS: u64 = 500; - /// Provider attributed to realtime sessions in the logging payload. #[cfg(feature = "server")] pub(crate) const DEFAULT_PROVIDER: &str = "openai"; diff --git a/litellm-rust/crates/ai-gateway/src/lib.rs b/litellm-rust/crates/ai-gateway/src/lib.rs index d8ef7bb5ba1..8ff749f2356 100644 --- a/litellm-rust/crates/ai-gateway/src/lib.rs +++ b/litellm-rust/crates/ai-gateway/src/lib.rs @@ -25,11 +25,9 @@ pub mod routes; #[cfg(feature = "server")] pub mod state; -// Realtime request logging. Only the server serves realtime, so these are -// `server`-gated; `io::realtime` exposes the generic `observe` hook while the -// collector and callback fan-out live here. +// Realtime server collector. Integration traits and callback fan-out live in +// `litellm-core`; the gateway only wires them into transport/runtime state. mod constants; -pub mod integrations; #[cfg(feature = "server")] mod realtime; diff --git a/litellm-rust/crates/ai-gateway/src/main.rs b/litellm-rust/crates/ai-gateway/src/main.rs index f9ce97801d3..29ade7bb707 100644 --- a/litellm-rust/crates/ai-gateway/src/main.rs +++ b/litellm-rust/crates/ai-gateway/src/main.rs @@ -16,10 +16,10 @@ use litellm_ai_gateway::routes; use litellm_ai_gateway::state::AppState; use litellm_core::router::{Deployment, LiteLLMParams, Router}; -use litellm_ai_gateway::integrations::custom_logger::CustomLogger; -use litellm_ai_gateway::integrations::litellm_python_proxy_api::LiteLLMPythonProxyAPILogger; #[cfg(feature = "python-config")] use litellm_ai_gateway::python; +use litellm_core::integrations::custom_logger::CustomLogger; +use litellm_core::integrations::litellm_python_proxy_api::LiteLLMPythonProxyAPILogger; /// Bind to localhost by default so the gateway is not a public, unauthenticated /// provider proxy out of the box. Override with `HOST` (e.g. `0.0.0.0`). diff --git a/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs b/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs index 4df26f336d3..5e826b4ff3f 100644 --- a/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs +++ b/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs @@ -13,13 +13,13 @@ use super::common_utils::{ convert_document_url_to_data_uri, has_header, ocr_provider_config, string_headers, }; use super::types::{PreparedOcrRequest, ProviderOcrRequest}; -use crate::integrations::custom_guardrail::{ +use litellm_core::integrations::custom_guardrail::{ CustomGuardrailRunner, GuardrailContext, GuardrailError, GuardrailRequest, }; -use crate::integrations::custom_logger::{ +use litellm_core::integrations::custom_logger::{ CallType, CallbackTiming, CallbackValue, CustomLoggerRunner, LoggingError, ModelCallDetails, }; -use crate::integrations::types::{ +use litellm_core::integrations::types::{ RequestMetadata, StandardLoggingMetadata, StandardLoggingPayload, }; diff --git a/litellm-rust/crates/ai-gateway/src/ocr/prepare.rs b/litellm-rust/crates/ai-gateway/src/ocr/prepare.rs index 6ff55a4520a..d05592e3494 100644 --- a/litellm-rust/crates/ai-gateway/src/ocr/prepare.rs +++ b/litellm-rust/crates/ai-gateway/src/ocr/prepare.rs @@ -7,8 +7,8 @@ use serde_json::{json, Value}; use super::hooks::OcrLifecycleHooks; use super::types::{OcrRequest, PreparedOcrRequest}; -use crate::integrations::custom_guardrail::CustomGuardrailRunner; -use crate::integrations::custom_logger::CustomLoggerRunner; +use litellm_core::integrations::custom_guardrail::CustomGuardrailRunner; +use litellm_core::integrations::custom_logger::CustomLoggerRunner; pub(crate) struct PreparedOcrCall { pub(crate) request: PreparedOcrRequest, diff --git a/litellm-rust/crates/ai-gateway/src/ocr/tests.rs b/litellm-rust/crates/ai-gateway/src/ocr/tests.rs index 299d3bdbcc2..7095d50b74d 100644 --- a/litellm-rust/crates/ai-gateway/src/ocr/tests.rs +++ b/litellm-rust/crates/ai-gateway/src/ocr/tests.rs @@ -9,14 +9,14 @@ use tokio::net::{TcpListener, TcpStream}; use super::common_utils::{has_header, ocr_provider_config, string_headers, truncate_error_body}; use super::{ocr, OcrRequest}; -use crate::integrations::custom_guardrail::{ +use litellm_core::integrations::custom_guardrail::{ CustomGuardrail, GuardrailContext, GuardrailDecision, GuardrailError, GuardrailEventHook, GuardrailFuture, GuardrailRequest, }; -use crate::integrations::custom_logger::{ +use litellm_core::integrations::custom_logger::{ CallbackTiming, CallbackValue, CustomLogger, LogFuture, ModelCallDetails, }; -use crate::integrations::types::RequestMetadata; +use litellm_core::integrations::types::RequestMetadata; async fn read_http_headers(socket: &mut TcpStream) -> String { let mut request = Vec::new(); diff --git a/litellm-rust/crates/ai-gateway/src/ocr/types.rs b/litellm-rust/crates/ai-gateway/src/ocr/types.rs index bde734a4dd1..f838ee6988c 100644 --- a/litellm-rust/crates/ai-gateway/src/ocr/types.rs +++ b/litellm-rust/crates/ai-gateway/src/ocr/types.rs @@ -5,9 +5,9 @@ use litellm_core::call_lifecycle::{CallLifecycleContext, CallLifecycleRequest}; use litellm_core::ocr::transformation::OcrProviderConfig; use serde_json::{Map, Value}; -use crate::integrations::custom_guardrail::CustomGuardrail; -use crate::integrations::custom_logger::CustomLogger; -use crate::integrations::types::RequestMetadata; +use litellm_core::integrations::custom_guardrail::CustomGuardrail; +use litellm_core::integrations::custom_logger::CustomLogger; +use litellm_core::integrations::types::RequestMetadata; pub struct OcrRequest<'a> { pub model: &'a str, diff --git a/litellm-rust/crates/ai-gateway/src/realtime/streaming.rs b/litellm-rust/crates/ai-gateway/src/realtime/streaming.rs index 9ebfa3a7d70..aa6990110a1 100644 --- a/litellm-rust/crates/ai-gateway/src/realtime/streaming.rs +++ b/litellm-rust/crates/ai-gateway/src/realtime/streaming.rs @@ -13,10 +13,10 @@ use litellm_core::realtime::types::RealtimeEvent; use serde_json::Value; use crate::constants::DEFAULT_PROVIDER; -use crate::integrations::custom_logger::{ +use litellm_core::integrations::custom_logger::{ CallbackTiming, CallbackValue, CustomLogger, CustomLoggerRunner, LoggingError, ModelCallDetails, }; -use crate::integrations::types::{ +use litellm_core::integrations::types::{ RequestMetadata, StandardLoggingMetadata, StandardLoggingPayload, Usage, }; @@ -233,8 +233,8 @@ impl RealTimeStreaming { #[cfg(test)] mod tests { use super::*; - use crate::integrations::custom_logger::LogError; - use crate::integrations::custom_logger::LogFuture; + use litellm_core::integrations::custom_logger::LogError; + use litellm_core::integrations::custom_logger::LogFuture; use std::sync::atomic::{AtomicU64, Ordering}; fn event(raw: &str) -> RealtimeEvent { diff --git a/litellm-rust/crates/ai-gateway/src/routes/realtime/mod.rs b/litellm-rust/crates/ai-gateway/src/routes/realtime/mod.rs index c3f929f5f0b..d4cf15a2ce1 100644 --- a/litellm-rust/crates/ai-gateway/src/routes/realtime/mod.rs +++ b/litellm-rust/crates/ai-gateway/src/routes/realtime/mod.rs @@ -23,10 +23,10 @@ use litellm_core::router::Router as ModelRouter; use serde::Deserialize; use crate::auth::RequireMasterKey; -use crate::integrations::custom_logger::CustomLogger; -use crate::integrations::types::RequestMetadata; use crate::realtime::streaming::{RealTimeStreaming, SessionStatus}; use crate::state::AppState; +use litellm_core::integrations::custom_logger::CustomLogger; +use litellm_core::integrations::types::RequestMetadata; /// Process-local monotonic counter, mixed into the per-session call id so two /// sessions opened in the same nanosecond still get distinct ids. diff --git a/litellm-rust/crates/ai-gateway/src/state.rs b/litellm-rust/crates/ai-gateway/src/state.rs index 3b61d8309ea..34b4534fc74 100644 --- a/litellm-rust/crates/ai-gateway/src/state.rs +++ b/litellm-rust/crates/ai-gateway/src/state.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use crate::io::realtime_pool::RealtimePool; use litellm_core::router::Router; -use crate::integrations::custom_logger::CustomLogger; +use litellm_core::integrations::custom_logger::CustomLogger; /// Shared application state handed to every route handler. #[derive(Clone)] diff --git a/litellm-rust/crates/core/Cargo.toml b/litellm-rust/crates/core/Cargo.toml index 9bd4634cc2a..2e857806390 100644 --- a/litellm-rust/crates/core/Cargo.toml +++ b/litellm-rust/crates/core/Cargo.toml @@ -7,9 +7,11 @@ repository.workspace = true [dependencies] rand.workspace = true +reqwest.workspace = true serde.workspace = true serde_json.workspace = true thiserror.workspace = true +tokio = { workspace = true, features = ["sync", "time"] } [dev-dependencies] tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/litellm-rust/crates/ai-gateway/src/integrations/README.md b/litellm-rust/crates/core/src/integrations/README.md similarity index 95% rename from litellm-rust/crates/ai-gateway/src/integrations/README.md rename to litellm-rust/crates/core/src/integrations/README.md index 16a162dac57..a0c2208b745 100644 --- a/litellm-rust/crates/ai-gateway/src/integrations/README.md +++ b/litellm-rust/crates/core/src/integrations/README.md @@ -26,7 +26,7 @@ Implement `CustomLogger` when Rust code needs to observe terminal success or failure events. Method names intentionally match Python `CustomLogger` names. ```rust -use litellm_ai_gateway::integrations::custom_logger::{ +use litellm_core::integrations::custom_logger::{ CallbackTiming, CallbackValue, CustomLogger, LogFuture, ModelCallDetails, }; @@ -80,7 +80,7 @@ during-call checks. Method names intentionally match Python `CustomGuardrail` entrypoints inherited from Python `CustomLogger`. ```rust -use litellm_ai_gateway::integrations::custom_guardrail::{ +use litellm_core::integrations::custom_guardrail::{ CustomGuardrail, GuardrailContext, GuardrailDecision, GuardrailEventHook, GuardrailFuture, GuardrailRequest, }; @@ -104,7 +104,7 @@ impl CustomGuardrail for BlocklistedPromptGuardrail { Box::pin(async move { if request.data.to_string().contains("blocked phrase") { return Ok(GuardrailDecision::Block( - litellm_ai_gateway::integrations::custom_guardrail::GuardrailError::blocked( + litellm_core::integrations::custom_guardrail::GuardrailError::blocked( "blocked phrase detected", ), )); diff --git a/litellm-rust/crates/ai-gateway/src/integrations/custom_guardrail/mod.rs b/litellm-rust/crates/core/src/integrations/custom_guardrail/mod.rs similarity index 100% rename from litellm-rust/crates/ai-gateway/src/integrations/custom_guardrail/mod.rs rename to litellm-rust/crates/core/src/integrations/custom_guardrail/mod.rs diff --git a/litellm-rust/crates/ai-gateway/src/integrations/custom_guardrail/types.rs b/litellm-rust/crates/core/src/integrations/custom_guardrail/types.rs similarity index 100% rename from litellm-rust/crates/ai-gateway/src/integrations/custom_guardrail/types.rs rename to litellm-rust/crates/core/src/integrations/custom_guardrail/types.rs diff --git a/litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs b/litellm-rust/crates/core/src/integrations/custom_logger/mod.rs similarity index 100% rename from litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs rename to litellm-rust/crates/core/src/integrations/custom_logger/mod.rs diff --git a/litellm-rust/crates/ai-gateway/src/integrations/custom_logger/types.rs b/litellm-rust/crates/core/src/integrations/custom_logger/types.rs similarity index 100% rename from litellm-rust/crates/ai-gateway/src/integrations/custom_logger/types.rs rename to litellm-rust/crates/core/src/integrations/custom_logger/types.rs diff --git a/litellm-rust/crates/ai-gateway/src/integrations/litellm_python_proxy_api/mod.rs b/litellm-rust/crates/core/src/integrations/litellm_python_proxy_api/mod.rs similarity index 94% rename from litellm-rust/crates/ai-gateway/src/integrations/litellm_python_proxy_api/mod.rs rename to litellm-rust/crates/core/src/integrations/litellm_python_proxy_api/mod.rs index 3dad18cb7a3..20115f476dd 100644 --- a/litellm-rust/crates/ai-gateway/src/integrations/litellm_python_proxy_api/mod.rs +++ b/litellm-rust/crates/core/src/integrations/litellm_python_proxy_api/mod.rs @@ -16,7 +16,6 @@ use reqwest::Client; use tokio::sync::mpsc::{self, Receiver, Sender}; use tokio::time::interval; -use crate::constants::{DEFAULT_PROXY_BASE_URL, RUST_CONTROL_PLANE_LOGS_PATH}; use crate::integrations::custom_logger::{ CallbackTiming, CallbackValue, CustomLogger, LogError, LogFuture, LoggingError, ModelCallDetails, @@ -25,6 +24,14 @@ use types::{CallbackLogsRequest, EgressTunables, LogRecord}; pub mod types; +/// Default LiteLLM control-plane base URL for request-log egress when +/// `LITELLM_PROXY_BASE_URL` is unset. +const DEFAULT_PROXY_BASE_URL: &str = "http://localhost:4000"; + +/// The logs ingest path appended to the proxy base. Not a tunable; it is the +/// proxy's API contract (the rust-control-plane router on the Python proxy). +const RUST_CONTROL_PLANE_LOGS_PATH: &str = "/v1/rust_control_plane/logs"; + /// Ships realtime logging events to the LiteLLM Python proxy. pub struct LiteLLMPythonProxyAPILogger { sink: Sender, diff --git a/litellm-rust/crates/ai-gateway/src/integrations/litellm_python_proxy_api/types.rs b/litellm-rust/crates/core/src/integrations/litellm_python_proxy_api/types.rs similarity index 79% rename from litellm-rust/crates/ai-gateway/src/integrations/litellm_python_proxy_api/types.rs rename to litellm-rust/crates/core/src/integrations/litellm_python_proxy_api/types.rs index 481a437747f..cd194517046 100644 --- a/litellm-rust/crates/ai-gateway/src/integrations/litellm_python_proxy_api/types.rs +++ b/litellm-rust/crates/core/src/integrations/litellm_python_proxy_api/types.rs @@ -2,11 +2,20 @@ use std::time::Duration; use serde::Serialize; -use crate::constants::{ - DEFAULT_CHANNEL_CAPACITY, DEFAULT_FLUSH_INTERVAL_MS, DEFAULT_MAX_BATCH_SIZE, -}; use crate::integrations::types::StandardLoggingPayload; +/// Default bounded channel depth for the log-egress worker. +/// Override: `LITELLM_LOG_CHANNEL_CAPACITY`. +const DEFAULT_CHANNEL_CAPACITY: usize = 4096; + +/// Default max records POSTed per request to the control plane. +/// Override: `LITELLM_LOG_BATCH_SIZE`. +const DEFAULT_MAX_BATCH_SIZE: usize = 256; + +/// Default partial-batch flush cadence, in ms. +/// Override: `LITELLM_LOG_FLUSH_INTERVAL_MS`. +const DEFAULT_FLUSH_INTERVAL_MS: u64 = 500; + #[derive(Serialize)] pub struct CallbackLogsRequest { pub records: Vec, diff --git a/litellm-rust/crates/ai-gateway/src/integrations/mod.rs b/litellm-rust/crates/core/src/integrations/mod.rs similarity index 100% rename from litellm-rust/crates/ai-gateway/src/integrations/mod.rs rename to litellm-rust/crates/core/src/integrations/mod.rs diff --git a/litellm-rust/crates/ai-gateway/src/integrations/types.rs b/litellm-rust/crates/core/src/integrations/types.rs similarity index 100% rename from litellm-rust/crates/ai-gateway/src/integrations/types.rs rename to litellm-rust/crates/core/src/integrations/types.rs diff --git a/litellm-rust/crates/core/src/lib.rs b/litellm-rust/crates/core/src/lib.rs index 555a04ce853..46f29122e5a 100644 --- a/litellm-rust/crates/core/src/lib.rs +++ b/litellm-rust/crates/core/src/lib.rs @@ -1,5 +1,6 @@ pub mod call_lifecycle; pub mod error; +pub mod integrations; pub mod ocr; pub mod providers; pub mod realtime; diff --git a/litellm-rust/crates/python-bridge/src/integrations_bridge/custom_guardrail_bridge.rs b/litellm-rust/crates/python-bridge/src/integrations_bridge/custom_guardrail_bridge.rs index 81f251cf933..1864d159d7b 100644 --- a/litellm-rust/crates/python-bridge/src/integrations_bridge/custom_guardrail_bridge.rs +++ b/litellm-rust/crates/python-bridge/src/integrations_bridge/custom_guardrail_bridge.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use litellm_ai_gateway::integrations::custom_guardrail::{ +use litellm_core::integrations::custom_guardrail::{ CustomGuardrail, GuardrailContext, GuardrailDecision, GuardrailError, GuardrailEventHook, GuardrailFuture, GuardrailRequest, }; diff --git a/litellm-rust/crates/python-bridge/src/integrations_bridge/custom_logger_bridge.rs b/litellm-rust/crates/python-bridge/src/integrations_bridge/custom_logger_bridge.rs index 15956310eda..714c4c802f5 100644 --- a/litellm-rust/crates/python-bridge/src/integrations_bridge/custom_logger_bridge.rs +++ b/litellm-rust/crates/python-bridge/src/integrations_bridge/custom_logger_bridge.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use litellm_ai_gateway::integrations::custom_logger::{ +use litellm_core::integrations::custom_logger::{ CallbackTiming, CallbackValue, CustomLogger, LogError, LogFuture, ModelCallDetails, }; use pyo3::prelude::*;