mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
fix(messages): send bases the native parser misreads through python's client on rust
A Messages api_base with credentials, no host or a port Python cannot read now goes through Python's client, as bases without an http(s) scheme or with whitespace already did. Before, reqwest read such a URL differently from httpx: for http:///d it sent the request, with the operator key, to host `d`. This is a security fix for both azure_ai and anthropic; the anthropic_base_empty_host parity case covers anthropic. For azure_ai the choice reads the base Python's URL builder reads, falling back to AZURE_API_BASE, so a padded environment base also goes through Python's client. Python reads AZURE_API_BASE once: transport_handler returns the value it read and the bridge seeds the call's secret source with it, so the native route reuses it rather than reading the secret again. _sent_url only rewrites anthropic URLs, since the azure_ai builder no longer trims. Parity: 20 azure_ai and anthropic URL cases added; 65 aspects failed on the previous extension, and the full messages parity suite now passes.
This commit is contained in:
parent
398f8ac165
commit
d1efdd2683
8 changed files with 341 additions and 30 deletions
|
|
@ -36,7 +36,7 @@ use serde_json::{Map, Value};
|
||||||
|
|
||||||
use super::agentic::HookedRequest;
|
use super::agentic::HookedRequest;
|
||||||
use super::safeguards::{SafeguardsCall, SafeguardsHost};
|
use super::safeguards::{SafeguardsCall, SafeguardsHost};
|
||||||
use super::transport::{HostedTransport, TransportCall, python_failure};
|
use super::transport::{HostedTransport, Selected, TransportCall, python_failure};
|
||||||
use crate::{
|
use crate::{
|
||||||
errors::{RequestMarker, route_failure},
|
errors::{RequestMarker, route_failure},
|
||||||
marshal::{
|
marshal::{
|
||||||
|
|
@ -236,7 +236,7 @@ fn azure_module_settings<'py>(
|
||||||
pub(super) struct MessagesHosted;
|
pub(super) struct MessagesHosted;
|
||||||
|
|
||||||
impl litellm_host::protocol::Protocol for MessagesHosted {
|
impl litellm_host::protocol::Protocol for MessagesHosted {
|
||||||
type Request = (MessagesCall, Selection, bool);
|
type Request = (MessagesCall, Selection, Selected);
|
||||||
type Response = <Messages as litellm_host::protocol::Protocol>::Response;
|
type Response = <Messages as litellm_host::protocol::Protocol>::Response;
|
||||||
type Error = Error;
|
type Error = Error;
|
||||||
type HostCall = MessagesHostCall;
|
type HostCall = MessagesHostCall;
|
||||||
|
|
@ -755,7 +755,7 @@ impl PythonBinding for MessagesPythonHost {
|
||||||
&mut self,
|
&mut self,
|
||||||
py: Python<'_>,
|
py: Python<'_>,
|
||||||
arguments: &Bound<'_, PyDict>,
|
arguments: &Bound<'_, PyDict>,
|
||||||
) -> Result<(MessagesCall, Selection, bool), InvokeError<Error>> {
|
) -> Result<(MessagesCall, Selection, Selected), InvokeError<Error>> {
|
||||||
self.view
|
self.view
|
||||||
.resolve(py, arguments)
|
.resolve(py, arguments)
|
||||||
.map_err(InvokeError::Python)?;
|
.map_err(InvokeError::Python)?;
|
||||||
|
|
@ -781,11 +781,11 @@ impl PythonBinding for MessagesPythonHost {
|
||||||
.is_some_and(|stream| stream.is(pyo3::types::PyBool::new(py, true)));
|
.is_some_and(|stream| stream.is(pyo3::types::PyBool::new(py, true)));
|
||||||
self.arguments = Some(arguments.clone().unbind());
|
self.arguments = Some(arguments.clone().unbind());
|
||||||
let request = self.decode_call(py, arguments)?;
|
let request = self.decode_call(py, arguments)?;
|
||||||
let hosted = self
|
let transport = self
|
||||||
.transport
|
.transport
|
||||||
.select(py, self.view.request(py).map_err(InvokeError::Python)?)
|
.select(py, self.view.request(py).map_err(InvokeError::Python)?)
|
||||||
.map_err(InvokeError::Python)?;
|
.map_err(InvokeError::Python)?;
|
||||||
Ok((request, selection, hosted))
|
Ok((request, selection, transport))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn encode_response(
|
fn encode_response(
|
||||||
|
|
|
||||||
|
|
@ -33,11 +33,14 @@ fn run_messages(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> PyR
|
||||||
move |py, arguments, request| {
|
move |py, arguments, request| {
|
||||||
let hooked = pre_request::runs_pre_request_hooks(py)?;
|
let hooked = pre_request::runs_pre_request_hooks(py)?;
|
||||||
let context = litellm_host_python::PythonContext::capture(py)?;
|
let context = litellm_host_python::PythonContext::capture(py)?;
|
||||||
|
let secrets = std::sync::Arc::new(crate::secrets::seeded::SeededSecrets::new(
|
||||||
|
crate::secrets::source(py)?,
|
||||||
|
));
|
||||||
let route = litellm_inference_messages::MessagesRoute::new(
|
let route = litellm_inference_messages::MessagesRoute::new(
|
||||||
crate::http::messages_client(py, arguments, asynchronous)?
|
crate::http::messages_client(py, arguments, asynchronous)?
|
||||||
.map_err(crate::http::client_error)?,
|
.map_err(crate::http::client_error)?,
|
||||||
crate::http::resources().auth.clone(),
|
crate::http::resources().auth.clone(),
|
||||||
crate::secrets::source(py)?,
|
std::sync::Arc::clone(&secrets) as _,
|
||||||
)
|
)
|
||||||
.with_workload_identity(std::sync::Arc::new(
|
.with_workload_identity(std::sync::Arc::new(
|
||||||
workload_identity::PythonWorkloadIdentity::new(context.clone()),
|
workload_identity::PythonWorkloadIdentity::new(context.clone()),
|
||||||
|
|
@ -57,11 +60,16 @@ fn run_messages(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> PyR
|
||||||
Ok(litellm_host::call::hosted_call(
|
Ok(litellm_host::call::hosted_call(
|
||||||
request,
|
request,
|
||||||
None,
|
None,
|
||||||
move |(call, selection, hosted): (_, crate::cache::Selection, bool),
|
move |(call, selection, transport): (
|
||||||
|
_,
|
||||||
|
crate::cache::Selection,
|
||||||
|
transport::Selected,
|
||||||
|
),
|
||||||
services,
|
services,
|
||||||
interceptors,
|
interceptors,
|
||||||
observers| async move {
|
observers| async move {
|
||||||
let route = if hosted {
|
secrets.seed(transport.read);
|
||||||
|
let route = if transport.hosted {
|
||||||
route.with_transport(std::sync::Arc::new(transport::PythonTransport::new(
|
route.with_transport(std::sync::Arc::new(transport::PythonTransport::new(
|
||||||
services.clone(),
|
services.clone(),
|
||||||
)))
|
)))
|
||||||
|
|
|
||||||
|
|
@ -124,6 +124,11 @@ enum Pending {
|
||||||
Read(Reply<Result<Option<Bytes>, Error>>),
|
Read(Reply<Result<Option<Bytes>, Error>>),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(super) struct Selected {
|
||||||
|
pub(super) hosted: bool,
|
||||||
|
pub(super) read: Vec<(String, Option<String>)>,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
pub(super) struct HostedTransport {
|
pub(super) struct HostedTransport {
|
||||||
handler: Option<Py<PyAny>>,
|
handler: Option<Py<PyAny>>,
|
||||||
|
|
@ -132,13 +137,23 @@ pub(super) struct HostedTransport {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl HostedTransport {
|
impl HostedTransport {
|
||||||
pub(super) fn select(&mut self, py: Python<'_>, request: &Bound<'_, PyDict>) -> PyResult<bool> {
|
/// Whether the call sends through Python's client, and the secrets the host read to decide it,
|
||||||
let handler = py
|
/// which the native route reuses rather than reading again.
|
||||||
|
pub(super) fn select(
|
||||||
|
&mut self,
|
||||||
|
py: Python<'_>,
|
||||||
|
request: &Bound<'_, PyDict>,
|
||||||
|
) -> PyResult<Selected> {
|
||||||
|
let (handler, read): (Bound<'_, PyAny>, Vec<(String, Option<String>)>) = py
|
||||||
.import(ROUTE_HOST_MODULE)?
|
.import(ROUTE_HOST_MODULE)?
|
||||||
.getattr("transport_handler")?
|
.getattr("transport_handler")?
|
||||||
.call1((request,))?;
|
.call1((request,))?
|
||||||
|
.extract()?;
|
||||||
self.handler = (!handler.is_none()).then(|| handler.unbind());
|
self.handler = (!handler.is_none()).then(|| handler.unbind());
|
||||||
Ok(self.handler.is_some())
|
Ok(Selected {
|
||||||
|
hosted: self.handler.is_some(),
|
||||||
|
read,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) fn pending(&self) -> bool {
|
pub(super) fn pending(&self) -> bool {
|
||||||
|
|
|
||||||
|
|
@ -7,6 +7,7 @@ mod provider;
|
||||||
mod python;
|
mod python;
|
||||||
pub(crate) mod resolved;
|
pub(crate) mod resolved;
|
||||||
pub(crate) mod runtime;
|
pub(crate) mod runtime;
|
||||||
|
pub(crate) mod seeded;
|
||||||
mod vault;
|
mod vault;
|
||||||
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
|
||||||
101
litellm/rust/crates/python-bridge/src/secrets/seeded.rs
Normal file
101
litellm/rust/crates/python-bridge/src/secrets/seeded.rs
Normal file
|
|
@ -0,0 +1,101 @@
|
||||||
|
use std::{
|
||||||
|
collections::HashMap,
|
||||||
|
sync::{Arc, Mutex, PoisonError},
|
||||||
|
};
|
||||||
|
|
||||||
|
use futures_util::future::{BoxFuture, FutureExt};
|
||||||
|
use litellm_secret_value::SecretValue;
|
||||||
|
use litellm_secrets::{Error, source::SecretSource};
|
||||||
|
|
||||||
|
/// A call's secret source that answers the secrets the host already read for the call with the
|
||||||
|
/// values it read, so a secret Python reads once is not read a second time.
|
||||||
|
pub(crate) struct SeededSecrets {
|
||||||
|
source: Arc<dyn SecretSource>,
|
||||||
|
read: Mutex<HashMap<String, Option<SecretValue>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SeededSecrets {
|
||||||
|
pub(crate) fn new(source: Arc<dyn SecretSource>) -> Self {
|
||||||
|
Self {
|
||||||
|
source,
|
||||||
|
read: Mutex::default(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn seed(&self, read: impl IntoIterator<Item = (String, Option<String>)>) {
|
||||||
|
self.read
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(PoisonError::into_inner)
|
||||||
|
.extend(
|
||||||
|
read.into_iter()
|
||||||
|
.map(|(name, value)| (name, value.map(SecretValue::new))),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SecretSource for SeededSecrets {
|
||||||
|
fn get_secret_str<'a>(
|
||||||
|
&'a self,
|
||||||
|
name: &'a str,
|
||||||
|
) -> BoxFuture<'a, Result<Option<SecretValue>, Error>> {
|
||||||
|
let seeded = self
|
||||||
|
.read
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(PoisonError::into_inner)
|
||||||
|
.get(name)
|
||||||
|
.cloned();
|
||||||
|
match seeded {
|
||||||
|
Some(value) => futures_util::future::ready(Ok(value)).boxed(),
|
||||||
|
None => self.source.get_secret_str(name),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct Counting(AtomicUsize);
|
||||||
|
|
||||||
|
impl SecretSource for Counting {
|
||||||
|
fn get_secret_str<'a>(
|
||||||
|
&'a self,
|
||||||
|
name: &'a str,
|
||||||
|
) -> BoxFuture<'a, Result<Option<SecretValue>, Error>> {
|
||||||
|
self.0.fetch_add(1, Ordering::SeqCst);
|
||||||
|
futures_util::future::ready(Ok(Some(SecretValue::new(format!("source-{name}")))))
|
||||||
|
.boxed()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_seeded_secret_is_answered_without_reading_the_source() {
|
||||||
|
let source = Arc::new(Counting::default());
|
||||||
|
let secrets = SeededSecrets::new(Arc::clone(&source) as Arc<dyn SecretSource>);
|
||||||
|
secrets.seed([
|
||||||
|
(
|
||||||
|
"AZURE_API_BASE".to_owned(),
|
||||||
|
Some("http://seeded".to_owned()),
|
||||||
|
),
|
||||||
|
("UNSET".to_owned(), None),
|
||||||
|
]);
|
||||||
|
|
||||||
|
let seeded = secrets.get_secret_str("AZURE_API_BASE").await.unwrap();
|
||||||
|
let unset = secrets.get_secret_str("UNSET").await.unwrap();
|
||||||
|
let other = secrets.get_secret_str("OTHER").await.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
seeded.map(|value| value.expose().to_owned()).as_deref(),
|
||||||
|
Some("http://seeded")
|
||||||
|
);
|
||||||
|
assert!(unset.is_none());
|
||||||
|
assert_eq!(
|
||||||
|
other.map(|value| value.expose().to_owned()).as_deref(),
|
||||||
|
Some("source-OTHER")
|
||||||
|
);
|
||||||
|
assert_eq!(source.0.load(Ordering::SeqCst), 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -12,6 +12,7 @@ import socket
|
||||||
from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Iterable, Iterator, Mapping, Sequence
|
from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Iterable, Iterator, Mapping, Sequence
|
||||||
from dataclasses import asdict, dataclass, replace
|
from dataclasses import asdict, dataclass, replace
|
||||||
from typing import Final, TypeAlias, cast # noqa: TID251 # narrows native payloads to public TypedDicts
|
from typing import Final, TypeAlias, cast # noqa: TID251 # narrows native payloads to public TypedDicts
|
||||||
|
from urllib.parse import SplitResult, urlsplit
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
from aiohttp.http_exceptions import ContentLengthError, TransferEncodingError
|
from aiohttp.http_exceptions import ContentLengthError, TransferEncodingError
|
||||||
|
|
@ -91,6 +92,7 @@ from litellm.rust_bridge.execution import AsyncStream
|
||||||
from litellm.rust_bridge.host import cache, callbacks_legacy_python, http_transport
|
from litellm.rust_bridge.host import cache, callbacks_legacy_python, http_transport
|
||||||
from litellm.rust_bridge.messages.entrypoints import MessagesStreamHead
|
from litellm.rust_bridge.messages.entrypoints import MessagesStreamHead
|
||||||
from litellm.rust_bridge.public_call import NativeCall, known_params, optional_sequence, optional_str
|
from litellm.rust_bridge.public_call import NativeCall, known_params, optional_sequence, optional_str
|
||||||
|
from litellm.secret_managers.main import get_secret_str
|
||||||
from litellm.types.integrations.custom_logger import WEBSEARCH_CONVERTED_STREAM_KEY
|
from litellm.types.integrations.custom_logger import WEBSEARCH_CONVERTED_STREAM_KEY
|
||||||
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse
|
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse
|
||||||
from litellm.types.router import GenericLiteLLMParams
|
from litellm.types.router import GenericLiteLLMParams
|
||||||
|
|
@ -1218,8 +1220,9 @@ class Sent:
|
||||||
|
|
||||||
def _unsendable_api_base(api_base: object) -> bool:
|
def _unsendable_api_base(api_base: object) -> bool:
|
||||||
"""A caller api_base the native URL parser rejects or reads otherwise than httpx and aiohttp
|
"""A caller api_base the native URL parser rejects or reads otherwise than httpx and aiohttp
|
||||||
(no http(s) scheme, whitespace, control or non-ASCII characters): Python's client decides what
|
(no http(s) scheme, whitespace, control or non-ASCII characters, credentials, no host or a port
|
||||||
such a URL does, from a relative-URL error to a request on the host it finds."""
|
Python cannot read): Python's client decides what such a URL does, from a relative-URL error to
|
||||||
|
a request on the host it finds."""
|
||||||
if not isinstance(api_base, str) or not api_base:
|
if not isinstance(api_base, str) or not api_base:
|
||||||
return False
|
return False
|
||||||
return not (
|
return not (
|
||||||
|
|
@ -1227,18 +1230,50 @@ def _unsendable_api_base(api_base: object) -> bool:
|
||||||
and api_base.isprintable()
|
and api_base.isprintable()
|
||||||
and " " not in api_base
|
and " " not in api_base
|
||||||
and api_base.lower().startswith(("http://", "https://"))
|
and api_base.lower().startswith(("http://", "https://"))
|
||||||
|
and _plain_authority(api_base)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def transport_handler(request: Mapping[str, object]) -> AsyncHTTPHandler | None:
|
def _plain_authority(api_base: str) -> bool:
|
||||||
|
try:
|
||||||
|
parts: Final = urlsplit(api_base)
|
||||||
|
except ValueError:
|
||||||
|
return True
|
||||||
|
return bool(parts.hostname) and parts.username is None and _readable_port(parts)
|
||||||
|
|
||||||
|
|
||||||
|
def _readable_port(parts: SplitResult) -> bool:
|
||||||
|
try:
|
||||||
|
return parts.port is None or parts.port >= 0
|
||||||
|
except ValueError:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
_SecretsRead: TypeAlias = list[tuple[str, str | None]]
|
||||||
|
|
||||||
|
|
||||||
|
def _sent_api_base(request: Mapping[str, object]) -> tuple[object, _SecretsRead]:
|
||||||
|
"""The base Python's URL builder reads, and the secrets read to find it: azure_ai falls back to
|
||||||
|
`AZURE_API_BASE`, read once, which the native route then reuses."""
|
||||||
|
api_base: Final = request.get("api_base")
|
||||||
|
if api_base or "model" not in request or _resolved_provider(request) != "azure_ai":
|
||||||
|
return api_base, []
|
||||||
|
environment: Final = get_secret_str("AZURE_API_BASE")
|
||||||
|
return environment, [("AZURE_API_BASE", environment)]
|
||||||
|
|
||||||
|
|
||||||
|
def transport_handler(request: Mapping[str, object]) -> tuple[AsyncHTTPHandler | None, _SecretsRead]:
|
||||||
|
"""The client the call sends through (None for the native one), and the secrets read to choose
|
||||||
|
it, which the native route reuses rather than reading them again."""
|
||||||
client: Final = request.get("client")
|
client: Final = request.get("client")
|
||||||
if isinstance(client, AsyncHTTPHandler):
|
if isinstance(client, AsyncHTTPHandler):
|
||||||
return client
|
return client, []
|
||||||
if not http_transport.sends_through_python(client, request.get("timeout")) and not _unsendable_api_base(
|
if http_transport.sends_through_python(client, request.get("timeout")):
|
||||||
request.get("api_base")
|
return get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC), []
|
||||||
):
|
api_base, read = _sent_api_base(request)
|
||||||
return None
|
if not _unsendable_api_base(api_base):
|
||||||
return get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC)
|
return None, read
|
||||||
|
return get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC), read
|
||||||
|
|
||||||
|
|
||||||
def decoding_failure(message: str, request: Mapping[str, object], request_provider: str) -> Exception:
|
def decoding_failure(message: str, request: Mapping[str, object], request_provider: str) -> Exception:
|
||||||
|
|
@ -1260,7 +1295,12 @@ def unencodable_header(value: bytes, received: bool) -> Exception:
|
||||||
def _sent_url(request: Mapping[str, object], url: str) -> str:
|
def _sent_url(request: Mapping[str, object], url: str) -> str:
|
||||||
"""The native URL builder trims the caller's api_base; Python sends it as given."""
|
"""The native URL builder trims the caller's api_base; Python sends it as given."""
|
||||||
api_base: Final = request.get("api_base")
|
api_base: Final = request.get("api_base")
|
||||||
if not isinstance(api_base, str) or api_base == api_base.strip() or not _unsendable_api_base(api_base):
|
if (
|
||||||
|
not isinstance(api_base, str)
|
||||||
|
or api_base == api_base.strip()
|
||||||
|
or not _unsendable_api_base(api_base)
|
||||||
|
or _resolved_provider(request) != "anthropic"
|
||||||
|
):
|
||||||
return url
|
return url
|
||||||
try:
|
try:
|
||||||
return _python_messages_url(api_base) if url == _python_messages_url(api_base.strip()) else url
|
return _python_messages_url(api_base) if url == _python_messages_url(api_base.strip()) else url
|
||||||
|
|
|
||||||
|
|
@ -3509,6 +3509,56 @@ AZURE_AUTH_CASES: Final = tuple(
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
AZURE_URL_CASES: Final = tuple(
|
||||||
|
replace(case, model=AZURE_CLAUDE, rust_providers=AZURE_AI)
|
||||||
|
for case in (
|
||||||
|
Case(
|
||||||
|
"azure_ai_base_userinfo_starting_with_anthropic",
|
||||||
|
_kw(api_base_userinfo="anthropic", api_base_suffix="/d/dep-u01/anthropic"),
|
||||||
|
),
|
||||||
|
Case("azure_ai_base_slash_before_query", _kw(api_base_suffix="/d/dep-u02/?q=1")),
|
||||||
|
Case("azure_ai_base_trailing_spaces_without_anthropic", _kw(api_base_suffix="/d/dep-u04 ")),
|
||||||
|
Case("azure_ai_base_trailing_newline_without_anthropic", _kw(api_base_suffix="/d/dep-u04\n"), upstream=()),
|
||||||
|
Case("azure_ai_base_trailing_spaces_after_anthropic", _kw(api_base_suffix="/d/dep-u04/anthropic ")),
|
||||||
|
Case("azure_ai_base_trailing_crlf_after_anthropic", _kw(api_base_suffix="/d/dep-u04/anthropic\r\n")),
|
||||||
|
Case("azure_ai_blank_base", _kw(api_base=" "), upstream=()),
|
||||||
|
Case(
|
||||||
|
"azure_ai_blank_base_with_environment_base",
|
||||||
|
_kw(api_base=" "),
|
||||||
|
env=MappingProxyType({"AZURE_API_BASE": "http://127.0.0.1:9/d/dep-env/anthropic"}),
|
||||||
|
upstream=(),
|
||||||
|
),
|
||||||
|
Case(
|
||||||
|
"azure_ai_padded_environment_base",
|
||||||
|
_kw(api_base=None),
|
||||||
|
env=MappingProxyType({"AZURE_API_BASE": " http://127.0.0.1:9/d/dep-env/anthropic "}),
|
||||||
|
upstream=(),
|
||||||
|
),
|
||||||
|
Case("azure_ai_missing_base", _kw(api_base=None), upstream=()),
|
||||||
|
Case("azure_ai_missing_base_stream", _kw(api_base=None, stream=True), upstream=()),
|
||||||
|
Case("azure_ai_empty_base", _kw(api_base=""), upstream=()),
|
||||||
|
Case("azure_ai_base_without_scheme", _kw(api_base="fuzz.invalid/d/dep-e04/anthropic"), upstream=()),
|
||||||
|
Case("azure_ai_scheme_relative_base", _kw(api_base="//fuzz.invalid/d/dep-e04/anthropic"), upstream=()),
|
||||||
|
Case("azure_ai_ftp_base", _kw(api_base="ftp://fuzz.invalid/d/dep-e04/anthropic"), upstream=()),
|
||||||
|
Case("azure_ai_base_not_a_url", _kw(api_base="not a url"), upstream=()),
|
||||||
|
Case(
|
||||||
|
"azure_ai_base_non_numeric_port",
|
||||||
|
_kw(api_base="http://fuzz.invalid:notaport/d/dep-e04/anthropic"),
|
||||||
|
upstream=(),
|
||||||
|
),
|
||||||
|
Case(
|
||||||
|
"azure_ai_base_out_of_range_port",
|
||||||
|
_kw(api_base="http://fuzz.invalid:99999/d/dep-e04/anthropic"),
|
||||||
|
upstream=(),
|
||||||
|
),
|
||||||
|
Case("azure_ai_base_empty_host", _kw(api_base="http:///d/dep-e04/anthropic"), upstream=()),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
# An anthropic base with no host goes through Python's client, so the key is never sent to the host
|
||||||
|
# the native parser read from the path (`d`).
|
||||||
|
ANTHROPIC_URL_CASES: Final = (Case("anthropic_base_empty_host", _kw(api_base="http:///d"), upstream=()),)
|
||||||
|
|
||||||
CASES: Final = (
|
CASES: Final = (
|
||||||
*REQUEST_CASES,
|
*REQUEST_CASES,
|
||||||
*INPUT_VALUE_CASES,
|
*INPUT_VALUE_CASES,
|
||||||
|
|
@ -3535,6 +3585,8 @@ CASES: Final = (
|
||||||
*DEPLOYMENT_HOOK_CASES,
|
*DEPLOYMENT_HOOK_CASES,
|
||||||
*HOOKED_STREAM_AND_CACHE_CASES,
|
*HOOKED_STREAM_AND_CACHE_CASES,
|
||||||
*AZURE_AUTH_CASES,
|
*AZURE_AUTH_CASES,
|
||||||
|
*AZURE_URL_CASES,
|
||||||
|
*ANTHROPIC_URL_CASES,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -3693,12 +3745,14 @@ def _kwargs(case: Case, server: RecordingServer) -> dict[str, object]:
|
||||||
# other leg.
|
# other leg.
|
||||||
supplied: Final = copy.deepcopy(dict(case.kwargs))
|
supplied: Final = copy.deepcopy(dict(case.kwargs))
|
||||||
suffix: Final = supplied.pop("api_base_suffix", "")
|
suffix: Final = supplied.pop("api_base_suffix", "")
|
||||||
|
userinfo: Final = supplied.pop("api_base_userinfo", "")
|
||||||
|
base: Final = server.base_url.replace("://", f"://{userinfo}@", 1) if userinfo else server.base_url
|
||||||
kwargs: Final[dict[str, object]] = {
|
kwargs: Final[dict[str, object]] = {
|
||||||
"model": case.model,
|
"model": case.model,
|
||||||
"max_tokens": 64,
|
"max_tokens": 64,
|
||||||
"messages": [dict(USER)],
|
"messages": [dict(USER)],
|
||||||
"api_key": API_KEY,
|
"api_key": API_KEY,
|
||||||
"api_base": f"{server.base_url}{suffix}",
|
"api_base": f"{base}{suffix}",
|
||||||
**supplied,
|
**supplied,
|
||||||
}
|
}
|
||||||
return {name: value for name, value in kwargs.items() if value is not None or name in ("safeguards", *case.nones)}
|
return {name: value for name, value in kwargs.items() if value is not None or name in ("safeguards", *case.nones)}
|
||||||
|
|
@ -3839,7 +3893,9 @@ async def _invoke(
|
||||||
if isinstance(result, dict):
|
if isinstance(result, dict):
|
||||||
body = cast(Mapping[str, object], result)
|
body = cast(Mapping[str, object], result)
|
||||||
return _response(body, case.private_keys), _hidden_params(body)
|
return _response(body, case.private_keys), _hidden_params(body)
|
||||||
time.sleep(case.busy_before_reading) # noqa: ASYNC251 # holds the event loop, as a caller's own work between the call and its first read does
|
time.sleep(
|
||||||
|
case.busy_before_reading
|
||||||
|
) # holds the event loop, as a caller's own work between the call and its first read does
|
||||||
chunks, failure = await _consume_async(cast(AsyncIterator[object], result))
|
chunks, failure = await _consume_async(cast(AsyncIterator[object], result))
|
||||||
hidden: Final = _hidden_params(result)
|
hidden: Final = _hidden_params(result)
|
||||||
partition: Final = {"partition": _partition(chunks)} if case.partitioned else {}
|
partition: Final = {"partition": _partition(chunks)} if case.partitioned else {}
|
||||||
|
|
|
||||||
|
|
@ -974,14 +974,14 @@ def no_tls_settings(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
"client", (None, object(), httpx.AsyncClient(), HTTPHandler()), ids=("none", "object", "httpx", "sync-handler")
|
"client", (None, object(), httpx.AsyncClient(), HTTPHandler()), ids=("none", "object", "httpx", "sync-handler")
|
||||||
)
|
)
|
||||||
def test_the_native_client_stands_in_for_any_client_python_ignores(client: object) -> None:
|
def test_the_native_client_stands_in_for_any_client_python_ignores(client: object) -> None:
|
||||||
assert messages_host.transport_handler({"client": client}) is None
|
assert messages_host.transport_handler({"client": client}) == (None, [])
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.usefixtures("no_tls_settings")
|
@pytest.mark.usefixtures("no_tls_settings")
|
||||||
def test_a_caller_async_handler_is_the_handler_python_posts_through() -> None:
|
def test_a_caller_async_handler_is_the_handler_python_posts_through() -> None:
|
||||||
client: Final = AsyncHTTPHandler()
|
client: Final = AsyncHTTPHandler()
|
||||||
|
|
||||||
assert messages_host.transport_handler({"client": client}) is client
|
assert messages_host.transport_handler({"client": client}) == (client, [])
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.usefixtures("no_tls_settings")
|
@pytest.mark.usefixtures("no_tls_settings")
|
||||||
|
|
@ -995,7 +995,7 @@ def test_tls_the_native_client_cannot_apply_posts_through_pythons_anthropic_hand
|
||||||
monkeypatch.setattr(litellm, name, value)
|
monkeypatch.setattr(litellm, name, value)
|
||||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||||
|
|
||||||
handler: Final = messages_host.transport_handler({"client": object()})
|
handler, _ = messages_host.transport_handler({"client": object()})
|
||||||
|
|
||||||
assert handler is get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC)
|
assert handler is get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC)
|
||||||
assert callbacks_legacy_python.times_provider_exchange(object(), None) is False
|
assert callbacks_legacy_python.times_provider_exchange(object(), None) is False
|
||||||
|
|
@ -1012,7 +1012,7 @@ def fips_cipher_exclusion() -> Iterator[None]:
|
||||||
def test_fips_cipher_exclusion_posts_through_pythons_anthropic_handler(monkeypatch: pytest.MonkeyPatch) -> None:
|
def test_fips_cipher_exclusion_posts_through_pythons_anthropic_handler(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||||
|
|
||||||
handler: Final = messages_host.transport_handler({"client": None})
|
handler, _ = messages_host.transport_handler({"client": None})
|
||||||
|
|
||||||
assert handler is get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC)
|
assert handler is get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC)
|
||||||
assert callbacks_legacy_python.times_provider_exchange(None, None) is False
|
assert callbacks_legacy_python.times_provider_exchange(None, None) is False
|
||||||
|
|
@ -1036,7 +1036,7 @@ def test_a_non_positive_timeout_posts_through_pythons_anthropic_handler(
|
||||||
) -> None:
|
) -> None:
|
||||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||||
|
|
||||||
handler: Final = messages_host.transport_handler({"client": None, "timeout": timeout})
|
handler, _ = messages_host.transport_handler({"client": None, "timeout": timeout})
|
||||||
|
|
||||||
assert handler is get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC)
|
assert handler is get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC)
|
||||||
assert callbacks_legacy_python.times_provider_exchange(None, timeout) is False
|
assert callbacks_legacy_python.times_provider_exchange(None, timeout) is False
|
||||||
|
|
@ -1045,7 +1045,97 @@ def test_a_non_positive_timeout_posts_through_pythons_anthropic_handler(
|
||||||
@pytest.mark.usefixtures("no_tls_settings")
|
@pytest.mark.usefixtures("no_tls_settings")
|
||||||
@pytest.mark.parametrize("timeout", (None, 0.5, httpx.Timeout(10.0)), ids=("unset", "positive", "httpx"))
|
@pytest.mark.parametrize("timeout", (None, 0.5, httpx.Timeout(10.0)), ids=("unset", "positive", "httpx"))
|
||||||
def test_a_positive_timeout_keeps_the_native_client(timeout: object) -> None:
|
def test_a_positive_timeout_keeps_the_native_client(timeout: object) -> None:
|
||||||
assert messages_host.transport_handler({"client": None, "timeout": timeout}) is None
|
assert messages_host.transport_handler({"client": None, "timeout": timeout}) == (None, [])
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.usefixtures("no_tls_settings")
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"api_base",
|
||||||
|
(
|
||||||
|
"http://user:secret@host.example/d",
|
||||||
|
"http://user@host.example/d",
|
||||||
|
"http:///d",
|
||||||
|
"http://:443/d",
|
||||||
|
"http://host.example:99999/d",
|
||||||
|
"http://host.example:port/d",
|
||||||
|
),
|
||||||
|
ids=("credentials", "user", "no-host", "port-only", "port-out-of-range", "port-not-a-number"),
|
||||||
|
)
|
||||||
|
@pytest.mark.parametrize("model", ("anthropic/claude-x", "azure_ai/claude-x"))
|
||||||
|
def test_a_base_the_native_parser_reads_otherwise_posts_through_pythons_handler(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, api_base: str, model: str
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||||
|
|
||||||
|
handler, read = messages_host.transport_handler({"client": None, "model": model, "api_base": api_base})
|
||||||
|
|
||||||
|
assert handler is get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC)
|
||||||
|
assert read == []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.usefixtures("no_tls_settings")
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"api_base",
|
||||||
|
("https://host.example/d", "http://host.example:8080/d?x=1", "https://[::1]:443/d"),
|
||||||
|
ids=("plain", "port-and-query", "ipv6"),
|
||||||
|
)
|
||||||
|
@pytest.mark.parametrize("model", ("anthropic/claude-x", "azure_ai/claude-x"))
|
||||||
|
def test_a_plain_base_keeps_the_native_client(api_base: str, model: str) -> None:
|
||||||
|
assert messages_host.transport_handler({"client": None, "model": model, "api_base": api_base}) == (None, [])
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.usefixtures("no_tls_settings")
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("environment", "hosted"),
|
||||||
|
((" https://env.example/d ", True), ("https://env.example/d", False), (None, False)),
|
||||||
|
ids=("padded", "plain", "unset"),
|
||||||
|
)
|
||||||
|
@pytest.mark.parametrize("api_base", (None, ""), ids=("absent", "empty"))
|
||||||
|
def test_azure_ai_reads_azure_api_base_once_and_hands_it_to_the_native_route(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, environment: str | None, hosted: bool, api_base: str | None
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||||
|
reads: Final[list[str]] = []
|
||||||
|
|
||||||
|
def get_secret_str(name: str) -> str | None:
|
||||||
|
reads.append(name)
|
||||||
|
return environment
|
||||||
|
|
||||||
|
monkeypatch.setattr(messages_host, "get_secret_str", get_secret_str)
|
||||||
|
|
||||||
|
handler, read = messages_host.transport_handler(
|
||||||
|
{"client": None, "model": "azure_ai/claude-x", "api_base": api_base}
|
||||||
|
)
|
||||||
|
|
||||||
|
assert (handler is not None) is hosted
|
||||||
|
assert read == [("AZURE_API_BASE", environment)]
|
||||||
|
assert reads == ["AZURE_API_BASE"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.usefixtures("no_tls_settings")
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"request_",
|
||||||
|
(
|
||||||
|
{"client": None, "model": "anthropic/claude-x"},
|
||||||
|
{"client": None, "model": "azure_ai/claude-x", "api_base": "https://call.example/d"},
|
||||||
|
{"client": AsyncHTTPHandler(), "model": "azure_ai/claude-x"},
|
||||||
|
{"client": None, "model": "azure_ai/claude-x", "timeout": 0},
|
||||||
|
),
|
||||||
|
ids=("anthropic", "azure-call-base", "azure-caller-handler", "azure-python-timeout"),
|
||||||
|
)
|
||||||
|
def test_azure_api_base_is_left_to_the_native_route_when_the_choice_does_not_need_it(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, request_: Mapping[str, object]
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||||
|
|
||||||
|
def get_secret_str(name: str) -> str | None:
|
||||||
|
raise AssertionError(name)
|
||||||
|
|
||||||
|
monkeypatch.setattr(messages_host, "get_secret_str", get_secret_str)
|
||||||
|
|
||||||
|
_, read = messages_host.transport_handler(request_)
|
||||||
|
|
||||||
|
assert read == []
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue