fix(messages): send bases the native parser misreads through python's client on rust

A Messages api_base with credentials, no host or a port Python cannot read
now goes through Python's client, as bases without an http(s) scheme or with
whitespace already did. Before, reqwest read such a URL differently from
httpx: for http:///d it sent the request, with the operator key, to host
`d`. This is a security fix for both azure_ai and anthropic; the
anthropic_base_empty_host parity case covers anthropic.

For azure_ai the choice reads the base Python's URL builder reads, falling
back to AZURE_API_BASE, so a padded environment base also goes through
Python's client. Python reads AZURE_API_BASE once: transport_handler returns
the value it read and the bridge seeds the call's secret source with it, so
the native route reuses it rather than reading the secret again. _sent_url
only rewrites anthropic URLs, since the azure_ai builder no longer trims.

Parity: 20 azure_ai and anthropic URL cases added; 65 aspects failed on the
previous extension, and the full messages parity suite now passes.
This commit is contained in:
Claude 2026-11-13 00:05:47 +00:00
parent 398f8ac165
commit d1efdd2683
No known key found for this signature in database
8 changed files with 341 additions and 30 deletions

View file

@ -36,7 +36,7 @@ use serde_json::{Map, Value};
use super::agentic::HookedRequest; use super::agentic::HookedRequest;
use super::safeguards::{SafeguardsCall, SafeguardsHost}; use super::safeguards::{SafeguardsCall, SafeguardsHost};
use super::transport::{HostedTransport, TransportCall, python_failure}; use super::transport::{HostedTransport, Selected, TransportCall, python_failure};
use crate::{ use crate::{
errors::{RequestMarker, route_failure}, errors::{RequestMarker, route_failure},
marshal::{ marshal::{
@ -236,7 +236,7 @@ fn azure_module_settings<'py>(
pub(super) struct MessagesHosted; pub(super) struct MessagesHosted;
impl litellm_host::protocol::Protocol for MessagesHosted { impl litellm_host::protocol::Protocol for MessagesHosted {
type Request = (MessagesCall, Selection, bool); type Request = (MessagesCall, Selection, Selected);
type Response = <Messages as litellm_host::protocol::Protocol>::Response; type Response = <Messages as litellm_host::protocol::Protocol>::Response;
type Error = Error; type Error = Error;
type HostCall = MessagesHostCall; type HostCall = MessagesHostCall;
@ -755,7 +755,7 @@ impl PythonBinding for MessagesPythonHost {
&mut self, &mut self,
py: Python<'_>, py: Python<'_>,
arguments: &Bound<'_, PyDict>, arguments: &Bound<'_, PyDict>,
) -> Result<(MessagesCall, Selection, bool), InvokeError<Error>> { ) -> Result<(MessagesCall, Selection, Selected), InvokeError<Error>> {
self.view self.view
.resolve(py, arguments) .resolve(py, arguments)
.map_err(InvokeError::Python)?; .map_err(InvokeError::Python)?;
@ -781,11 +781,11 @@ impl PythonBinding for MessagesPythonHost {
.is_some_and(|stream| stream.is(pyo3::types::PyBool::new(py, true))); .is_some_and(|stream| stream.is(pyo3::types::PyBool::new(py, true)));
self.arguments = Some(arguments.clone().unbind()); self.arguments = Some(arguments.clone().unbind());
let request = self.decode_call(py, arguments)?; let request = self.decode_call(py, arguments)?;
let hosted = self let transport = self
.transport .transport
.select(py, self.view.request(py).map_err(InvokeError::Python)?) .select(py, self.view.request(py).map_err(InvokeError::Python)?)
.map_err(InvokeError::Python)?; .map_err(InvokeError::Python)?;
Ok((request, selection, hosted)) Ok((request, selection, transport))
} }
fn encode_response( fn encode_response(

View file

@ -33,11 +33,14 @@ fn run_messages(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> PyR
move |py, arguments, request| { move |py, arguments, request| {
let hooked = pre_request::runs_pre_request_hooks(py)?; let hooked = pre_request::runs_pre_request_hooks(py)?;
let context = litellm_host_python::PythonContext::capture(py)?; let context = litellm_host_python::PythonContext::capture(py)?;
let secrets = std::sync::Arc::new(crate::secrets::seeded::SeededSecrets::new(
crate::secrets::source(py)?,
));
let route = litellm_inference_messages::MessagesRoute::new( let route = litellm_inference_messages::MessagesRoute::new(
crate::http::messages_client(py, arguments, asynchronous)? crate::http::messages_client(py, arguments, asynchronous)?
.map_err(crate::http::client_error)?, .map_err(crate::http::client_error)?,
crate::http::resources().auth.clone(), crate::http::resources().auth.clone(),
crate::secrets::source(py)?, std::sync::Arc::clone(&secrets) as _,
) )
.with_workload_identity(std::sync::Arc::new( .with_workload_identity(std::sync::Arc::new(
workload_identity::PythonWorkloadIdentity::new(context.clone()), workload_identity::PythonWorkloadIdentity::new(context.clone()),
@ -57,11 +60,16 @@ fn run_messages(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> PyR
Ok(litellm_host::call::hosted_call( Ok(litellm_host::call::hosted_call(
request, request,
None, None,
move |(call, selection, hosted): (_, crate::cache::Selection, bool), move |(call, selection, transport): (
_,
crate::cache::Selection,
transport::Selected,
),
services, services,
interceptors, interceptors,
observers| async move { observers| async move {
let route = if hosted { secrets.seed(transport.read);
let route = if transport.hosted {
route.with_transport(std::sync::Arc::new(transport::PythonTransport::new( route.with_transport(std::sync::Arc::new(transport::PythonTransport::new(
services.clone(), services.clone(),
))) )))

View file

@ -124,6 +124,11 @@ enum Pending {
Read(Reply<Result<Option<Bytes>, Error>>), Read(Reply<Result<Option<Bytes>, Error>>),
} }
pub(super) struct Selected {
pub(super) hosted: bool,
pub(super) read: Vec<(String, Option<String>)>,
}
#[derive(Default)] #[derive(Default)]
pub(super) struct HostedTransport { pub(super) struct HostedTransport {
handler: Option<Py<PyAny>>, handler: Option<Py<PyAny>>,
@ -132,13 +137,23 @@ pub(super) struct HostedTransport {
} }
impl HostedTransport { impl HostedTransport {
pub(super) fn select(&mut self, py: Python<'_>, request: &Bound<'_, PyDict>) -> PyResult<bool> { /// Whether the call sends through Python's client, and the secrets the host read to decide it,
let handler = py /// which the native route reuses rather than reading again.
pub(super) fn select(
&mut self,
py: Python<'_>,
request: &Bound<'_, PyDict>,
) -> PyResult<Selected> {
let (handler, read): (Bound<'_, PyAny>, Vec<(String, Option<String>)>) = py
.import(ROUTE_HOST_MODULE)? .import(ROUTE_HOST_MODULE)?
.getattr("transport_handler")? .getattr("transport_handler")?
.call1((request,))?; .call1((request,))?
.extract()?;
self.handler = (!handler.is_none()).then(|| handler.unbind()); self.handler = (!handler.is_none()).then(|| handler.unbind());
Ok(self.handler.is_some()) Ok(Selected {
hosted: self.handler.is_some(),
read,
})
} }
pub(super) fn pending(&self) -> bool { pub(super) fn pending(&self) -> bool {

View file

@ -7,6 +7,7 @@ mod provider;
mod python; mod python;
pub(crate) mod resolved; pub(crate) mod resolved;
pub(crate) mod runtime; pub(crate) mod runtime;
pub(crate) mod seeded;
mod vault; mod vault;
use std::sync::Arc; use std::sync::Arc;

View file

@ -0,0 +1,101 @@
use std::{
collections::HashMap,
sync::{Arc, Mutex, PoisonError},
};
use futures_util::future::{BoxFuture, FutureExt};
use litellm_secret_value::SecretValue;
use litellm_secrets::{Error, source::SecretSource};
/// A call's secret source that answers the secrets the host already read for the call with the
/// values it read, so a secret Python reads once is not read a second time.
pub(crate) struct SeededSecrets {
source: Arc<dyn SecretSource>,
read: Mutex<HashMap<String, Option<SecretValue>>>,
}
impl SeededSecrets {
pub(crate) fn new(source: Arc<dyn SecretSource>) -> Self {
Self {
source,
read: Mutex::default(),
}
}
pub(crate) fn seed(&self, read: impl IntoIterator<Item = (String, Option<String>)>) {
self.read
.lock()
.unwrap_or_else(PoisonError::into_inner)
.extend(
read.into_iter()
.map(|(name, value)| (name, value.map(SecretValue::new))),
);
}
}
impl SecretSource for SeededSecrets {
fn get_secret_str<'a>(
&'a self,
name: &'a str,
) -> BoxFuture<'a, Result<Option<SecretValue>, Error>> {
let seeded = self
.read
.lock()
.unwrap_or_else(PoisonError::into_inner)
.get(name)
.cloned();
match seeded {
Some(value) => futures_util::future::ready(Ok(value)).boxed(),
None => self.source.get_secret_str(name),
}
}
}
#[cfg(test)]
mod tests {
use std::sync::atomic::{AtomicUsize, Ordering};
use super::*;
#[derive(Default)]
struct Counting(AtomicUsize);
impl SecretSource for Counting {
fn get_secret_str<'a>(
&'a self,
name: &'a str,
) -> BoxFuture<'a, Result<Option<SecretValue>, Error>> {
self.0.fetch_add(1, Ordering::SeqCst);
futures_util::future::ready(Ok(Some(SecretValue::new(format!("source-{name}")))))
.boxed()
}
}
#[tokio::test]
async fn a_seeded_secret_is_answered_without_reading_the_source() {
let source = Arc::new(Counting::default());
let secrets = SeededSecrets::new(Arc::clone(&source) as Arc<dyn SecretSource>);
secrets.seed([
(
"AZURE_API_BASE".to_owned(),
Some("http://seeded".to_owned()),
),
("UNSET".to_owned(), None),
]);
let seeded = secrets.get_secret_str("AZURE_API_BASE").await.unwrap();
let unset = secrets.get_secret_str("UNSET").await.unwrap();
let other = secrets.get_secret_str("OTHER").await.unwrap();
assert_eq!(
seeded.map(|value| value.expose().to_owned()).as_deref(),
Some("http://seeded")
);
assert!(unset.is_none());
assert_eq!(
other.map(|value| value.expose().to_owned()).as_deref(),
Some("source-OTHER")
);
assert_eq!(source.0.load(Ordering::SeqCst), 1);
}
}

View file

@ -12,6 +12,7 @@ import socket
from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Iterable, Iterator, Mapping, Sequence from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Iterable, Iterator, Mapping, Sequence
from dataclasses import asdict, dataclass, replace from dataclasses import asdict, dataclass, replace
from typing import Final, TypeAlias, cast # noqa: TID251 # narrows native payloads to public TypedDicts from typing import Final, TypeAlias, cast # noqa: TID251 # narrows native payloads to public TypedDicts
from urllib.parse import SplitResult, urlsplit
import httpx import httpx
from aiohttp.http_exceptions import ContentLengthError, TransferEncodingError from aiohttp.http_exceptions import ContentLengthError, TransferEncodingError
@ -91,6 +92,7 @@ from litellm.rust_bridge.execution import AsyncStream
from litellm.rust_bridge.host import cache, callbacks_legacy_python, http_transport from litellm.rust_bridge.host import cache, callbacks_legacy_python, http_transport
from litellm.rust_bridge.messages.entrypoints import MessagesStreamHead from litellm.rust_bridge.messages.entrypoints import MessagesStreamHead
from litellm.rust_bridge.public_call import NativeCall, known_params, optional_sequence, optional_str from litellm.rust_bridge.public_call import NativeCall, known_params, optional_sequence, optional_str
from litellm.secret_managers.main import get_secret_str
from litellm.types.integrations.custom_logger import WEBSEARCH_CONVERTED_STREAM_KEY from litellm.types.integrations.custom_logger import WEBSEARCH_CONVERTED_STREAM_KEY
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse
from litellm.types.router import GenericLiteLLMParams from litellm.types.router import GenericLiteLLMParams
@ -1218,8 +1220,9 @@ class Sent:
def _unsendable_api_base(api_base: object) -> bool: def _unsendable_api_base(api_base: object) -> bool:
"""A caller api_base the native URL parser rejects or reads otherwise than httpx and aiohttp """A caller api_base the native URL parser rejects or reads otherwise than httpx and aiohttp
(no http(s) scheme, whitespace, control or non-ASCII characters): Python's client decides what (no http(s) scheme, whitespace, control or non-ASCII characters, credentials, no host or a port
such a URL does, from a relative-URL error to a request on the host it finds.""" Python cannot read): Python's client decides what such a URL does, from a relative-URL error to
a request on the host it finds."""
if not isinstance(api_base, str) or not api_base: if not isinstance(api_base, str) or not api_base:
return False return False
return not ( return not (
@ -1227,18 +1230,50 @@ def _unsendable_api_base(api_base: object) -> bool:
and api_base.isprintable() and api_base.isprintable()
and " " not in api_base and " " not in api_base
and api_base.lower().startswith(("http://", "https://")) and api_base.lower().startswith(("http://", "https://"))
and _plain_authority(api_base)
) )
def transport_handler(request: Mapping[str, object]) -> AsyncHTTPHandler | None: def _plain_authority(api_base: str) -> bool:
try:
parts: Final = urlsplit(api_base)
except ValueError:
return True
return bool(parts.hostname) and parts.username is None and _readable_port(parts)
def _readable_port(parts: SplitResult) -> bool:
try:
return parts.port is None or parts.port >= 0
except ValueError:
return False
_SecretsRead: TypeAlias = list[tuple[str, str | None]]
def _sent_api_base(request: Mapping[str, object]) -> tuple[object, _SecretsRead]:
"""The base Python's URL builder reads, and the secrets read to find it: azure_ai falls back to
`AZURE_API_BASE`, read once, which the native route then reuses."""
api_base: Final = request.get("api_base")
if api_base or "model" not in request or _resolved_provider(request) != "azure_ai":
return api_base, []
environment: Final = get_secret_str("AZURE_API_BASE")
return environment, [("AZURE_API_BASE", environment)]
def transport_handler(request: Mapping[str, object]) -> tuple[AsyncHTTPHandler | None, _SecretsRead]:
"""The client the call sends through (None for the native one), and the secrets read to choose
it, which the native route reuses rather than reading them again."""
client: Final = request.get("client") client: Final = request.get("client")
if isinstance(client, AsyncHTTPHandler): if isinstance(client, AsyncHTTPHandler):
return client return client, []
if not http_transport.sends_through_python(client, request.get("timeout")) and not _unsendable_api_base( if http_transport.sends_through_python(client, request.get("timeout")):
request.get("api_base") return get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC), []
): api_base, read = _sent_api_base(request)
return None if not _unsendable_api_base(api_base):
return get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC) return None, read
return get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC), read
def decoding_failure(message: str, request: Mapping[str, object], request_provider: str) -> Exception: def decoding_failure(message: str, request: Mapping[str, object], request_provider: str) -> Exception:
@ -1260,7 +1295,12 @@ def unencodable_header(value: bytes, received: bool) -> Exception:
def _sent_url(request: Mapping[str, object], url: str) -> str: def _sent_url(request: Mapping[str, object], url: str) -> str:
"""The native URL builder trims the caller's api_base; Python sends it as given.""" """The native URL builder trims the caller's api_base; Python sends it as given."""
api_base: Final = request.get("api_base") api_base: Final = request.get("api_base")
if not isinstance(api_base, str) or api_base == api_base.strip() or not _unsendable_api_base(api_base): if (
not isinstance(api_base, str)
or api_base == api_base.strip()
or not _unsendable_api_base(api_base)
or _resolved_provider(request) != "anthropic"
):
return url return url
try: try:
return _python_messages_url(api_base) if url == _python_messages_url(api_base.strip()) else url return _python_messages_url(api_base) if url == _python_messages_url(api_base.strip()) else url

View file

@ -3509,6 +3509,56 @@ AZURE_AUTH_CASES: Final = tuple(
) )
) )
AZURE_URL_CASES: Final = tuple(
replace(case, model=AZURE_CLAUDE, rust_providers=AZURE_AI)
for case in (
Case(
"azure_ai_base_userinfo_starting_with_anthropic",
_kw(api_base_userinfo="anthropic", api_base_suffix="/d/dep-u01/anthropic"),
),
Case("azure_ai_base_slash_before_query", _kw(api_base_suffix="/d/dep-u02/?q=1")),
Case("azure_ai_base_trailing_spaces_without_anthropic", _kw(api_base_suffix="/d/dep-u04 ")),
Case("azure_ai_base_trailing_newline_without_anthropic", _kw(api_base_suffix="/d/dep-u04\n"), upstream=()),
Case("azure_ai_base_trailing_spaces_after_anthropic", _kw(api_base_suffix="/d/dep-u04/anthropic ")),
Case("azure_ai_base_trailing_crlf_after_anthropic", _kw(api_base_suffix="/d/dep-u04/anthropic\r\n")),
Case("azure_ai_blank_base", _kw(api_base=" "), upstream=()),
Case(
"azure_ai_blank_base_with_environment_base",
_kw(api_base=" "),
env=MappingProxyType({"AZURE_API_BASE": "http://127.0.0.1:9/d/dep-env/anthropic"}),
upstream=(),
),
Case(
"azure_ai_padded_environment_base",
_kw(api_base=None),
env=MappingProxyType({"AZURE_API_BASE": " http://127.0.0.1:9/d/dep-env/anthropic "}),
upstream=(),
),
Case("azure_ai_missing_base", _kw(api_base=None), upstream=()),
Case("azure_ai_missing_base_stream", _kw(api_base=None, stream=True), upstream=()),
Case("azure_ai_empty_base", _kw(api_base=""), upstream=()),
Case("azure_ai_base_without_scheme", _kw(api_base="fuzz.invalid/d/dep-e04/anthropic"), upstream=()),
Case("azure_ai_scheme_relative_base", _kw(api_base="//fuzz.invalid/d/dep-e04/anthropic"), upstream=()),
Case("azure_ai_ftp_base", _kw(api_base="ftp://fuzz.invalid/d/dep-e04/anthropic"), upstream=()),
Case("azure_ai_base_not_a_url", _kw(api_base="not a url"), upstream=()),
Case(
"azure_ai_base_non_numeric_port",
_kw(api_base="http://fuzz.invalid:notaport/d/dep-e04/anthropic"),
upstream=(),
),
Case(
"azure_ai_base_out_of_range_port",
_kw(api_base="http://fuzz.invalid:99999/d/dep-e04/anthropic"),
upstream=(),
),
Case("azure_ai_base_empty_host", _kw(api_base="http:///d/dep-e04/anthropic"), upstream=()),
)
)
# An anthropic base with no host goes through Python's client, so the key is never sent to the host
# the native parser read from the path (`d`).
ANTHROPIC_URL_CASES: Final = (Case("anthropic_base_empty_host", _kw(api_base="http:///d"), upstream=()),)
CASES: Final = ( CASES: Final = (
*REQUEST_CASES, *REQUEST_CASES,
*INPUT_VALUE_CASES, *INPUT_VALUE_CASES,
@ -3535,6 +3585,8 @@ CASES: Final = (
*DEPLOYMENT_HOOK_CASES, *DEPLOYMENT_HOOK_CASES,
*HOOKED_STREAM_AND_CACHE_CASES, *HOOKED_STREAM_AND_CACHE_CASES,
*AZURE_AUTH_CASES, *AZURE_AUTH_CASES,
*AZURE_URL_CASES,
*ANTHROPIC_URL_CASES,
) )
@ -3693,12 +3745,14 @@ def _kwargs(case: Case, server: RecordingServer) -> dict[str, object]:
# other leg. # other leg.
supplied: Final = copy.deepcopy(dict(case.kwargs)) supplied: Final = copy.deepcopy(dict(case.kwargs))
suffix: Final = supplied.pop("api_base_suffix", "") suffix: Final = supplied.pop("api_base_suffix", "")
userinfo: Final = supplied.pop("api_base_userinfo", "")
base: Final = server.base_url.replace("://", f"://{userinfo}@", 1) if userinfo else server.base_url
kwargs: Final[dict[str, object]] = { kwargs: Final[dict[str, object]] = {
"model": case.model, "model": case.model,
"max_tokens": 64, "max_tokens": 64,
"messages": [dict(USER)], "messages": [dict(USER)],
"api_key": API_KEY, "api_key": API_KEY,
"api_base": f"{server.base_url}{suffix}", "api_base": f"{base}{suffix}",
**supplied, **supplied,
} }
return {name: value for name, value in kwargs.items() if value is not None or name in ("safeguards", *case.nones)} return {name: value for name, value in kwargs.items() if value is not None or name in ("safeguards", *case.nones)}
@ -3839,7 +3893,9 @@ async def _invoke(
if isinstance(result, dict): if isinstance(result, dict):
body = cast(Mapping[str, object], result) body = cast(Mapping[str, object], result)
return _response(body, case.private_keys), _hidden_params(body) return _response(body, case.private_keys), _hidden_params(body)
time.sleep(case.busy_before_reading) # noqa: ASYNC251 # holds the event loop, as a caller's own work between the call and its first read does time.sleep(
case.busy_before_reading
) # holds the event loop, as a caller's own work between the call and its first read does
chunks, failure = await _consume_async(cast(AsyncIterator[object], result)) chunks, failure = await _consume_async(cast(AsyncIterator[object], result))
hidden: Final = _hidden_params(result) hidden: Final = _hidden_params(result)
partition: Final = {"partition": _partition(chunks)} if case.partitioned else {} partition: Final = {"partition": _partition(chunks)} if case.partitioned else {}

View file

@ -974,14 +974,14 @@ def no_tls_settings(monkeypatch: pytest.MonkeyPatch) -> None:
"client", (None, object(), httpx.AsyncClient(), HTTPHandler()), ids=("none", "object", "httpx", "sync-handler") "client", (None, object(), httpx.AsyncClient(), HTTPHandler()), ids=("none", "object", "httpx", "sync-handler")
) )
def test_the_native_client_stands_in_for_any_client_python_ignores(client: object) -> None: def test_the_native_client_stands_in_for_any_client_python_ignores(client: object) -> None:
assert messages_host.transport_handler({"client": client}) is None assert messages_host.transport_handler({"client": client}) == (None, [])
@pytest.mark.usefixtures("no_tls_settings") @pytest.mark.usefixtures("no_tls_settings")
def test_a_caller_async_handler_is_the_handler_python_posts_through() -> None: def test_a_caller_async_handler_is_the_handler_python_posts_through() -> None:
client: Final = AsyncHTTPHandler() client: Final = AsyncHTTPHandler()
assert messages_host.transport_handler({"client": client}) is client assert messages_host.transport_handler({"client": client}) == (client, [])
@pytest.mark.usefixtures("no_tls_settings") @pytest.mark.usefixtures("no_tls_settings")
@ -995,7 +995,7 @@ def test_tls_the_native_client_cannot_apply_posts_through_pythons_anthropic_hand
monkeypatch.setattr(litellm, name, value) monkeypatch.setattr(litellm, name, value)
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
handler: Final = messages_host.transport_handler({"client": object()}) handler, _ = messages_host.transport_handler({"client": object()})
assert handler is get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC) assert handler is get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC)
assert callbacks_legacy_python.times_provider_exchange(object(), None) is False assert callbacks_legacy_python.times_provider_exchange(object(), None) is False
@ -1012,7 +1012,7 @@ def fips_cipher_exclusion() -> Iterator[None]:
def test_fips_cipher_exclusion_posts_through_pythons_anthropic_handler(monkeypatch: pytest.MonkeyPatch) -> None: def test_fips_cipher_exclusion_posts_through_pythons_anthropic_handler(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
handler: Final = messages_host.transport_handler({"client": None}) handler, _ = messages_host.transport_handler({"client": None})
assert handler is get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC) assert handler is get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC)
assert callbacks_legacy_python.times_provider_exchange(None, None) is False assert callbacks_legacy_python.times_provider_exchange(None, None) is False
@ -1036,7 +1036,7 @@ def test_a_non_positive_timeout_posts_through_pythons_anthropic_handler(
) -> None: ) -> None:
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
handler: Final = messages_host.transport_handler({"client": None, "timeout": timeout}) handler, _ = messages_host.transport_handler({"client": None, "timeout": timeout})
assert handler is get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC) assert handler is get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC)
assert callbacks_legacy_python.times_provider_exchange(None, timeout) is False assert callbacks_legacy_python.times_provider_exchange(None, timeout) is False
@ -1045,7 +1045,97 @@ def test_a_non_positive_timeout_posts_through_pythons_anthropic_handler(
@pytest.mark.usefixtures("no_tls_settings") @pytest.mark.usefixtures("no_tls_settings")
@pytest.mark.parametrize("timeout", (None, 0.5, httpx.Timeout(10.0)), ids=("unset", "positive", "httpx")) @pytest.mark.parametrize("timeout", (None, 0.5, httpx.Timeout(10.0)), ids=("unset", "positive", "httpx"))
def test_a_positive_timeout_keeps_the_native_client(timeout: object) -> None: def test_a_positive_timeout_keeps_the_native_client(timeout: object) -> None:
assert messages_host.transport_handler({"client": None, "timeout": timeout}) is None assert messages_host.transport_handler({"client": None, "timeout": timeout}) == (None, [])
@pytest.mark.usefixtures("no_tls_settings")
@pytest.mark.parametrize(
"api_base",
(
"http://user:secret@host.example/d",
"http://user@host.example/d",
"http:///d",
"http://:443/d",
"http://host.example:99999/d",
"http://host.example:port/d",
),
ids=("credentials", "user", "no-host", "port-only", "port-out-of-range", "port-not-a-number"),
)
@pytest.mark.parametrize("model", ("anthropic/claude-x", "azure_ai/claude-x"))
def test_a_base_the_native_parser_reads_otherwise_posts_through_pythons_handler(
monkeypatch: pytest.MonkeyPatch, api_base: str, model: str
) -> None:
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
handler, read = messages_host.transport_handler({"client": None, "model": model, "api_base": api_base})
assert handler is get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC)
assert read == []
@pytest.mark.usefixtures("no_tls_settings")
@pytest.mark.parametrize(
"api_base",
("https://host.example/d", "http://host.example:8080/d?x=1", "https://[::1]:443/d"),
ids=("plain", "port-and-query", "ipv6"),
)
@pytest.mark.parametrize("model", ("anthropic/claude-x", "azure_ai/claude-x"))
def test_a_plain_base_keeps_the_native_client(api_base: str, model: str) -> None:
assert messages_host.transport_handler({"client": None, "model": model, "api_base": api_base}) == (None, [])
@pytest.mark.usefixtures("no_tls_settings")
@pytest.mark.parametrize(
("environment", "hosted"),
((" https://env.example/d ", True), ("https://env.example/d", False), (None, False)),
ids=("padded", "plain", "unset"),
)
@pytest.mark.parametrize("api_base", (None, ""), ids=("absent", "empty"))
def test_azure_ai_reads_azure_api_base_once_and_hands_it_to_the_native_route(
monkeypatch: pytest.MonkeyPatch, environment: str | None, hosted: bool, api_base: str | None
) -> None:
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
reads: Final[list[str]] = []
def get_secret_str(name: str) -> str | None:
reads.append(name)
return environment
monkeypatch.setattr(messages_host, "get_secret_str", get_secret_str)
handler, read = messages_host.transport_handler(
{"client": None, "model": "azure_ai/claude-x", "api_base": api_base}
)
assert (handler is not None) is hosted
assert read == [("AZURE_API_BASE", environment)]
assert reads == ["AZURE_API_BASE"]
@pytest.mark.usefixtures("no_tls_settings")
@pytest.mark.parametrize(
"request_",
(
{"client": None, "model": "anthropic/claude-x"},
{"client": None, "model": "azure_ai/claude-x", "api_base": "https://call.example/d"},
{"client": AsyncHTTPHandler(), "model": "azure_ai/claude-x"},
{"client": None, "model": "azure_ai/claude-x", "timeout": 0},
),
ids=("anthropic", "azure-call-base", "azure-caller-handler", "azure-python-timeout"),
)
def test_azure_api_base_is_left_to_the_native_route_when_the_choice_does_not_need_it(
monkeypatch: pytest.MonkeyPatch, request_: Mapping[str, object]
) -> None:
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
def get_secret_str(name: str) -> str | None:
raise AssertionError(name)
monkeypatch.setattr(messages_host, "get_secret_str", get_secret_str)
_, read = messages_host.transport_handler(request_)
assert read == []
@pytest.mark.asyncio @pytest.mark.asyncio