mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
fix(messages): send bases the native parser misreads through python's client on rust
A Messages api_base with credentials, no host or a port Python cannot read now goes through Python's client, as bases without an http(s) scheme or with whitespace already did. Before, reqwest read such a URL differently from httpx: for http:///d it sent the request, with the operator key, to host `d`. This is a security fix for both azure_ai and anthropic; the anthropic_base_empty_host parity case covers anthropic. For azure_ai the choice reads the base Python's URL builder reads, falling back to AZURE_API_BASE, so a padded environment base also goes through Python's client. Python reads AZURE_API_BASE once: transport_handler returns the value it read and the bridge seeds the call's secret source with it, so the native route reuses it rather than reading the secret again. _sent_url only rewrites anthropic URLs, since the azure_ai builder no longer trims. Parity: 20 azure_ai and anthropic URL cases added; 65 aspects failed on the previous extension, and the full messages parity suite now passes.
This commit is contained in:
Claude
parent
398f8ac165
commit
d1efdd2683
8 changed files with 341 additions and 30 deletions
|
|
@ -36,7 +36,7 @@ use serde_json::{Map, Value};
|
|||
|
||||
use super::agentic::HookedRequest;
|
||||
use super::safeguards::{SafeguardsCall, SafeguardsHost};
|
||||
use super::transport::{HostedTransport, TransportCall, python_failure};
|
||||
use super::transport::{HostedTransport, Selected, TransportCall, python_failure};
|
||||
use crate::{
|
||||
errors::{RequestMarker, route_failure},
|
||||
marshal::{
|
||||
|
|
@ -236,7 +236,7 @@ fn azure_module_settings<'py>(
|
|||
pub(super) struct MessagesHosted;
|
||||
|
||||
impl litellm_host::protocol::Protocol for MessagesHosted {
|
||||
type Request = (MessagesCall, Selection, bool);
|
||||
type Request = (MessagesCall, Selection, Selected);
|
||||
type Response = <Messages as litellm_host::protocol::Protocol>::Response;
|
||||
type Error = Error;
|
||||
type HostCall = MessagesHostCall;
|
||||
|
|
@ -755,7 +755,7 @@ impl PythonBinding for MessagesPythonHost {
|
|||
&mut self,
|
||||
py: Python<'_>,
|
||||
arguments: &Bound<'_, PyDict>,
|
||||
) -> Result<(MessagesCall, Selection, bool), InvokeError<Error>> {
|
||||
) -> Result<(MessagesCall, Selection, Selected), InvokeError<Error>> {
|
||||
self.view
|
||||
.resolve(py, arguments)
|
||||
.map_err(InvokeError::Python)?;
|
||||
|
|
@ -781,11 +781,11 @@ impl PythonBinding for MessagesPythonHost {
|
|||
.is_some_and(|stream| stream.is(pyo3::types::PyBool::new(py, true)));
|
||||
self.arguments = Some(arguments.clone().unbind());
|
||||
let request = self.decode_call(py, arguments)?;
|
||||
let hosted = self
|
||||
let transport = self
|
||||
.transport
|
||||
.select(py, self.view.request(py).map_err(InvokeError::Python)?)
|
||||
.map_err(InvokeError::Python)?;
|
||||
Ok((request, selection, hosted))
|
||||
Ok((request, selection, transport))
|
||||
}
|
||||
|
||||
fn encode_response(
|
||||
|
|
|
|||
|
|
@ -33,11 +33,14 @@ fn run_messages(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> PyR
|
|||
move |py, arguments, request| {
|
||||
let hooked = pre_request::runs_pre_request_hooks(py)?;
|
||||
let context = litellm_host_python::PythonContext::capture(py)?;
|
||||
let secrets = std::sync::Arc::new(crate::secrets::seeded::SeededSecrets::new(
|
||||
crate::secrets::source(py)?,
|
||||
));
|
||||
let route = litellm_inference_messages::MessagesRoute::new(
|
||||
crate::http::messages_client(py, arguments, asynchronous)?
|
||||
.map_err(crate::http::client_error)?,
|
||||
crate::http::resources().auth.clone(),
|
||||
crate::secrets::source(py)?,
|
||||
std::sync::Arc::clone(&secrets) as _,
|
||||
)
|
||||
.with_workload_identity(std::sync::Arc::new(
|
||||
workload_identity::PythonWorkloadIdentity::new(context.clone()),
|
||||
|
|
@ -57,11 +60,16 @@ fn run_messages(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> PyR
|
|||
Ok(litellm_host::call::hosted_call(
|
||||
request,
|
||||
None,
|
||||
move |(call, selection, hosted): (_, crate::cache::Selection, bool),
|
||||
move |(call, selection, transport): (
|
||||
_,
|
||||
crate::cache::Selection,
|
||||
transport::Selected,
|
||||
),
|
||||
services,
|
||||
interceptors,
|
||||
observers| async move {
|
||||
let route = if hosted {
|
||||
secrets.seed(transport.read);
|
||||
let route = if transport.hosted {
|
||||
route.with_transport(std::sync::Arc::new(transport::PythonTransport::new(
|
||||
services.clone(),
|
||||
)))
|
||||
|
|
|
|||
|
|
@ -124,6 +124,11 @@ enum Pending {
|
|||
Read(Reply<Result<Option<Bytes>, Error>>),
|
||||
}
|
||||
|
||||
pub(super) struct Selected {
|
||||
pub(super) hosted: bool,
|
||||
pub(super) read: Vec<(String, Option<String>)>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct HostedTransport {
|
||||
handler: Option<Py<PyAny>>,
|
||||
|
|
@ -132,13 +137,23 @@ pub(super) struct HostedTransport {
|
|||
}
|
||||
|
||||
impl HostedTransport {
|
||||
pub(super) fn select(&mut self, py: Python<'_>, request: &Bound<'_, PyDict>) -> PyResult<bool> {
|
||||
let handler = py
|
||||
/// Whether the call sends through Python's client, and the secrets the host read to decide it,
|
||||
/// which the native route reuses rather than reading again.
|
||||
pub(super) fn select(
|
||||
&mut self,
|
||||
py: Python<'_>,
|
||||
request: &Bound<'_, PyDict>,
|
||||
) -> PyResult<Selected> {
|
||||
let (handler, read): (Bound<'_, PyAny>, Vec<(String, Option<String>)>) = py
|
||||
.import(ROUTE_HOST_MODULE)?
|
||||
.getattr("transport_handler")?
|
||||
.call1((request,))?;
|
||||
.call1((request,))?
|
||||
.extract()?;
|
||||
self.handler = (!handler.is_none()).then(|| handler.unbind());
|
||||
Ok(self.handler.is_some())
|
||||
Ok(Selected {
|
||||
hosted: self.handler.is_some(),
|
||||
read,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn pending(&self) -> bool {
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ mod provider;
|
|||
mod python;
|
||||
pub(crate) mod resolved;
|
||||
pub(crate) mod runtime;
|
||||
pub(crate) mod seeded;
|
||||
mod vault;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
|
|
|||
101
litellm/rust/crates/python-bridge/src/secrets/seeded.rs
Normal file
101
litellm/rust/crates/python-bridge/src/secrets/seeded.rs
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
use std::{
|
||||
collections::HashMap,
|
||||
sync::{Arc, Mutex, PoisonError},
|
||||
};
|
||||
|
||||
use futures_util::future::{BoxFuture, FutureExt};
|
||||
use litellm_secret_value::SecretValue;
|
||||
use litellm_secrets::{Error, source::SecretSource};
|
||||
|
||||
/// A call's secret source that answers the secrets the host already read for the call with the
|
||||
/// values it read, so a secret Python reads once is not read a second time.
|
||||
pub(crate) struct SeededSecrets {
|
||||
source: Arc<dyn SecretSource>,
|
||||
read: Mutex<HashMap<String, Option<SecretValue>>>,
|
||||
}
|
||||
|
||||
impl SeededSecrets {
|
||||
pub(crate) fn new(source: Arc<dyn SecretSource>) -> Self {
|
||||
Self {
|
||||
source,
|
||||
read: Mutex::default(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn seed(&self, read: impl IntoIterator<Item = (String, Option<String>)>) {
|
||||
self.read
|
||||
.lock()
|
||||
.unwrap_or_else(PoisonError::into_inner)
|
||||
.extend(
|
||||
read.into_iter()
|
||||
.map(|(name, value)| (name, value.map(SecretValue::new))),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
impl SecretSource for SeededSecrets {
|
||||
fn get_secret_str<'a>(
|
||||
&'a self,
|
||||
name: &'a str,
|
||||
) -> BoxFuture<'a, Result<Option<SecretValue>, Error>> {
|
||||
let seeded = self
|
||||
.read
|
||||
.lock()
|
||||
.unwrap_or_else(PoisonError::into_inner)
|
||||
.get(name)
|
||||
.cloned();
|
||||
match seeded {
|
||||
Some(value) => futures_util::future::ready(Ok(value)).boxed(),
|
||||
None => self.source.get_secret_str(name),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
use super::*;
|
||||
|
||||
#[derive(Default)]
|
||||
struct Counting(AtomicUsize);
|
||||
|
||||
impl SecretSource for Counting {
|
||||
fn get_secret_str<'a>(
|
||||
&'a self,
|
||||
name: &'a str,
|
||||
) -> BoxFuture<'a, Result<Option<SecretValue>, Error>> {
|
||||
self.0.fetch_add(1, Ordering::SeqCst);
|
||||
futures_util::future::ready(Ok(Some(SecretValue::new(format!("source-{name}")))))
|
||||
.boxed()
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_seeded_secret_is_answered_without_reading_the_source() {
|
||||
let source = Arc::new(Counting::default());
|
||||
let secrets = SeededSecrets::new(Arc::clone(&source) as Arc<dyn SecretSource>);
|
||||
secrets.seed([
|
||||
(
|
||||
"AZURE_API_BASE".to_owned(),
|
||||
Some("http://seeded".to_owned()),
|
||||
),
|
||||
("UNSET".to_owned(), None),
|
||||
]);
|
||||
|
||||
let seeded = secrets.get_secret_str("AZURE_API_BASE").await.unwrap();
|
||||
let unset = secrets.get_secret_str("UNSET").await.unwrap();
|
||||
let other = secrets.get_secret_str("OTHER").await.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
seeded.map(|value| value.expose().to_owned()).as_deref(),
|
||||
Some("http://seeded")
|
||||
);
|
||||
assert!(unset.is_none());
|
||||
assert_eq!(
|
||||
other.map(|value| value.expose().to_owned()).as_deref(),
|
||||
Some("source-OTHER")
|
||||
);
|
||||
assert_eq!(source.0.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
}
|
||||
|
|
@ -12,6 +12,7 @@ import socket
|
|||
from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Iterable, Iterator, Mapping, Sequence
|
||||
from dataclasses import asdict, dataclass, replace
|
||||
from typing import Final, TypeAlias, cast # noqa: TID251 # narrows native payloads to public TypedDicts
|
||||
from urllib.parse import SplitResult, urlsplit
|
||||
|
||||
import httpx
|
||||
from aiohttp.http_exceptions import ContentLengthError, TransferEncodingError
|
||||
|
|
@ -91,6 +92,7 @@ from litellm.rust_bridge.execution import AsyncStream
|
|||
from litellm.rust_bridge.host import cache, callbacks_legacy_python, http_transport
|
||||
from litellm.rust_bridge.messages.entrypoints import MessagesStreamHead
|
||||
from litellm.rust_bridge.public_call import NativeCall, known_params, optional_sequence, optional_str
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.integrations.custom_logger import WEBSEARCH_CONVERTED_STREAM_KEY
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
|
@ -1218,8 +1220,9 @@ class Sent:
|
|||
|
||||
def _unsendable_api_base(api_base: object) -> bool:
|
||||
"""A caller api_base the native URL parser rejects or reads otherwise than httpx and aiohttp
|
||||
(no http(s) scheme, whitespace, control or non-ASCII characters): Python's client decides what
|
||||
such a URL does, from a relative-URL error to a request on the host it finds."""
|
||||
(no http(s) scheme, whitespace, control or non-ASCII characters, credentials, no host or a port
|
||||
Python cannot read): Python's client decides what such a URL does, from a relative-URL error to
|
||||
a request on the host it finds."""
|
||||
if not isinstance(api_base, str) or not api_base:
|
||||
return False
|
||||
return not (
|
||||
|
|
@ -1227,18 +1230,50 @@ def _unsendable_api_base(api_base: object) -> bool:
|
|||
and api_base.isprintable()
|
||||
and " " not in api_base
|
||||
and api_base.lower().startswith(("http://", "https://"))
|
||||
and _plain_authority(api_base)
|
||||
)
|
||||
|
||||
|
||||
def transport_handler(request: Mapping[str, object]) -> AsyncHTTPHandler | None:
|
||||
def _plain_authority(api_base: str) -> bool:
|
||||
try:
|
||||
parts: Final = urlsplit(api_base)
|
||||
except ValueError:
|
||||
return True
|
||||
return bool(parts.hostname) and parts.username is None and _readable_port(parts)
|
||||
|
||||
|
||||
def _readable_port(parts: SplitResult) -> bool:
|
||||
try:
|
||||
return parts.port is None or parts.port >= 0
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
_SecretsRead: TypeAlias = list[tuple[str, str | None]]
|
||||
|
||||
|
||||
def _sent_api_base(request: Mapping[str, object]) -> tuple[object, _SecretsRead]:
|
||||
"""The base Python's URL builder reads, and the secrets read to find it: azure_ai falls back to
|
||||
`AZURE_API_BASE`, read once, which the native route then reuses."""
|
||||
api_base: Final = request.get("api_base")
|
||||
if api_base or "model" not in request or _resolved_provider(request) != "azure_ai":
|
||||
return api_base, []
|
||||
environment: Final = get_secret_str("AZURE_API_BASE")
|
||||
return environment, [("AZURE_API_BASE", environment)]
|
||||
|
||||
|
||||
def transport_handler(request: Mapping[str, object]) -> tuple[AsyncHTTPHandler | None, _SecretsRead]:
|
||||
"""The client the call sends through (None for the native one), and the secrets read to choose
|
||||
it, which the native route reuses rather than reading them again."""
|
||||
client: Final = request.get("client")
|
||||
if isinstance(client, AsyncHTTPHandler):
|
||||
return client
|
||||
if not http_transport.sends_through_python(client, request.get("timeout")) and not _unsendable_api_base(
|
||||
request.get("api_base")
|
||||
):
|
||||
return None
|
||||
return get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC)
|
||||
return client, []
|
||||
if http_transport.sends_through_python(client, request.get("timeout")):
|
||||
return get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC), []
|
||||
api_base, read = _sent_api_base(request)
|
||||
if not _unsendable_api_base(api_base):
|
||||
return None, read
|
||||
return get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC), read
|
||||
|
||||
|
||||
def decoding_failure(message: str, request: Mapping[str, object], request_provider: str) -> Exception:
|
||||
|
|
@ -1260,7 +1295,12 @@ def unencodable_header(value: bytes, received: bool) -> Exception:
|
|||
def _sent_url(request: Mapping[str, object], url: str) -> str:
|
||||
"""The native URL builder trims the caller's api_base; Python sends it as given."""
|
||||
api_base: Final = request.get("api_base")
|
||||
if not isinstance(api_base, str) or api_base == api_base.strip() or not _unsendable_api_base(api_base):
|
||||
if (
|
||||
not isinstance(api_base, str)
|
||||
or api_base == api_base.strip()
|
||||
or not _unsendable_api_base(api_base)
|
||||
or _resolved_provider(request) != "anthropic"
|
||||
):
|
||||
return url
|
||||
try:
|
||||
return _python_messages_url(api_base) if url == _python_messages_url(api_base.strip()) else url
|
||||
|
|
|
|||
|
|
@ -3509,6 +3509,56 @@ AZURE_AUTH_CASES: Final = tuple(
|
|||
)
|
||||
)
|
||||
|
||||
AZURE_URL_CASES: Final = tuple(
|
||||
replace(case, model=AZURE_CLAUDE, rust_providers=AZURE_AI)
|
||||
for case in (
|
||||
Case(
|
||||
"azure_ai_base_userinfo_starting_with_anthropic",
|
||||
_kw(api_base_userinfo="anthropic", api_base_suffix="/d/dep-u01/anthropic"),
|
||||
),
|
||||
Case("azure_ai_base_slash_before_query", _kw(api_base_suffix="/d/dep-u02/?q=1")),
|
||||
Case("azure_ai_base_trailing_spaces_without_anthropic", _kw(api_base_suffix="/d/dep-u04 ")),
|
||||
Case("azure_ai_base_trailing_newline_without_anthropic", _kw(api_base_suffix="/d/dep-u04\n"), upstream=()),
|
||||
Case("azure_ai_base_trailing_spaces_after_anthropic", _kw(api_base_suffix="/d/dep-u04/anthropic ")),
|
||||
Case("azure_ai_base_trailing_crlf_after_anthropic", _kw(api_base_suffix="/d/dep-u04/anthropic\r\n")),
|
||||
Case("azure_ai_blank_base", _kw(api_base=" "), upstream=()),
|
||||
Case(
|
||||
"azure_ai_blank_base_with_environment_base",
|
||||
_kw(api_base=" "),
|
||||
env=MappingProxyType({"AZURE_API_BASE": "http://127.0.0.1:9/d/dep-env/anthropic"}),
|
||||
upstream=(),
|
||||
),
|
||||
Case(
|
||||
"azure_ai_padded_environment_base",
|
||||
_kw(api_base=None),
|
||||
env=MappingProxyType({"AZURE_API_BASE": " http://127.0.0.1:9/d/dep-env/anthropic "}),
|
||||
upstream=(),
|
||||
),
|
||||
Case("azure_ai_missing_base", _kw(api_base=None), upstream=()),
|
||||
Case("azure_ai_missing_base_stream", _kw(api_base=None, stream=True), upstream=()),
|
||||
Case("azure_ai_empty_base", _kw(api_base=""), upstream=()),
|
||||
Case("azure_ai_base_without_scheme", _kw(api_base="fuzz.invalid/d/dep-e04/anthropic"), upstream=()),
|
||||
Case("azure_ai_scheme_relative_base", _kw(api_base="//fuzz.invalid/d/dep-e04/anthropic"), upstream=()),
|
||||
Case("azure_ai_ftp_base", _kw(api_base="ftp://fuzz.invalid/d/dep-e04/anthropic"), upstream=()),
|
||||
Case("azure_ai_base_not_a_url", _kw(api_base="not a url"), upstream=()),
|
||||
Case(
|
||||
"azure_ai_base_non_numeric_port",
|
||||
_kw(api_base="http://fuzz.invalid:notaport/d/dep-e04/anthropic"),
|
||||
upstream=(),
|
||||
),
|
||||
Case(
|
||||
"azure_ai_base_out_of_range_port",
|
||||
_kw(api_base="http://fuzz.invalid:99999/d/dep-e04/anthropic"),
|
||||
upstream=(),
|
||||
),
|
||||
Case("azure_ai_base_empty_host", _kw(api_base="http:///d/dep-e04/anthropic"), upstream=()),
|
||||
)
|
||||
)
|
||||
|
||||
# An anthropic base with no host goes through Python's client, so the key is never sent to the host
|
||||
# the native parser read from the path (`d`).
|
||||
ANTHROPIC_URL_CASES: Final = (Case("anthropic_base_empty_host", _kw(api_base="http:///d"), upstream=()),)
|
||||
|
||||
CASES: Final = (
|
||||
*REQUEST_CASES,
|
||||
*INPUT_VALUE_CASES,
|
||||
|
|
@ -3535,6 +3585,8 @@ CASES: Final = (
|
|||
*DEPLOYMENT_HOOK_CASES,
|
||||
*HOOKED_STREAM_AND_CACHE_CASES,
|
||||
*AZURE_AUTH_CASES,
|
||||
*AZURE_URL_CASES,
|
||||
*ANTHROPIC_URL_CASES,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -3693,12 +3745,14 @@ def _kwargs(case: Case, server: RecordingServer) -> dict[str, object]:
|
|||
# other leg.
|
||||
supplied: Final = copy.deepcopy(dict(case.kwargs))
|
||||
suffix: Final = supplied.pop("api_base_suffix", "")
|
||||
userinfo: Final = supplied.pop("api_base_userinfo", "")
|
||||
base: Final = server.base_url.replace("://", f"://{userinfo}@", 1) if userinfo else server.base_url
|
||||
kwargs: Final[dict[str, object]] = {
|
||||
"model": case.model,
|
||||
"max_tokens": 64,
|
||||
"messages": [dict(USER)],
|
||||
"api_key": API_KEY,
|
||||
"api_base": f"{server.base_url}{suffix}",
|
||||
"api_base": f"{base}{suffix}",
|
||||
**supplied,
|
||||
}
|
||||
return {name: value for name, value in kwargs.items() if value is not None or name in ("safeguards", *case.nones)}
|
||||
|
|
@ -3839,7 +3893,9 @@ async def _invoke(
|
|||
if isinstance(result, dict):
|
||||
body = cast(Mapping[str, object], result)
|
||||
return _response(body, case.private_keys), _hidden_params(body)
|
||||
time.sleep(case.busy_before_reading) # noqa: ASYNC251 # holds the event loop, as a caller's own work between the call and its first read does
|
||||
time.sleep(
|
||||
case.busy_before_reading
|
||||
) # holds the event loop, as a caller's own work between the call and its first read does
|
||||
chunks, failure = await _consume_async(cast(AsyncIterator[object], result))
|
||||
hidden: Final = _hidden_params(result)
|
||||
partition: Final = {"partition": _partition(chunks)} if case.partitioned else {}
|
||||
|
|
|
|||
|
|
@ -974,14 +974,14 @@ def no_tls_settings(monkeypatch: pytest.MonkeyPatch) -> None:
|
|||
"client", (None, object(), httpx.AsyncClient(), HTTPHandler()), ids=("none", "object", "httpx", "sync-handler")
|
||||
)
|
||||
def test_the_native_client_stands_in_for_any_client_python_ignores(client: object) -> None:
|
||||
assert messages_host.transport_handler({"client": client}) is None
|
||||
assert messages_host.transport_handler({"client": client}) == (None, [])
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("no_tls_settings")
|
||||
def test_a_caller_async_handler_is_the_handler_python_posts_through() -> None:
|
||||
client: Final = AsyncHTTPHandler()
|
||||
|
||||
assert messages_host.transport_handler({"client": client}) is client
|
||||
assert messages_host.transport_handler({"client": client}) == (client, [])
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("no_tls_settings")
|
||||
|
|
@ -995,7 +995,7 @@ def test_tls_the_native_client_cannot_apply_posts_through_pythons_anthropic_hand
|
|||
monkeypatch.setattr(litellm, name, value)
|
||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||
|
||||
handler: Final = messages_host.transport_handler({"client": object()})
|
||||
handler, _ = messages_host.transport_handler({"client": object()})
|
||||
|
||||
assert handler is get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC)
|
||||
assert callbacks_legacy_python.times_provider_exchange(object(), None) is False
|
||||
|
|
@ -1012,7 +1012,7 @@ def fips_cipher_exclusion() -> Iterator[None]:
|
|||
def test_fips_cipher_exclusion_posts_through_pythons_anthropic_handler(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||
|
||||
handler: Final = messages_host.transport_handler({"client": None})
|
||||
handler, _ = messages_host.transport_handler({"client": None})
|
||||
|
||||
assert handler is get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC)
|
||||
assert callbacks_legacy_python.times_provider_exchange(None, None) is False
|
||||
|
|
@ -1036,7 +1036,7 @@ def test_a_non_positive_timeout_posts_through_pythons_anthropic_handler(
|
|||
) -> None:
|
||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||
|
||||
handler: Final = messages_host.transport_handler({"client": None, "timeout": timeout})
|
||||
handler, _ = messages_host.transport_handler({"client": None, "timeout": timeout})
|
||||
|
||||
assert handler is get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC)
|
||||
assert callbacks_legacy_python.times_provider_exchange(None, timeout) is False
|
||||
|
|
@ -1045,7 +1045,97 @@ def test_a_non_positive_timeout_posts_through_pythons_anthropic_handler(
|
|||
@pytest.mark.usefixtures("no_tls_settings")
|
||||
@pytest.mark.parametrize("timeout", (None, 0.5, httpx.Timeout(10.0)), ids=("unset", "positive", "httpx"))
|
||||
def test_a_positive_timeout_keeps_the_native_client(timeout: object) -> None:
|
||||
assert messages_host.transport_handler({"client": None, "timeout": timeout}) is None
|
||||
assert messages_host.transport_handler({"client": None, "timeout": timeout}) == (None, [])
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("no_tls_settings")
|
||||
@pytest.mark.parametrize(
|
||||
"api_base",
|
||||
(
|
||||
"http://user:secret@host.example/d",
|
||||
"http://user@host.example/d",
|
||||
"http:///d",
|
||||
"http://:443/d",
|
||||
"http://host.example:99999/d",
|
||||
"http://host.example:port/d",
|
||||
),
|
||||
ids=("credentials", "user", "no-host", "port-only", "port-out-of-range", "port-not-a-number"),
|
||||
)
|
||||
@pytest.mark.parametrize("model", ("anthropic/claude-x", "azure_ai/claude-x"))
|
||||
def test_a_base_the_native_parser_reads_otherwise_posts_through_pythons_handler(
|
||||
monkeypatch: pytest.MonkeyPatch, api_base: str, model: str
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||
|
||||
handler, read = messages_host.transport_handler({"client": None, "model": model, "api_base": api_base})
|
||||
|
||||
assert handler is get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC)
|
||||
assert read == []
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("no_tls_settings")
|
||||
@pytest.mark.parametrize(
|
||||
"api_base",
|
||||
("https://host.example/d", "http://host.example:8080/d?x=1", "https://[::1]:443/d"),
|
||||
ids=("plain", "port-and-query", "ipv6"),
|
||||
)
|
||||
@pytest.mark.parametrize("model", ("anthropic/claude-x", "azure_ai/claude-x"))
|
||||
def test_a_plain_base_keeps_the_native_client(api_base: str, model: str) -> None:
|
||||
assert messages_host.transport_handler({"client": None, "model": model, "api_base": api_base}) == (None, [])
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("no_tls_settings")
|
||||
@pytest.mark.parametrize(
|
||||
("environment", "hosted"),
|
||||
((" https://env.example/d ", True), ("https://env.example/d", False), (None, False)),
|
||||
ids=("padded", "plain", "unset"),
|
||||
)
|
||||
@pytest.mark.parametrize("api_base", (None, ""), ids=("absent", "empty"))
|
||||
def test_azure_ai_reads_azure_api_base_once_and_hands_it_to_the_native_route(
|
||||
monkeypatch: pytest.MonkeyPatch, environment: str | None, hosted: bool, api_base: str | None
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||
reads: Final[list[str]] = []
|
||||
|
||||
def get_secret_str(name: str) -> str | None:
|
||||
reads.append(name)
|
||||
return environment
|
||||
|
||||
monkeypatch.setattr(messages_host, "get_secret_str", get_secret_str)
|
||||
|
||||
handler, read = messages_host.transport_handler(
|
||||
{"client": None, "model": "azure_ai/claude-x", "api_base": api_base}
|
||||
)
|
||||
|
||||
assert (handler is not None) is hosted
|
||||
assert read == [("AZURE_API_BASE", environment)]
|
||||
assert reads == ["AZURE_API_BASE"]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("no_tls_settings")
|
||||
@pytest.mark.parametrize(
|
||||
"request_",
|
||||
(
|
||||
{"client": None, "model": "anthropic/claude-x"},
|
||||
{"client": None, "model": "azure_ai/claude-x", "api_base": "https://call.example/d"},
|
||||
{"client": AsyncHTTPHandler(), "model": "azure_ai/claude-x"},
|
||||
{"client": None, "model": "azure_ai/claude-x", "timeout": 0},
|
||||
),
|
||||
ids=("anthropic", "azure-call-base", "azure-caller-handler", "azure-python-timeout"),
|
||||
)
|
||||
def test_azure_api_base_is_left_to_the_native_route_when_the_choice_does_not_need_it(
|
||||
monkeypatch: pytest.MonkeyPatch, request_: Mapping[str, object]
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||
|
||||
def get_secret_str(name: str) -> str | None:
|
||||
raise AssertionError(name)
|
||||
|
||||
monkeypatch.setattr(messages_host, "get_secret_str", get_secret_str)
|
||||
|
||||
_, read = messages_host.transport_handler(request_)
|
||||
|
||||
assert read == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue