mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(messages): load the anthropic beta map only when a filter has a beta to filter (#52188)
* fix(messages): load the anthropic beta map only when a filter has a beta to filter Since #52042 the Rust messages route asked Python for the beta map on every azure_ai, Bedrock Invoke and Foundry call while decoding the request, so the first such call on each worker downloaded the remote map even when it sent no anthropic-beta, and the download ran synchronously on the request path. The bridge now installs the map only once Python has loaded it. When a filter first meets a beta and no map is installed, it asks the host to load it on the loader thread and uses the compiled table for that call, as before #52042. * fix(messages): start at most one background beta map load at a time A burst of beta-carrying calls while the first download was still in flight queued one load task per call on the loader thread. A non-blocking lock now lets only one load wait in the queue, and it is released when that load ends.
This commit is contained in:
parent
8b0bb6ae42
commit
adaa460566
6 changed files with 149 additions and 61 deletions
|
|
@ -40,9 +40,10 @@ from litellm._logging import verbose_logger
|
|||
from litellm.litellm_core_utils.offline_mode import is_offline_mode
|
||||
|
||||
# Cache for the loaded configuration
|
||||
_BETA_HEADERS_CONFIG: dict | None = None
|
||||
_BETA_HEADERS_CONFIG: dict[str, object] | None = None
|
||||
_BETA_HEADERS_CONFIG_LOCK: Final = threading.Lock()
|
||||
_BETA_HEADERS_CONFIG_LOADER: Final = ThreadPoolExecutor(max_workers=1, thread_name_prefix="litellm-beta-headers-config")
|
||||
_BACKGROUND_LOAD_IN_FLIGHT: Final = threading.Lock()
|
||||
|
||||
BetaHeadersConfig: TypeAlias = Mapping[str, str | Mapping[str, str | None]]
|
||||
_BETA_HEADERS_CONFIG_ADAPTER: Final = TypeAdapter(BetaHeadersConfig)
|
||||
|
|
@ -210,6 +211,22 @@ async def aload_beta_headers_config() -> None:
|
|||
await asyncio.get_running_loop().run_in_executor(_BETA_HEADERS_CONFIG_LOADER, _load_beta_headers_config)
|
||||
|
||||
|
||||
def loaded_beta_headers_config() -> Mapping[str, object] | None:
|
||||
return _BETA_HEADERS_CONFIG
|
||||
|
||||
|
||||
def _cache_beta_headers_config() -> None:
|
||||
try:
|
||||
_load_beta_headers_config()
|
||||
finally:
|
||||
_BACKGROUND_LOAD_IN_FLIGHT.release()
|
||||
|
||||
|
||||
def load_beta_headers_config_in_background() -> None:
|
||||
if _BETA_HEADERS_CONFIG is None and _BACKGROUND_LOAD_IN_FLIGHT.acquire(blocking=False):
|
||||
_ = _BETA_HEADERS_CONFIG_LOADER.submit(_cache_beta_headers_config)
|
||||
|
||||
|
||||
@cache
|
||||
def _bundled_beta_headers_config() -> BetaHeadersConfig:
|
||||
return _BETA_HEADERS_CONFIG_ADAPTER.validate_python(GetAnthropicBetaHeadersConfig.load_local_beta_headers_config())
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
use std::{
|
||||
collections::HashMap,
|
||||
sync::{Arc, PoisonError, RwLock},
|
||||
sync::{Arc, OnceLock, PoisonError, RwLock},
|
||||
};
|
||||
|
||||
use litellm_http::request::{with_header, without_headers};
|
||||
|
|
@ -58,6 +58,9 @@ impl BetaPolicy {
|
|||
}
|
||||
|
||||
pub fn accepted_or_unlisted(betas: BetaSet, provider: BetaProvider) -> BetaSet {
|
||||
if betas.is_empty() {
|
||||
return betas;
|
||||
}
|
||||
let loaded = loaded_config();
|
||||
betas
|
||||
.into_iter()
|
||||
|
|
@ -107,6 +110,12 @@ impl BetaHeadersConfig {
|
|||
}
|
||||
|
||||
static LOADED_CONFIG: RwLock<Option<Arc<BetaHeadersConfig>>> = RwLock::new(None);
|
||||
static REQUEST_LOAD: OnceLock<fn()> = OnceLock::new();
|
||||
|
||||
/// Registers how a filter that runs before the beta map is installed asks the host to load it.
|
||||
pub fn on_beta_headers_config_needed(request_load: fn()) {
|
||||
let _ = REQUEST_LOAD.set(request_load);
|
||||
}
|
||||
|
||||
pub fn install_beta_headers_config(config: BetaHeadersConfig) {
|
||||
*LOADED_CONFIG
|
||||
|
|
@ -115,10 +124,16 @@ pub fn install_beta_headers_config(config: BetaHeadersConfig) {
|
|||
}
|
||||
|
||||
fn loaded_config() -> Option<Arc<BetaHeadersConfig>> {
|
||||
LOADED_CONFIG
|
||||
let loaded = LOADED_CONFIG
|
||||
.read()
|
||||
.unwrap_or_else(PoisonError::into_inner)
|
||||
.clone()
|
||||
.clone();
|
||||
if loaded.is_none()
|
||||
&& let Some(request_load) = REQUEST_LOAD.get()
|
||||
{
|
||||
request_load();
|
||||
}
|
||||
loaded
|
||||
}
|
||||
|
||||
/// Python's `filter_and_transform_beta_headers` for one beta: the loaded config's cell, or the
|
||||
|
|
|
|||
|
|
@ -0,0 +1,34 @@
|
|||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
use litellm_llms::anthropic::beta_headers::{
|
||||
BetaPolicy, accepted_or_unlisted, on_beta_headers_config_needed,
|
||||
};
|
||||
use litellm_llms_types::providers::anthropic::{BetaProvider, BetaSet};
|
||||
|
||||
static LOAD_REQUESTS: AtomicUsize = AtomicUsize::new(0);
|
||||
|
||||
fn count_load_request() {
|
||||
LOAD_REQUESTS.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
fn headers(beta: Option<&str>) -> Vec<(String, String)> {
|
||||
let plain = ("content-type".to_string(), "application/json".to_string());
|
||||
match beta {
|
||||
Some(beta) => vec![plain, ("anthropic-beta".to_string(), beta.to_string())],
|
||||
None => vec![plain],
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_a_filter_with_a_beta_asks_for_the_unloaded_map() {
|
||||
on_beta_headers_config_needed(count_load_request);
|
||||
|
||||
let unfiltered = BetaPolicy::Filter(BetaProvider::AzureAi).apply(headers(None));
|
||||
let foundry_unfiltered = accepted_or_unlisted(BetaSet::default(), BetaProvider::AzureAi);
|
||||
assert_eq!(unfiltered, headers(None));
|
||||
assert!(foundry_unfiltered.is_empty());
|
||||
assert_eq!(LOAD_REQUESTS.load(Ordering::SeqCst), 0);
|
||||
|
||||
let _ = BetaPolicy::Filter(BetaProvider::Bedrock).apply(headers(Some("effort-2025-11-24")));
|
||||
assert!(LOAD_REQUESTS.load(Ordering::SeqCst) >= 1);
|
||||
}
|
||||
|
|
@ -234,19 +234,17 @@ fn workload_identity_fields(py: Python<'_>) -> PyResult<&'static [String]> {
|
|||
.map(Vec::as_slice)
|
||||
}
|
||||
|
||||
/// Installs the beta map Python loaded for a call whose betas are filtered, converting it again
|
||||
/// only when a reload replaced it.
|
||||
fn install_beta_headers_config(
|
||||
py: Python<'_>,
|
||||
model: &str,
|
||||
custom_llm_provider: Option<&str>,
|
||||
api_base: Option<&str>,
|
||||
) -> PyResult<()> {
|
||||
/// Installs the beta map once Python has loaded it. Loading starts only when a filter has a beta
|
||||
/// to filter, off the calling thread, so a call that sends no beta never fetches the remote map.
|
||||
fn install_loaded_beta_headers_config(py: Python<'_>) -> PyResult<()> {
|
||||
static INSTALLED: Mutex<Option<Py<PyAny>>> = Mutex::new(None);
|
||||
litellm_llms::anthropic::beta_headers::on_beta_headers_config_needed(
|
||||
request_beta_headers_config,
|
||||
);
|
||||
let loaded = py
|
||||
.import(ROUTE_HOST_MODULE)?
|
||||
.getattr("beta_headers_config")?
|
||||
.call1((model, custom_llm_provider, api_base))?;
|
||||
.getattr("loaded_beta_headers_config")?
|
||||
.call0()?;
|
||||
if loaded.is_none() {
|
||||
return Ok(());
|
||||
}
|
||||
|
|
@ -262,6 +260,17 @@ fn install_beta_headers_config(
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn request_beta_headers_config() {
|
||||
Python::attach(|py| {
|
||||
if let Err(error) = py
|
||||
.import(ROUTE_HOST_MODULE)
|
||||
.and_then(|module| module.call_method0("load_beta_headers_config_in_background"))
|
||||
{
|
||||
error.write_unraisable(py, None);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn module_global<'py>(py: Python<'py>, name: &str) -> PyResult<Option<Bound<'py, PyAny>>> {
|
||||
PythonSettings::ProviderDefaults
|
||||
.read_or_unset(py)?
|
||||
|
|
@ -656,12 +665,7 @@ impl MessagesPythonHost {
|
|||
.getattr("filters_beta_headers")?
|
||||
.call1((&model, custom_llm_provider.as_deref(), api_base.as_deref()))?
|
||||
.extract()?;
|
||||
install_beta_headers_config(
|
||||
py,
|
||||
&model,
|
||||
custom_llm_provider.as_deref(),
|
||||
api_base.as_deref(),
|
||||
)?;
|
||||
install_loaded_beta_headers_config(py)?;
|
||||
let (extra_headers, mut unencodable_request_header) = match settings.extra_headers {
|
||||
Some(headers) => (Some(headers), None),
|
||||
None => self.merged_headers(py, arguments, filters_beta)?,
|
||||
|
|
|
|||
|
|
@ -24,11 +24,8 @@ from aiohttp.http_exceptions import ContentLengthError, TransferEncodingError
|
|||
from pydantic import JsonValue, TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm import anthropic_beta_headers_manager
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.anthropic_beta_headers_manager import (
|
||||
_load_beta_headers_config, # pyright: ignore[reportPrivateUsage, reportUnknownVariableType] # the manager's own cached loader, typed bare dict
|
||||
resolve_beta_provider,
|
||||
)
|
||||
from litellm.caching.caching_handler import LLMCachingHandler
|
||||
from litellm.constants import (
|
||||
ANTHROPIC_MESSAGES_DETACHED_STREAM_DRAIN_TIMEOUT_SECONDS,
|
||||
|
|
@ -1251,21 +1248,12 @@ def filters_beta_headers(model: str, custom_llm_provider: str | None, api_base:
|
|||
return config is not None and config.should_filter_anthropic_beta_headers()
|
||||
|
||||
|
||||
def beta_headers_config(
|
||||
model: str, custom_llm_provider: str | None, api_base: str | None
|
||||
) -> Mapping[str, object] | None:
|
||||
"""The beta map Python's `anthropic-beta` filter reads, loaded as the handler loads it, for a call whose betas
|
||||
are filtered. ``None`` when the call filters none."""
|
||||
affinity_api_base: Final = (
|
||||
AnthropicModelInfo.get_api_base(api_base) if custom_llm_provider == "anthropic" else api_base
|
||||
)
|
||||
takes_another_beta_row: Final = (
|
||||
custom_llm_provider is not None
|
||||
and resolve_beta_provider(custom_llm_provider, affinity_api_base) != custom_llm_provider
|
||||
)
|
||||
if not takes_another_beta_row and not filters_beta_headers(model, custom_llm_provider, api_base):
|
||||
return None
|
||||
return _load_beta_headers_config() # pyright: ignore[reportUnknownVariableType] # the loader returns a bare dict
|
||||
def loaded_beta_headers_config() -> Mapping[str, object] | None:
|
||||
return anthropic_beta_headers_manager.loaded_beta_headers_config()
|
||||
|
||||
|
||||
def load_beta_headers_config_in_background() -> None:
|
||||
anthropic_beta_headers_manager.load_beta_headers_config_in_background()
|
||||
|
||||
|
||||
def record_guardrail(public: Mapping[str, object], request: Mapping[str, object]) -> None:
|
||||
|
|
|
|||
|
|
@ -5,11 +5,13 @@ import functools
|
|||
import json
|
||||
import logging
|
||||
import os
|
||||
import queue
|
||||
import socket
|
||||
import ssl
|
||||
import struct
|
||||
import threading
|
||||
from collections.abc import Callable, Iterator, Mapping, Sequence
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
|
@ -299,31 +301,59 @@ def test_filters_beta_headers_is_what_the_provider_config_says(model: str, provi
|
|||
assert messages_host.filters_beta_headers(model, provider, None) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "provider", "api_base", "carried"),
|
||||
[
|
||||
pytest.param("anthropic/claude-sonnet-4-5", "anthropic", None, False, id="anthropic"),
|
||||
pytest.param(
|
||||
"anthropic/claude-sonnet-4-5",
|
||||
"anthropic",
|
||||
"https://example-resource.services.ai.azure.com/anthropic",
|
||||
True,
|
||||
id="anthropic_foundry",
|
||||
),
|
||||
pytest.param("azure_ai/claude-sonnet-4-5", "azure_ai", None, True, id="azure_ai"),
|
||||
pytest.param(_BEDROCK_INVOKE, "bedrock", None, True, id="bedrock_invoke"),
|
||||
],
|
||||
)
|
||||
def test_beta_headers_config_is_the_map_python_loaded_for_a_filtering_call(
|
||||
monkeypatch: pytest.MonkeyPatch, model: str, provider: str, api_base: str | None, carried: bool
|
||||
def test_the_beta_map_stays_unloaded_until_a_filter_asks_and_then_loads_off_the_calling_thread(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
loaded: Final = {"provider_aliases": {}, "azure_ai": {"advisor-tool-2026-03-01": "advisor-tool-2026-03-01"}}
|
||||
monkeypatch.setattr(anthropic_beta_headers_manager, "_BETA_HEADERS_CONFIG", loaded)
|
||||
remote: Final[dict[str, object]] = {"provider_aliases": {}, "azure_ai": {"advisor-tool-2026-03-01": None}}
|
||||
fetched_on: Final = queue.SimpleQueue[str]()
|
||||
release: Final = threading.Event()
|
||||
|
||||
config: Final = messages_host.beta_headers_config(model, provider, api_base)
|
||||
def fetch(url: str) -> dict[str, object]:
|
||||
fetched_on.put(threading.current_thread().name)
|
||||
assert release.wait(timeout=10)
|
||||
return remote
|
||||
|
||||
assert (config is loaded) is carried
|
||||
assert carried or config is None
|
||||
monkeypatch.setattr(anthropic_beta_headers_manager, "_BETA_HEADERS_CONFIG", None)
|
||||
monkeypatch.setattr(anthropic_beta_headers_manager, "get_beta_headers_config", fetch)
|
||||
|
||||
assert messages_host.loaded_beta_headers_config() is None
|
||||
assert fetched_on.empty()
|
||||
|
||||
messages_host.load_beta_headers_config_in_background()
|
||||
|
||||
assert fetched_on.get(timeout=10) != threading.current_thread().name
|
||||
assert messages_host.loaded_beta_headers_config() is None
|
||||
release.set()
|
||||
assert anthropic_beta_headers_manager.get_provider_name("azure_ai") == "azure_ai"
|
||||
assert messages_host.loaded_beta_headers_config() is remote
|
||||
assert fetched_on.empty()
|
||||
|
||||
|
||||
def test_repeated_load_requests_during_a_slow_download_queue_a_single_load(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
release: Final = threading.Event()
|
||||
submitted: Final = queue.SimpleQueue[str]()
|
||||
loader: Final = ThreadPoolExecutor(max_workers=1)
|
||||
|
||||
class RecordingLoader:
|
||||
def submit(self, load: Callable[[], None]) -> Future[None]:
|
||||
submitted.put(load.__name__)
|
||||
return loader.submit(load)
|
||||
|
||||
def fetch(url: str) -> dict[str, object]:
|
||||
assert release.wait(timeout=10)
|
||||
return {"provider_aliases": {}}
|
||||
|
||||
monkeypatch.setattr(anthropic_beta_headers_manager, "_BETA_HEADERS_CONFIG", None)
|
||||
monkeypatch.setattr(anthropic_beta_headers_manager, "get_beta_headers_config", fetch)
|
||||
monkeypatch.setattr(anthropic_beta_headers_manager, "_BETA_HEADERS_CONFIG_LOADER", RecordingLoader())
|
||||
|
||||
for _ in range(20):
|
||||
messages_host.load_beta_headers_config_in_background()
|
||||
release.set()
|
||||
loader.shutdown(wait=True)
|
||||
|
||||
assert submitted.qsize() == 1
|
||||
assert messages_host.loaded_beta_headers_config() is not None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue