fix(messages): load the anthropic beta map only when a filter has a beta to filter (#52188)

* fix(messages): load the anthropic beta map only when a filter has a beta to filter

Since #52042 the Rust messages route asked Python for the beta map on every
azure_ai, Bedrock Invoke and Foundry call while decoding the request, so the
first such call on each worker downloaded the remote map even when it sent no
anthropic-beta, and the download ran synchronously on the request path.

The bridge now installs the map only once Python has loaded it. When a filter
first meets a beta and no map is installed, it asks the host to load it on the
loader thread and uses the compiled table for that call, as before #52042.

* fix(messages): start at most one background beta map load at a time

A burst of beta-carrying calls while the first download was still in flight
queued one load task per call on the loader thread. A non-blocking lock now
lets only one load wait in the queue, and it is released when that load ends.
This commit is contained in:
maciej-berri 2026-11-16 20:03:08 -08:00 • committed by GitHub
parent 8b0bb6ae42
commit adaa460566
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 149 additions and 61 deletions

View file

@ -40,9 +40,10 @@ from litellm._logging import verbose_logger
from litellm.litellm_core_utils.offline_mode import is_offline_mode
# Cache for the loaded configuration
_BETA_HEADERS_CONFIG: dict | None = None
_BETA_HEADERS_CONFIG: dict[str, object] | None = None
_BETA_HEADERS_CONFIG_LOCK: Final = threading.Lock()
_BETA_HEADERS_CONFIG_LOADER: Final = ThreadPoolExecutor(max_workers=1, thread_name_prefix="litellm-beta-headers-config")
_BACKGROUND_LOAD_IN_FLIGHT: Final = threading.Lock()
BetaHeadersConfig: TypeAlias = Mapping[str, str | Mapping[str, str | None]]
_BETA_HEADERS_CONFIG_ADAPTER: Final = TypeAdapter(BetaHeadersConfig)
@ -210,6 +211,22 @@ async def aload_beta_headers_config() -> None:
await asyncio.get_running_loop().run_in_executor(_BETA_HEADERS_CONFIG_LOADER, _load_beta_headers_config)
def loaded_beta_headers_config() -> Mapping[str, object] | None:
return _BETA_HEADERS_CONFIG
def _cache_beta_headers_config() -> None:
try:
_load_beta_headers_config()
finally:
_BACKGROUND_LOAD_IN_FLIGHT.release()
def load_beta_headers_config_in_background() -> None:
if _BETA_HEADERS_CONFIG is None and _BACKGROUND_LOAD_IN_FLIGHT.acquire(blocking=False):
_ = _BETA_HEADERS_CONFIG_LOADER.submit(_cache_beta_headers_config)
@cache
def _bundled_beta_headers_config() -> BetaHeadersConfig:
return _BETA_HEADERS_CONFIG_ADAPTER.validate_python(GetAnthropicBetaHeadersConfig.load_local_beta_headers_config())

View file

@ -1,6 +1,6 @@
use std::{
collections::HashMap,
sync::{Arc, PoisonError, RwLock},
sync::{Arc, OnceLock, PoisonError, RwLock},
};
use litellm_http::request::{with_header, without_headers};
@ -58,6 +58,9 @@ impl BetaPolicy {
}
pub fn accepted_or_unlisted(betas: BetaSet, provider: BetaProvider) -> BetaSet {
if betas.is_empty() {
return betas;
}
let loaded = loaded_config();
betas
.into_iter()
@ -107,6 +110,12 @@ impl BetaHeadersConfig {
}
static LOADED_CONFIG: RwLock<Option<Arc<BetaHeadersConfig>>> = RwLock::new(None);
static REQUEST_LOAD: OnceLock<fn()> = OnceLock::new();
/// Registers how a filter that runs before the beta map is installed asks the host to load it.
pub fn on_beta_headers_config_needed(request_load: fn()) {
let _ = REQUEST_LOAD.set(request_load);
}
pub fn install_beta_headers_config(config: BetaHeadersConfig) {
*LOADED_CONFIG
@ -115,10 +124,16 @@ pub fn install_beta_headers_config(config: BetaHeadersConfig) {
}
fn loaded_config() -> Option<Arc<BetaHeadersConfig>> {
LOADED_CONFIG
let loaded = LOADED_CONFIG
.read()
.unwrap_or_else(PoisonError::into_inner)
.clone()
.clone();
if loaded.is_none()
&& let Some(request_load) = REQUEST_LOAD.get()
{
request_load();
}
loaded
}
/// Python's `filter_and_transform_beta_headers` for one beta: the loaded config's cell, or the

View file

@ -0,0 +1,34 @@
use std::sync::atomic::{AtomicUsize, Ordering};
use litellm_llms::anthropic::beta_headers::{
BetaPolicy, accepted_or_unlisted, on_beta_headers_config_needed,
};
use litellm_llms_types::providers::anthropic::{BetaProvider, BetaSet};
static LOAD_REQUESTS: AtomicUsize = AtomicUsize::new(0);
fn count_load_request() {
LOAD_REQUESTS.fetch_add(1, Ordering::SeqCst);
}
fn headers(beta: Option<&str>) -> Vec<(String, String)> {
let plain = ("content-type".to_string(), "application/json".to_string());
match beta {
Some(beta) => vec![plain, ("anthropic-beta".to_string(), beta.to_string())],
None => vec![plain],
}
}
#[test]
fn only_a_filter_with_a_beta_asks_for_the_unloaded_map() {
on_beta_headers_config_needed(count_load_request);
let unfiltered = BetaPolicy::Filter(BetaProvider::AzureAi).apply(headers(None));
let foundry_unfiltered = accepted_or_unlisted(BetaSet::default(), BetaProvider::AzureAi);
assert_eq!(unfiltered, headers(None));
assert!(foundry_unfiltered.is_empty());
assert_eq!(LOAD_REQUESTS.load(Ordering::SeqCst), 0);
let _ = BetaPolicy::Filter(BetaProvider::Bedrock).apply(headers(Some("effort-2025-11-24")));
assert!(LOAD_REQUESTS.load(Ordering::SeqCst) >= 1);
}

View file

@ -234,19 +234,17 @@ fn workload_identity_fields(py: Python<'_>) -> PyResult<&'static [String]> {
.map(Vec::as_slice)
}
/// Installs the beta map Python loaded for a call whose betas are filtered, converting it again
/// only when a reload replaced it.
fn install_beta_headers_config(
py: Python<'_>,
model: &str,
custom_llm_provider: Option<&str>,
api_base: Option<&str>,
) -> PyResult<()> {
/// Installs the beta map once Python has loaded it. Loading starts only when a filter has a beta
/// to filter, off the calling thread, so a call that sends no beta never fetches the remote map.
fn install_loaded_beta_headers_config(py: Python<'_>) -> PyResult<()> {
static INSTALLED: Mutex<Option<Py<PyAny>>> = Mutex::new(None);
litellm_llms::anthropic::beta_headers::on_beta_headers_config_needed(
request_beta_headers_config,
);
let loaded = py
.import(ROUTE_HOST_MODULE)?
.getattr("beta_headers_config")?
.call1((model, custom_llm_provider, api_base))?;
.getattr("loaded_beta_headers_config")?
.call0()?;
if loaded.is_none() {
return Ok(());
}
@ -262,6 +260,17 @@ fn install_beta_headers_config(
Ok(())
}
fn request_beta_headers_config() {
Python::attach(|py| {
if let Err(error) = py
.import(ROUTE_HOST_MODULE)
.and_then(|module| module.call_method0("load_beta_headers_config_in_background"))
{
error.write_unraisable(py, None);
}
});
}
fn module_global<'py>(py: Python<'py>, name: &str) -> PyResult<Option<Bound<'py, PyAny>>> {
PythonSettings::ProviderDefaults
.read_or_unset(py)?
@ -656,12 +665,7 @@ impl MessagesPythonHost {
.getattr("filters_beta_headers")?
.call1((&model, custom_llm_provider.as_deref(), api_base.as_deref()))?
.extract()?;
install_beta_headers_config(
py,
&model,
custom_llm_provider.as_deref(),
api_base.as_deref(),
)?;
install_loaded_beta_headers_config(py)?;
let (extra_headers, mut unencodable_request_header) = match settings.extra_headers {
Some(headers) => (Some(headers), None),
None => self.merged_headers(py, arguments, filters_beta)?,

View file

@ -24,11 +24,8 @@ from aiohttp.http_exceptions import ContentLengthError, TransferEncodingError
from pydantic import JsonValue, TypeAdapter, ValidationError
import litellm
from litellm import anthropic_beta_headers_manager
from litellm._logging import verbose_logger
from litellm.anthropic_beta_headers_manager import (
_load_beta_headers_config, # pyright: ignore[reportPrivateUsage, reportUnknownVariableType] # the manager's own cached loader, typed bare dict
resolve_beta_provider,
)
from litellm.caching.caching_handler import LLMCachingHandler
from litellm.constants import (
ANTHROPIC_MESSAGES_DETACHED_STREAM_DRAIN_TIMEOUT_SECONDS,
@ -1251,21 +1248,12 @@ def filters_beta_headers(model: str, custom_llm_provider: str | None, api_base:
return config is not None and config.should_filter_anthropic_beta_headers()
def beta_headers_config(
model: str, custom_llm_provider: str | None, api_base: str | None
) -> Mapping[str, object] | None:
"""The beta map Python's `anthropic-beta` filter reads, loaded as the handler loads it, for a call whose betas
are filtered. ``None`` when the call filters none."""
affinity_api_base: Final = (
AnthropicModelInfo.get_api_base(api_base) if custom_llm_provider == "anthropic" else api_base
)
takes_another_beta_row: Final = (
custom_llm_provider is not None
and resolve_beta_provider(custom_llm_provider, affinity_api_base) != custom_llm_provider
)
if not takes_another_beta_row and not filters_beta_headers(model, custom_llm_provider, api_base):
return None
return _load_beta_headers_config() # pyright: ignore[reportUnknownVariableType] # the loader returns a bare dict
def loaded_beta_headers_config() -> Mapping[str, object] | None:
return anthropic_beta_headers_manager.loaded_beta_headers_config()
def load_beta_headers_config_in_background() -> None:
anthropic_beta_headers_manager.load_beta_headers_config_in_background()
def record_guardrail(public: Mapping[str, object], request: Mapping[str, object]) -> None:

View file

@ -5,11 +5,13 @@ import functools
import json
import logging
import os
import queue
import socket
import ssl
import struct
import threading
from collections.abc import Callable, Iterator, Mapping, Sequence
from concurrent.futures import Future, ThreadPoolExecutor
from dataclasses import dataclass
from types import MappingProxyType
from typing import Final
@ -299,31 +301,59 @@ def test_filters_beta_headers_is_what_the_provider_config_says(model: str, provi
assert messages_host.filters_beta_headers(model, provider, None) is expected
@pytest.mark.parametrize(
("model", "provider", "api_base", "carried"),
[
pytest.param("anthropic/claude-sonnet-4-5", "anthropic", None, False, id="anthropic"),
pytest.param(
"anthropic/claude-sonnet-4-5",
"anthropic",
"https://example-resource.services.ai.azure.com/anthropic",
True,
id="anthropic_foundry",
),
pytest.param("azure_ai/claude-sonnet-4-5", "azure_ai", None, True, id="azure_ai"),
pytest.param(_BEDROCK_INVOKE, "bedrock", None, True, id="bedrock_invoke"),
],
)
def test_beta_headers_config_is_the_map_python_loaded_for_a_filtering_call(
monkeypatch: pytest.MonkeyPatch, model: str, provider: str, api_base: str | None, carried: bool
def test_the_beta_map_stays_unloaded_until_a_filter_asks_and_then_loads_off_the_calling_thread(
monkeypatch: pytest.MonkeyPatch,
) -> None:
loaded: Final = {"provider_aliases": {}, "azure_ai": {"advisor-tool-2026-03-01": "advisor-tool-2026-03-01"}}
monkeypatch.setattr(anthropic_beta_headers_manager, "_BETA_HEADERS_CONFIG", loaded)
remote: Final[dict[str, object]] = {"provider_aliases": {}, "azure_ai": {"advisor-tool-2026-03-01": None}}
fetched_on: Final = queue.SimpleQueue[str]()
release: Final = threading.Event()
config: Final = messages_host.beta_headers_config(model, provider, api_base)
def fetch(url: str) -> dict[str, object]:
fetched_on.put(threading.current_thread().name)
assert release.wait(timeout=10)
return remote
assert (config is loaded) is carried
assert carried or config is None
monkeypatch.setattr(anthropic_beta_headers_manager, "_BETA_HEADERS_CONFIG", None)
monkeypatch.setattr(anthropic_beta_headers_manager, "get_beta_headers_config", fetch)
assert messages_host.loaded_beta_headers_config() is None
assert fetched_on.empty()
messages_host.load_beta_headers_config_in_background()
assert fetched_on.get(timeout=10) != threading.current_thread().name
assert messages_host.loaded_beta_headers_config() is None
release.set()
assert anthropic_beta_headers_manager.get_provider_name("azure_ai") == "azure_ai"
assert messages_host.loaded_beta_headers_config() is remote
assert fetched_on.empty()
def test_repeated_load_requests_during_a_slow_download_queue_a_single_load(monkeypatch: pytest.MonkeyPatch) -> None:
release: Final = threading.Event()
submitted: Final = queue.SimpleQueue[str]()
loader: Final = ThreadPoolExecutor(max_workers=1)
class RecordingLoader:
def submit(self, load: Callable[[], None]) -> Future[None]:
submitted.put(load.__name__)
return loader.submit(load)
def fetch(url: str) -> dict[str, object]:
assert release.wait(timeout=10)
return {"provider_aliases": {}}
monkeypatch.setattr(anthropic_beta_headers_manager, "_BETA_HEADERS_CONFIG", None)
monkeypatch.setattr(anthropic_beta_headers_manager, "get_beta_headers_config", fetch)
monkeypatch.setattr(anthropic_beta_headers_manager, "_BETA_HEADERS_CONFIG_LOADER", RecordingLoader())
for _ in range(20):
messages_host.load_beta_headers_config_in_background()
release.set()
loader.shutdown(wait=True)
assert submitted.qsize() == 1
assert messages_host.loaded_beta_headers_config() is not None
@pytest.mark.parametrize(