From adaa460566af6fbca0d7dedacfcf25d1a695ef9c Mon Sep 17 00:00:00 2001 From: maciej-berri Date: Mon, 16 Nov 2026 20:03:08 -0800 Subject: [PATCH] fix(messages): load the anthropic beta map only when a filter has a beta to filter (#52188) * fix(messages): load the anthropic beta map only when a filter has a beta to filter Since #52042 the Rust messages route asked Python for the beta map on every azure_ai, Bedrock Invoke and Foundry call while decoding the request, so the first such call on each worker downloaded the remote map even when it sent no anthropic-beta, and the download ran synchronously on the request path. The bridge now installs the map only once Python has loaded it. When a filter first meets a beta and no map is installed, it asks the host to load it on the loader thread and uses the compiled table for that call, as before #52042. * fix(messages): start at most one background beta map load at a time A burst of beta-carrying calls while the first download was still in flight queued one load task per call on the loader thread. A non-blocking lock now lets only one load wait in the queue, and it is released when that load ends. --- litellm/anthropic_beta_headers_manager.py | 19 ++++- .../crates/llms/src/anthropic/beta_headers.rs | 21 +++++- .../tests/beta_headers_config_on_demand.rs | 34 +++++++++ .../python-bridge/src/routes/messages/host.rs | 36 +++++---- litellm/rust_bridge/host/routes/messages.py | 26 ++----- .../rust_bridge/host/routes/test_messages.py | 74 +++++++++++++------ 6 files changed, 149 insertions(+), 61 deletions(-) create mode 100644 litellm/rust/crates/llms/tests/beta_headers_config_on_demand.rs diff --git a/litellm/anthropic_beta_headers_manager.py b/litellm/anthropic_beta_headers_manager.py index b93c88cb935..2f350bb9718 100644 --- a/litellm/anthropic_beta_headers_manager.py +++ b/litellm/anthropic_beta_headers_manager.py @@ -40,9 +40,10 @@ from litellm._logging import verbose_logger from litellm.litellm_core_utils.offline_mode import is_offline_mode # Cache for the loaded configuration -_BETA_HEADERS_CONFIG: dict | None = None +_BETA_HEADERS_CONFIG: dict[str, object] | None = None _BETA_HEADERS_CONFIG_LOCK: Final = threading.Lock() _BETA_HEADERS_CONFIG_LOADER: Final = ThreadPoolExecutor(max_workers=1, thread_name_prefix="litellm-beta-headers-config") +_BACKGROUND_LOAD_IN_FLIGHT: Final = threading.Lock() BetaHeadersConfig: TypeAlias = Mapping[str, str | Mapping[str, str | None]] _BETA_HEADERS_CONFIG_ADAPTER: Final = TypeAdapter(BetaHeadersConfig) @@ -210,6 +211,22 @@ async def aload_beta_headers_config() -> None: await asyncio.get_running_loop().run_in_executor(_BETA_HEADERS_CONFIG_LOADER, _load_beta_headers_config) +def loaded_beta_headers_config() -> Mapping[str, object] | None: + return _BETA_HEADERS_CONFIG + + +def _cache_beta_headers_config() -> None: + try: + _load_beta_headers_config() + finally: + _BACKGROUND_LOAD_IN_FLIGHT.release() + + +def load_beta_headers_config_in_background() -> None: + if _BETA_HEADERS_CONFIG is None and _BACKGROUND_LOAD_IN_FLIGHT.acquire(blocking=False): + _ = _BETA_HEADERS_CONFIG_LOADER.submit(_cache_beta_headers_config) + + @cache def _bundled_beta_headers_config() -> BetaHeadersConfig: return _BETA_HEADERS_CONFIG_ADAPTER.validate_python(GetAnthropicBetaHeadersConfig.load_local_beta_headers_config()) diff --git a/litellm/rust/crates/llms/src/anthropic/beta_headers.rs b/litellm/rust/crates/llms/src/anthropic/beta_headers.rs index e1aa9080e0f..2c852aafa74 100644 --- a/litellm/rust/crates/llms/src/anthropic/beta_headers.rs +++ b/litellm/rust/crates/llms/src/anthropic/beta_headers.rs @@ -1,6 +1,6 @@ use std::{ collections::HashMap, - sync::{Arc, PoisonError, RwLock}, + sync::{Arc, OnceLock, PoisonError, RwLock}, }; use litellm_http::request::{with_header, without_headers}; @@ -58,6 +58,9 @@ impl BetaPolicy { } pub fn accepted_or_unlisted(betas: BetaSet, provider: BetaProvider) -> BetaSet { + if betas.is_empty() { + return betas; + } let loaded = loaded_config(); betas .into_iter() @@ -107,6 +110,12 @@ impl BetaHeadersConfig { } static LOADED_CONFIG: RwLock>> = RwLock::new(None); +static REQUEST_LOAD: OnceLock = OnceLock::new(); + +/// Registers how a filter that runs before the beta map is installed asks the host to load it. +pub fn on_beta_headers_config_needed(request_load: fn()) { + let _ = REQUEST_LOAD.set(request_load); +} pub fn install_beta_headers_config(config: BetaHeadersConfig) { *LOADED_CONFIG @@ -115,10 +124,16 @@ pub fn install_beta_headers_config(config: BetaHeadersConfig) { } fn loaded_config() -> Option> { - LOADED_CONFIG + let loaded = LOADED_CONFIG .read() .unwrap_or_else(PoisonError::into_inner) - .clone() + .clone(); + if loaded.is_none() + && let Some(request_load) = REQUEST_LOAD.get() + { + request_load(); + } + loaded } /// Python's `filter_and_transform_beta_headers` for one beta: the loaded config's cell, or the diff --git a/litellm/rust/crates/llms/tests/beta_headers_config_on_demand.rs b/litellm/rust/crates/llms/tests/beta_headers_config_on_demand.rs new file mode 100644 index 00000000000..1023d2a3f1d --- /dev/null +++ b/litellm/rust/crates/llms/tests/beta_headers_config_on_demand.rs @@ -0,0 +1,34 @@ +use std::sync::atomic::{AtomicUsize, Ordering}; + +use litellm_llms::anthropic::beta_headers::{ + BetaPolicy, accepted_or_unlisted, on_beta_headers_config_needed, +}; +use litellm_llms_types::providers::anthropic::{BetaProvider, BetaSet}; + +static LOAD_REQUESTS: AtomicUsize = AtomicUsize::new(0); + +fn count_load_request() { + LOAD_REQUESTS.fetch_add(1, Ordering::SeqCst); +} + +fn headers(beta: Option<&str>) -> Vec<(String, String)> { + let plain = ("content-type".to_string(), "application/json".to_string()); + match beta { + Some(beta) => vec![plain, ("anthropic-beta".to_string(), beta.to_string())], + None => vec![plain], + } +} + +#[test] +fn only_a_filter_with_a_beta_asks_for_the_unloaded_map() { + on_beta_headers_config_needed(count_load_request); + + let unfiltered = BetaPolicy::Filter(BetaProvider::AzureAi).apply(headers(None)); + let foundry_unfiltered = accepted_or_unlisted(BetaSet::default(), BetaProvider::AzureAi); + assert_eq!(unfiltered, headers(None)); + assert!(foundry_unfiltered.is_empty()); + assert_eq!(LOAD_REQUESTS.load(Ordering::SeqCst), 0); + + let _ = BetaPolicy::Filter(BetaProvider::Bedrock).apply(headers(Some("effort-2025-11-24"))); + assert!(LOAD_REQUESTS.load(Ordering::SeqCst) >= 1); +} diff --git a/litellm/rust/crates/python-bridge/src/routes/messages/host.rs b/litellm/rust/crates/python-bridge/src/routes/messages/host.rs index 2efa39fafa4..a63f1ccb4d4 100644 --- a/litellm/rust/crates/python-bridge/src/routes/messages/host.rs +++ b/litellm/rust/crates/python-bridge/src/routes/messages/host.rs @@ -234,19 +234,17 @@ fn workload_identity_fields(py: Python<'_>) -> PyResult<&'static [String]> { .map(Vec::as_slice) } -/// Installs the beta map Python loaded for a call whose betas are filtered, converting it again -/// only when a reload replaced it. -fn install_beta_headers_config( - py: Python<'_>, - model: &str, - custom_llm_provider: Option<&str>, - api_base: Option<&str>, -) -> PyResult<()> { +/// Installs the beta map once Python has loaded it. Loading starts only when a filter has a beta +/// to filter, off the calling thread, so a call that sends no beta never fetches the remote map. +fn install_loaded_beta_headers_config(py: Python<'_>) -> PyResult<()> { static INSTALLED: Mutex>> = Mutex::new(None); + litellm_llms::anthropic::beta_headers::on_beta_headers_config_needed( + request_beta_headers_config, + ); let loaded = py .import(ROUTE_HOST_MODULE)? - .getattr("beta_headers_config")? - .call1((model, custom_llm_provider, api_base))?; + .getattr("loaded_beta_headers_config")? + .call0()?; if loaded.is_none() { return Ok(()); } @@ -262,6 +260,17 @@ fn install_beta_headers_config( Ok(()) } +fn request_beta_headers_config() { + Python::attach(|py| { + if let Err(error) = py + .import(ROUTE_HOST_MODULE) + .and_then(|module| module.call_method0("load_beta_headers_config_in_background")) + { + error.write_unraisable(py, None); + } + }); +} + fn module_global<'py>(py: Python<'py>, name: &str) -> PyResult>> { PythonSettings::ProviderDefaults .read_or_unset(py)? @@ -656,12 +665,7 @@ impl MessagesPythonHost { .getattr("filters_beta_headers")? .call1((&model, custom_llm_provider.as_deref(), api_base.as_deref()))? .extract()?; - install_beta_headers_config( - py, - &model, - custom_llm_provider.as_deref(), - api_base.as_deref(), - )?; + install_loaded_beta_headers_config(py)?; let (extra_headers, mut unencodable_request_header) = match settings.extra_headers { Some(headers) => (Some(headers), None), None => self.merged_headers(py, arguments, filters_beta)?, diff --git a/litellm/rust_bridge/host/routes/messages.py b/litellm/rust_bridge/host/routes/messages.py index 954866a00fd..c1a1477c7c0 100644 --- a/litellm/rust_bridge/host/routes/messages.py +++ b/litellm/rust_bridge/host/routes/messages.py @@ -24,11 +24,8 @@ from aiohttp.http_exceptions import ContentLengthError, TransferEncodingError from pydantic import JsonValue, TypeAdapter, ValidationError import litellm +from litellm import anthropic_beta_headers_manager from litellm._logging import verbose_logger -from litellm.anthropic_beta_headers_manager import ( - _load_beta_headers_config, # pyright: ignore[reportPrivateUsage, reportUnknownVariableType] # the manager's own cached loader, typed bare dict - resolve_beta_provider, -) from litellm.caching.caching_handler import LLMCachingHandler from litellm.constants import ( ANTHROPIC_MESSAGES_DETACHED_STREAM_DRAIN_TIMEOUT_SECONDS, @@ -1251,21 +1248,12 @@ def filters_beta_headers(model: str, custom_llm_provider: str | None, api_base: return config is not None and config.should_filter_anthropic_beta_headers() -def beta_headers_config( - model: str, custom_llm_provider: str | None, api_base: str | None -) -> Mapping[str, object] | None: - """The beta map Python's `anthropic-beta` filter reads, loaded as the handler loads it, for a call whose betas - are filtered. ``None`` when the call filters none.""" - affinity_api_base: Final = ( - AnthropicModelInfo.get_api_base(api_base) if custom_llm_provider == "anthropic" else api_base - ) - takes_another_beta_row: Final = ( - custom_llm_provider is not None - and resolve_beta_provider(custom_llm_provider, affinity_api_base) != custom_llm_provider - ) - if not takes_another_beta_row and not filters_beta_headers(model, custom_llm_provider, api_base): - return None - return _load_beta_headers_config() # pyright: ignore[reportUnknownVariableType] # the loader returns a bare dict +def loaded_beta_headers_config() -> Mapping[str, object] | None: + return anthropic_beta_headers_manager.loaded_beta_headers_config() + + +def load_beta_headers_config_in_background() -> None: + anthropic_beta_headers_manager.load_beta_headers_config_in_background() def record_guardrail(public: Mapping[str, object], request: Mapping[str, object]) -> None: diff --git a/tests/unit/rust_bridge/host/routes/test_messages.py b/tests/unit/rust_bridge/host/routes/test_messages.py index 28ae550d98d..62126b75b86 100644 --- a/tests/unit/rust_bridge/host/routes/test_messages.py +++ b/tests/unit/rust_bridge/host/routes/test_messages.py @@ -5,11 +5,13 @@ import functools import json import logging import os +import queue import socket import ssl import struct import threading from collections.abc import Callable, Iterator, Mapping, Sequence +from concurrent.futures import Future, ThreadPoolExecutor from dataclasses import dataclass from types import MappingProxyType from typing import Final @@ -299,31 +301,59 @@ def test_filters_beta_headers_is_what_the_provider_config_says(model: str, provi assert messages_host.filters_beta_headers(model, provider, None) is expected -@pytest.mark.parametrize( - ("model", "provider", "api_base", "carried"), - [ - pytest.param("anthropic/claude-sonnet-4-5", "anthropic", None, False, id="anthropic"), - pytest.param( - "anthropic/claude-sonnet-4-5", - "anthropic", - "https://example-resource.services.ai.azure.com/anthropic", - True, - id="anthropic_foundry", - ), - pytest.param("azure_ai/claude-sonnet-4-5", "azure_ai", None, True, id="azure_ai"), - pytest.param(_BEDROCK_INVOKE, "bedrock", None, True, id="bedrock_invoke"), - ], -) -def test_beta_headers_config_is_the_map_python_loaded_for_a_filtering_call( - monkeypatch: pytest.MonkeyPatch, model: str, provider: str, api_base: str | None, carried: bool +def test_the_beta_map_stays_unloaded_until_a_filter_asks_and_then_loads_off_the_calling_thread( + monkeypatch: pytest.MonkeyPatch, ) -> None: - loaded: Final = {"provider_aliases": {}, "azure_ai": {"advisor-tool-2026-03-01": "advisor-tool-2026-03-01"}} - monkeypatch.setattr(anthropic_beta_headers_manager, "_BETA_HEADERS_CONFIG", loaded) + remote: Final[dict[str, object]] = {"provider_aliases": {}, "azure_ai": {"advisor-tool-2026-03-01": None}} + fetched_on: Final = queue.SimpleQueue[str]() + release: Final = threading.Event() - config: Final = messages_host.beta_headers_config(model, provider, api_base) + def fetch(url: str) -> dict[str, object]: + fetched_on.put(threading.current_thread().name) + assert release.wait(timeout=10) + return remote - assert (config is loaded) is carried - assert carried or config is None + monkeypatch.setattr(anthropic_beta_headers_manager, "_BETA_HEADERS_CONFIG", None) + monkeypatch.setattr(anthropic_beta_headers_manager, "get_beta_headers_config", fetch) + + assert messages_host.loaded_beta_headers_config() is None + assert fetched_on.empty() + + messages_host.load_beta_headers_config_in_background() + + assert fetched_on.get(timeout=10) != threading.current_thread().name + assert messages_host.loaded_beta_headers_config() is None + release.set() + assert anthropic_beta_headers_manager.get_provider_name("azure_ai") == "azure_ai" + assert messages_host.loaded_beta_headers_config() is remote + assert fetched_on.empty() + + +def test_repeated_load_requests_during_a_slow_download_queue_a_single_load(monkeypatch: pytest.MonkeyPatch) -> None: + release: Final = threading.Event() + submitted: Final = queue.SimpleQueue[str]() + loader: Final = ThreadPoolExecutor(max_workers=1) + + class RecordingLoader: + def submit(self, load: Callable[[], None]) -> Future[None]: + submitted.put(load.__name__) + return loader.submit(load) + + def fetch(url: str) -> dict[str, object]: + assert release.wait(timeout=10) + return {"provider_aliases": {}} + + monkeypatch.setattr(anthropic_beta_headers_manager, "_BETA_HEADERS_CONFIG", None) + monkeypatch.setattr(anthropic_beta_headers_manager, "get_beta_headers_config", fetch) + monkeypatch.setattr(anthropic_beta_headers_manager, "_BETA_HEADERS_CONFIG_LOADER", RecordingLoader()) + + for _ in range(20): + messages_host.load_beta_headers_config_in_background() + release.set() + loader.shutdown(wait=True) + + assert submitted.qsize() == 1 + assert messages_host.loaded_beta_headers_config() is not None @pytest.mark.parametrize(