scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-08 03:08:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 115

refactor(llms): expose public names for private provider helpers (#45037)

Browse source 
* refactor(litellm): migrate private usage in llms

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(litellm): preserve Bedrock batch signature marker

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* refactor(litellm): migrate llms private usage symbols

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(litellm): preserve UUID Watsonx project IDs

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(litellm): retarget llms mocks to public names

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* refactor(litellm): limit llms changes to renames and forwarders

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(litellm): keep GCS mock client patching private Vertex auth methods

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(litellm): assert forwarder arguments and type forwarder signatures

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: mateo <mateo@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot] 2026-10-07 08:49:25 -07:00 • committed by GitHub
parent da1053f068
commit e2971e0af4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
376 changed files with 4431 additions and 2434 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py
View file

@ -186,11 +186,11 @@ class BedrockAgentCoreA2ATransformation:
headers: Final[dict] = {}
session_id: Final = _validate_runtime_session_id(
_request_scoped_runtime_session_id(params, litellm_params)
or agentcore_config._get_runtime_session_id(optional_params),
or agentcore_config.get_runtime_session_id(optional_params),
model=model,
)
headers["X-Amzn-Bedrock-AgentCore-Runtime-Session-Id"] = session_id
runtime_user_id: Final = agentcore_config._get_runtime_user_id(optional_params)
runtime_user_id: Final = agentcore_config.get_runtime_user_id(optional_params)
if runtime_user_id:
headers["X-Amzn-Bedrock-AgentCore-Runtime-User-Id"] = runtime_user_id

2
litellm/batches/batch_utils.py
View file

@ -369,7 +369,7 @@ def calculate_vertex_ai_batch_cost_and_usage(
row,
model_name,
model_info=model_info,
calculate_usage=VertexGeminiConfig._calculate_usage,
calculate_usage=VertexGeminiConfig.calculate_usage,
cost_calculator=batch_cost_calculator,
)
for row in vertex_ai_batch_responses

8
litellm/batches/main.py
View file

@ -628,7 +628,7 @@ def retrieve_batch(
async_kwargs: Final = kwargs.copy()
async_kwargs.pop("aws_region_name", None)
return BedrockBatchesHandler._handle_async_invoke_status(
return BedrockBatchesHandler.handle_async_invoke_status(
batch_id=batch_id,
aws_region_name=kwargs.get("aws_region_name", "us-east-1"),
logging_obj=litellm_logging_obj,
@ -638,7 +638,7 @@ def retrieve_batch(
mij_kwargs: Final = kwargs.copy()
mij_kwargs.pop("aws_region_name", None)
return BedrockBatchesHandler._handle_model_invocation_job_status(
return BedrockBatchesHandler.handle_model_invocation_job_status(
batch_id=batch_id,
aws_region_name=kwargs.get("aws_region_name"),
logging_obj=litellm_logging_obj,
@ -1107,7 +1107,7 @@ def _handle_async_invoke_status(batch_id: str, aws_region_name: str, logging_obj
embedding_handler: Final = BedrockEmbedding()
# Get the status of the async invoke job
status_response: Final = await embedding_handler._get_async_invoke_status(
status_response: Final = await embedding_handler.get_async_invoke_status(
invocation_arn=batch_id,
aws_region_name=aws_region_name,
logging_obj=logging_obj,
@ -1151,7 +1151,7 @@ def _handle_async_invoke_status(batch_id: str, aws_region_name: str, logging_obj
failed_at,
_,
_,
) = BedrockBatchesConfig()._parse_timestamps_and_status(status_response, aws_status_raw)
) = BedrockBatchesConfig().parse_timestamps_and_status(status_response, aws_status_raw)
result: Final = LiteLLMBatch(
id=status_response["invocationArn"],
object="batch",

4
litellm/caching/gcs_cache.py
View file

@ -10,8 +10,8 @@ from urllib.parse import quote
from litellm._logging import print_verbose, verbose_logger
from litellm.integrations.gcs_bucket.gcs_bucket_base import GCSBucketBase
from litellm.llms.custom_httpx.http_handler import (
_get_httpx_client,
get_async_httpx_client,
get_httpx_client,
httpxSpecialProvider,
)
@ -31,7 +31,7 @@ class GCSCache(BaseCache):
self.key_prefix = gcs_path.rstrip("/") + "/" if gcs_path else ""
# create httpx clients
self.async_client = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback)
self.sync_client = _get_httpx_client()
self.sync_client = get_httpx_client()
def _construct_headers(self) -> dict:
base: Final = GCSBucketBase(bucket_name=self.bucket_name)

4
litellm/caching/qdrant_semantic_cache.py
View file

@ -68,8 +68,8 @@ class QdrantSemanticCache(BaseCache):
embedding_timeout: float | None = None,
):
from litellm.llms.custom_httpx.http_handler import (
_get_httpx_client,
get_async_httpx_client,
get_httpx_client,
httpxSpecialProvider,
)
from litellm.secret_managers.main import get_secret_str
@ -112,7 +112,7 @@ class QdrantSemanticCache(BaseCache):
self.headers = headers
self.sync_client = _get_httpx_client()
self.sync_client = get_httpx_client()
self.async_client = get_async_httpx_client(llm_provider=httpxSpecialProvider.Caching)
if quantization_config is None:

2
litellm/compression/compress.py
View file

@ -62,7 +62,7 @@ def _build_retrieval_tools(keys: list[str], call_type: str) -> list[dict]:
# module import for non-Anthropic call paths.
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
anthropic_tools, _mcp_servers = AnthropicConfig()._map_tools(openai_tools)
anthropic_tools, _mcp_servers = AnthropicConfig().map_tools(openai_tools)
return cast(list[dict], anthropic_tools)

8
litellm/cost_calculator.py
View file

@ -70,15 +70,15 @@ from litellm.llms.gemini.cost_calculator import cost_per_token as gemini_cost_pe
from litellm.llms.lemonade.cost_calculator import (
cost_per_token as lemonade_cost_per_token,
)
from litellm.llms.openai.cost_calculation import (
_video_output_cost_per_second,
)
from litellm.llms.openai.cost_calculation import (
cost_per_second as openai_cost_per_second,
)
from litellm.llms.openai.cost_calculation import (
cost_per_token as openai_cost_per_token,
)
from litellm.llms.openai.cost_calculation import (
video_output_cost_per_second,
)
from litellm.llms.perplexity.cost_calculator import (
cost_per_token as perplexity_cost_per_token,
)
@ -2552,7 +2552,7 @@ def default_video_cost_calculator(
if video_cost_per_second is not None:
return video_cost_per_second * duration_seconds
output_cost_per_second: Final = _video_output_cost_per_second(cost_info, video_resolution)
output_cost_per_second: Final = video_output_cost_per_second(cost_info, video_resolution)
if output_cost_per_second is not None:
return output_cost_per_second * duration_seconds

4
litellm/decisions/main.py
View file

@ -10,7 +10,7 @@ import litellm
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.decisions.transformation import DecisionsProviderConfig
from litellm.llms.cloudflare.decisions.transformation import CLOUDFLARE_DECISIONS_ENDPOINT
from litellm.llms.custom_httpx.http_handler import _get_httpx_client, get_async_httpx_client
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client, get_httpx_client
from litellm.llms.openrouter.decisions.transformation import OPENROUTER_DECISIONS_ENDPOINT
from litellm.llms.perplexity.decisions.transformation import PERPLEXITY_DECISIONS_ENDPOINT
from litellm.llms.strands_decider.decisions.transformation import STRANDS_DECIDER_DECISIONS_ENDPOINT
@ -283,7 +283,7 @@ def decisions(
)
logging_obj: Final = _log_request(prepared, kwargs)
try:
handler: Final = _get_httpx_client()
handler: Final = get_httpx_client()
response: Final = handler.post(
prepared.url,
json=dict(prepared.body),

4
litellm/integrations/agentops/agentops.py
View file

@ -7,7 +7,7 @@ from dataclasses import dataclass
from typing import Any, Final
from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig
from litellm.llms.custom_httpx.http_handler import _get_httpx_client
from litellm.llms.custom_httpx.http_handler import get_httpx_client
@dataclass
@ -99,7 +99,7 @@ class AgentOps(OpenTelemetry):
"Connection": "keep-alive",
}
client: Final = _get_httpx_client()
client: Final = get_httpx_client()
try:
response: Final = client.post(
url=auth_endpoint,

4
litellm/integrations/datadog/datadog.py
View file

@ -44,8 +44,8 @@ from litellm.integrations.datadog.datadog_mock_client import (
)
from litellm.litellm_core_utils.dd_tracing import tracer
from litellm.llms.custom_httpx.http_handler import (
_get_httpx_client,
get_async_httpx_client,
get_httpx_client,
httpxSpecialProvider,
)
from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus
@ -180,7 +180,7 @@ class DataDogLogger(
dd_base_url: Final = get_datadog_base_url_from_env()
if dd_base_url:
self.intake_url = f"{dd_base_url}/api/v2/logs"
self.sync_client = _get_httpx_client()
self.sync_client = get_httpx_client()
asyncio.create_task(self.periodic_flush())
self.flush_lock = asyncio.Lock()
super().__init__(

10
litellm/integrations/gcs_bucket/gcs_bucket_base.py
View file

@ -53,13 +53,13 @@ class GCSBucketBase(CustomBatchLogger):
if vertex_instance is None:
vertex_instance = vertex_chat_completion
_auth_header, vertex_project = await vertex_instance._ensure_access_token_async(
_auth_header, vertex_project = await vertex_instance.ensure_access_token_async(
credentials=service_account_json,
project_id=None,
custom_llm_provider="vertex_ai",
)
auth_header, _ = vertex_instance._get_token_and_url(
auth_header, _ = vertex_instance.get_token_and_url(
model="gcs-bucket",
auth_header=_auth_header,
vertex_credentials=service_account_json,
@ -89,13 +89,13 @@ class GCSBucketBase(CustomBatchLogger):
# from Secret Manager.
project_id: Final = os.getenv("GOOGLE_SECRET_MANAGER_PROJECT_ID")
_auth_header, vertex_project = vertex_chat_completion._ensure_access_token(
_auth_header, vertex_project = vertex_chat_completion.ensure_access_token(
credentials=self.path_service_account_json,
project_id=project_id,
custom_llm_provider="vertex_ai",
)
auth_header, _ = vertex_chat_completion._get_token_and_url(
auth_header, _ = vertex_chat_completion.get_token_and_url(
model="gcs-bucket",
auth_header=_auth_header,
vertex_credentials=self.path_service_account_json,
@ -192,7 +192,7 @@ class GCSBucketBase(CustomBatchLogger):
_in_memory_key: Final = self._get_in_memory_key_for_vertex_instance(credentials)
if _in_memory_key not in self.vertex_instances:
vertex_instance: Final = VertexBase()
await vertex_instance._ensure_access_token_async(
await vertex_instance.ensure_access_token_async(
credentials=credentials,
project_id=None,
custom_llm_provider="vertex_ai",

4
litellm/integrations/gcs_pubsub/pub_sub.py
View file

@ -69,13 +69,13 @@ class GcsPubSubLogger(CustomBatchLogger):
(
_auth_header,
vertex_project,
) = await vertex_chat_completion._ensure_access_token_async(
) = await vertex_chat_completion.ensure_access_token_async(
credentials=self.path_service_account_json,
project_id=self.project_id,
custom_llm_provider="vertex_ai",
)
auth_header, _ = vertex_chat_completion._get_token_and_url(
auth_header, _ = vertex_chat_completion.get_token_and_url(
model="pub-sub",
auth_header=_auth_header,
vertex_credentials=self.path_service_account_json,

4
litellm/integrations/generic_prompt_management/generic_prompt_manager.py
View file

@ -15,8 +15,8 @@ from litellm.integrations.prompt_management_base import (
PromptManagementClient,
)
from litellm.llms.custom_httpx.http_handler import (
_get_httpx_client,
get_async_httpx_client,
get_httpx_client,
)
from litellm.types.llms.custom_http import httpxSpecialProvider
from litellm.types.llms.openai import AllMessageValues
@ -114,7 +114,7 @@ class GenericPromptManager(CustomPromptManagement):
"prompt_id": prompt_id,
**(self.additional_provider_specific_query_params or {}),
}
http_client: Final = _get_httpx_client()
http_client: Final = get_httpx_client()
try:
response: Final = http_client.get(

4
litellm/integrations/humanloop.py
View file

@ -11,7 +11,7 @@ from typing_extensions import TypedDict
import litellm
from litellm.caching import DualCache
from litellm.llms.custom_httpx.http_handler import _get_httpx_client
from litellm.llms.custom_httpx.http_handler import get_httpx_client
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
from litellm.types.prompts.init_prompts import PromptSpec
@ -61,7 +61,7 @@ class HumanLoopPromptManager(DualCache):
return compiled_prompts
def _get_prompt_from_id_api(self, humanloop_prompt_id: str, humanloop_api_key: str) -> PromptManagementClient:
client: Final = _get_httpx_client()
client: Final = get_httpx_client()
base_url: Final = f"https://api.humanloop.com/v5/prompts/{humanloop_prompt_id}"

4
litellm/integrations/langfuse/langfuse.py
View file

@ -28,7 +28,7 @@ from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
validate_langfuse_environment_value,
)
from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info
from litellm.llms.custom_httpx.http_handler import _get_httpx_client
from litellm.llms.custom_httpx.http_handler import get_httpx_client
from litellm.secret_managers.main import str_to_bool
from litellm.types.integrations.langfuse import *
from litellm.types.llms.openai import HttpxBinaryResponseContent, ResponsesAPIResponse
@ -324,7 +324,7 @@ class LangFuseLogger:
self.langfuse_client = create_mock_langfuse_client()
self.is_mock_mode = True
else:
self._http_handler: Final = _get_httpx_client()
self._http_handler: Final = get_httpx_client()
self.langfuse_client = self._http_handler.client
self.is_mock_mode = False

4
litellm/integrations/langfuse/langfuse_sdk.py
View file

@ -40,7 +40,7 @@ import litellm
from litellm._logging import verbose_logger
from litellm.integrations.langfuse.langfuse import PROMPT_CACHE_TTL_ENV, parse_langfuse_debug, whole_number
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.llms.custom_httpx.http_handler import HTTPHandler, _get_httpx_client
from litellm.llms.custom_httpx.http_handler import HTTPHandler, get_httpx_client
from litellm.types.llms.base import LiteLLMBaseModel
__all__ = (
@ -793,7 +793,7 @@ def _build_span_exporter(*, public_key: str, secret_key: str, base_url: str) ->
export_path: Final = os.getenv("LANGFUSE_OTEL_TRACES_EXPORT_PATH") or "/api/public/otel/v1/traces"
encoded_auth: Final = b64encode(f"{public_key}:{secret_key}".encode()).decode("ascii")
return LangfuseSpanExporter(
handler=_get_httpx_client(),
handler=get_httpx_client(),
endpoint=f"{base_url.rstrip('/')}/{export_path.lstrip('/')}",
headers=MappingProxyType(
{

4
litellm/integrations/opik/opik.py
View file

@ -13,8 +13,8 @@ from typing_extensions import ReadOnly, TypedDict, Unpack
from litellm._logging import verbose_logger
from litellm.integrations.custom_batch_logger import CustomBatchLogger
from litellm.llms.custom_httpx.http_handler import (
_get_httpx_client,
get_async_httpx_client,
get_httpx_client,
httpxSpecialProvider,
)
@ -53,7 +53,7 @@ class OpikLogger(CustomBatchLogger):
def __init__(self, **kwargs: Unpack[_OpikLoggerKwargs]) -> None:
self.async_httpx_client = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback)
self.sync_httpx_client = _get_httpx_client()
self.sync_httpx_client = get_httpx_client()
self.opik_project_name: str = (
utils.get_opik_config_variable(

4
litellm/integrations/posthog.py
View file

@ -26,8 +26,8 @@ from litellm.integrations.posthog_mock_client import (
)
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.llms.custom_httpx.http_handler import (
_get_httpx_client,
get_async_httpx_client,
get_httpx_client,
httpxSpecialProvider,
)
from litellm.types.integrations.posthog import (
@ -73,7 +73,7 @@ class PostHogLogger(CustomBatchLogger):
raise Exception("POSTHOG_API_KEY is not set, set 'POSTHOG_API_KEY=<>'")
self.async_client = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback)
self.sync_client = _get_httpx_client()
self.sync_client = get_httpx_client()
self.POSTHOG_API_KEY = os.getenv("POSTHOG_API_KEY")
posthog_api_url: Final = os.getenv("POSTHOG_API_URL", "https://us.i.posthog.com")

4
litellm/integrations/s3_v2.py
View file

@ -52,8 +52,8 @@ from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, run_aws_signing
from litellm.llms.custom_httpx.http_handler import (
_get_httpx_client,
get_async_httpx_client,
get_httpx_client,
httpxSpecialProvider,
)
from litellm.types.integrations.s3_v2 import S3PartitionGranularity, s3BatchLoggingElement
@ -875,7 +875,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
prepared: Final = self._prepare_put(batch_logging_element)
httpx_client: Final = _get_httpx_client(
httpx_client: Final = get_httpx_client(
params=({"ssl_verify": self.s3_verify} if self.s3_verify is not None else None)
)

12
litellm/interactions/http_handler.py
View file

@ -20,8 +20,8 @@ from litellm.llms.base_llm.interactions.transformation import BaseInteractionsAP
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
_get_httpx_client,
get_async_httpx_client,
get_httpx_client,
)
from litellm.types.interactions import (
CancelInteractionResult,
@ -57,7 +57,7 @@ class _BaseHTTPHandler:
litellm_params: GenericLiteLLMParams,
client: HTTPHandler | None,
) -> HTTPHandler:
return client or _get_httpx_client(params={"ssl_verify": litellm_params.get("ssl_verify", None)})
return client or get_httpx_client(params={"ssl_verify": litellm_params.get("ssl_verify", None)})
def _async_client(
self,
@ -129,7 +129,7 @@ class InteractionsHTTPHandler(_BaseHTTPHandler):
)
if client is None:
sync_httpx_client = _get_httpx_client(params={"ssl_verify": litellm_params.get("ssl_verify", None)})
sync_httpx_client = get_httpx_client(params={"ssl_verify": litellm_params.get("ssl_verify", None)})
else:
sync_httpx_client = client
@ -363,7 +363,7 @@ class InteractionsHTTPHandler(_BaseHTTPHandler):
)
if client is None:
sync_httpx_client = _get_httpx_client(params={"ssl_verify": litellm_params.get("ssl_verify", None)})
sync_httpx_client = get_httpx_client(params={"ssl_verify": litellm_params.get("ssl_verify", None)})
else:
sync_httpx_client = client
@ -482,7 +482,7 @@ class InteractionsHTTPHandler(_BaseHTTPHandler):
)
if client is None:
sync_httpx_client = _get_httpx_client(params={"ssl_verify": litellm_params.get("ssl_verify", None)})
sync_httpx_client = get_httpx_client(params={"ssl_verify": litellm_params.get("ssl_verify", None)})
else:
sync_httpx_client = client
@ -603,7 +603,7 @@ class InteractionsHTTPHandler(_BaseHTTPHandler):
)
if client is None:
sync_httpx_client = _get_httpx_client(params={"ssl_verify": litellm_params.get("ssl_verify", None)})
sync_httpx_client = get_httpx_client(params={"ssl_verify": litellm_params.get("ssl_verify", None)})
else:
sync_httpx_client = client

80
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -119,7 +119,7 @@ def handle_anthropic_text_model_custom_llm_provider(
"""
if custom_llm_provider:
if custom_llm_provider == "anthropic" and litellm.AnthropicTextConfig._is_anthropic_text_model(model):
if custom_llm_provider == "anthropic" and litellm.AnthropicTextConfig.is_anthropic_text_model(model):
return model, "anthropic_text"
if model and "/" in model:
@ -127,7 +127,7 @@ def handle_anthropic_text_model_custom_llm_provider(
if (
_custom_llm_provider
and _custom_llm_provider == "anthropic"
and litellm.AnthropicTextConfig._is_anthropic_text_model(_model)
and litellm.AnthropicTextConfig.is_anthropic_text_model(_model)
):
return _model, "anthropic_text"
@ -426,7 +426,7 @@ def get_llm_provider(
custom_llm_provider = "text-completion-openai"
## anthropic
elif model in litellm.anthropic_models:
if litellm.AnthropicTextConfig._is_anthropic_text_model(model):
if litellm.AnthropicTextConfig.is_anthropic_text_model(model):
custom_llm_provider = "anthropic_text"
else:
custom_llm_provider = "anthropic"
@ -605,7 +605,7 @@ def _get_openai_compatible_provider_info(
if provider_config is None:
raise ValueError(f"Provider {custom_llm_provider} not found")
config_class: Final = create_config_class(provider_config)
api_base, dynamic_api_key = config_class()._get_openai_compatible_provider_info(api_base, api_key)
api_base, dynamic_api_key = config_class().get_openai_compatible_provider_info(api_base, api_key)
return model, custom_llm_provider, dynamic_api_key, api_base
if custom_llm_provider == "perplexity":
@ -613,7 +613,7 @@ def _get_openai_compatible_provider_info(
(
api_base,
dynamic_api_key,
) = litellm.PerplexityChatConfig()._get_openai_compatible_provider_info(api_base, api_key)
) = litellm.PerplexityChatConfig().get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "aiohttp_openai":
return model, "aiohttp_openai", api_key, api_base
elif custom_llm_provider == "anyscale":
@ -624,7 +624,7 @@ def _get_openai_compatible_provider_info(
(
api_base,
dynamic_api_key,
) = litellm.DeepInfraConfig()._get_openai_compatible_provider_info(api_base, api_key)
) = litellm.DeepInfraConfig().get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "empower":
api_base = api_base or get_secret("EMPOWER_API_BASE") or "https://app.empower.dev/api/v1"
dynamic_api_key = api_key or get_secret_str("EMPOWER_API_KEY")
@ -632,14 +632,14 @@ def _get_openai_compatible_provider_info(
(
api_base,
dynamic_api_key,
) = litellm.GroqChatConfig()._get_openai_compatible_provider_info(api_base, api_key)
) = litellm.GroqChatConfig().get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "bedrock_mantle":
from litellm.llms.bedrock_mantle.common_utils import split_mantle_region_prefix
(
api_base,
dynamic_api_key,
) = litellm.BedrockMantleChatConfig()._get_openai_compatible_provider_info(
) = litellm.BedrockMantleChatConfig().get_openai_compatible_provider_info(
api_base, api_key, litellm_params=litellm_params, model=model
)
model = split_mantle_region_prefix(model)[1] # rebind-ok: the prefix is routing only, not a Mantle model id
@ -704,25 +704,25 @@ def _get_openai_compatible_provider_info(
(
api_base,
dynamic_api_key,
) = litellm.HostedVLLMChatConfig()._get_openai_compatible_provider_info(api_base, api_key)
) = litellm.HostedVLLMChatConfig().get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "llamafile":
# llamafile is OpenAI compatible.
(
api_base,
dynamic_api_key,
) = litellm.LlamafileChatConfig()._get_openai_compatible_provider_info(api_base, api_key)
) = litellm.LlamafileChatConfig().get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "datarobot":
# DataRobot is OpenAI compatible.
(
api_base,
dynamic_api_key,
) = litellm.DataRobotConfig()._get_openai_compatible_provider_info(api_base, api_key)
) = litellm.DataRobotConfig().get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "lm_studio":
# lm_studio is openai compatible, we just need to set this to custom_openai
(
api_base,
dynamic_api_key,
) = litellm.LMStudioChatConfig()._get_openai_compatible_provider_info(api_base, api_key)
) = litellm.LMStudioChatConfig().get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "deepseek":
# deepseek is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.deepseek.com/v1
api_base = api_base or get_secret("DEEPSEEK_API_BASE") or "https://api.deepseek.com/beta"
@ -737,13 +737,13 @@ def _get_openai_compatible_provider_info(
(
api_base,
dynamic_api_key,
) = litellm.FireworksAIConfig()._get_openai_compatible_provider_info(api_base=api_base, api_key=api_key)
) = litellm.FireworksAIConfig().get_openai_compatible_provider_info(api_base=api_base, api_key=api_key)
elif custom_llm_provider == "azure_ai":
(
api_base,
dynamic_api_key,
custom_llm_provider,
) = litellm.AzureAIStudioConfig()._get_openai_compatible_provider_info(
) = litellm.AzureAIStudioConfig().get_openai_compatible_provider_info(
model, api_base, api_key, custom_llm_provider
)
elif custom_llm_provider == "github":
@ -757,29 +757,29 @@ def _get_openai_compatible_provider_info(
(
api_base,
dynamic_api_key,
) = litellm.LiteLLMProxyChatConfig()._get_openai_compatible_provider_info(api_base=api_base, api_key=api_key)
) = litellm.LiteLLMProxyChatConfig().get_openai_compatible_provider_info(api_base=api_base, api_key=api_key)
elif custom_llm_provider == "mistral":
(
api_base,
dynamic_api_key,
) = litellm.MistralConfig()._get_openai_compatible_provider_info(api_base, api_key)
) = litellm.MistralConfig().get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "jina_ai":
(
custom_llm_provider,
api_base,
dynamic_api_key,
) = litellm.JinaAIEmbeddingConfig()._get_openai_compatible_provider_info(api_base, api_key)
) = litellm.JinaAIEmbeddingConfig().get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "xai":
(
api_base,
dynamic_api_key,
) = litellm.XAIChatConfig()._get_openai_compatible_provider_info(api_base, api_key)
) = litellm.XAIChatConfig().get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "zai":
(
api_base,
dynamic_api_key,
) = litellm.ZAIChatConfig()._get_openai_compatible_provider_info(api_base, api_key)
) = litellm.ZAIChatConfig().get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "together_ai":
api_base = api_base or get_secret_str("TOGETHER_AI_API_BASE") or "https://api.together.ai/v1"
dynamic_api_key = api_key or (
@ -799,7 +799,7 @@ def _get_openai_compatible_provider_info(
api_base,
dynamic_api_key,
custom_llm_provider,
) = litellm.GithubCopilotConfig()._get_openai_compatible_provider_info(
) = litellm.GithubCopilotConfig().get_openai_compatible_provider_info(
model, api_base, api_key, custom_llm_provider
)
elif custom_llm_provider == "chatgpt":
@ -807,7 +807,7 @@ def _get_openai_compatible_provider_info(
api_base,
dynamic_api_key,
custom_llm_provider,
) = litellm.ChatGPTConfig()._get_openai_compatible_provider_info(model, api_base, api_key, custom_llm_provider)
) = litellm.ChatGPTConfig().get_openai_compatible_provider_info(model, api_base, api_key, custom_llm_provider)
elif custom_llm_provider == "novita":
api_base = api_base or get_secret("NOVITA_API_BASE") or "https://api.novita.ai/v3/openai"
dynamic_api_key = api_key or get_secret_str("NOVITA_API_KEY")
@ -815,78 +815,78 @@ def _get_openai_compatible_provider_info(
(
api_base,
dynamic_api_key,
) = litellm.SnowflakeConfig()._get_openai_compatible_provider_info(api_base, api_key)
) = litellm.SnowflakeConfig().get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "gradient_ai":
(
api_base,
dynamic_api_key,
) = litellm.GradientAIConfig()._get_openai_compatible_provider_info(api_base, api_key)
) = litellm.GradientAIConfig().get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "featherless_ai":
(
api_base,
dynamic_api_key,
) = litellm.FeatherlessAIConfig()._get_openai_compatible_provider_info(api_base, api_key)
) = litellm.FeatherlessAIConfig().get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "nscale":
(
api_base,
dynamic_api_key,
) = litellm.NscaleConfig()._get_openai_compatible_provider_info(api_base=api_base, api_key=api_key)
) = litellm.NscaleConfig().get_openai_compatible_provider_info(api_base=api_base, api_key=api_key)
elif custom_llm_provider == "heroku":
(
api_base,
dynamic_api_key,
) = litellm.HerokuChatConfig()._get_openai_compatible_provider_info(api_base, api_key)
) = litellm.HerokuChatConfig().get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider in ("dashscope", "qwencloud", "qwen_ai_platform"):
(
api_base,
dynamic_api_key,
) = _dashscope_family_chat_config(custom_llm_provider)._get_openai_compatible_provider_info(api_base, api_key)
) = _dashscope_family_chat_config(custom_llm_provider).get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "modelscope":
(
api_base,
dynamic_api_key,
) = litellm.ModelScopeChatConfig()._get_openai_compatible_provider_info(api_base, api_key)
) = litellm.ModelScopeChatConfig().get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "moonshot":
(
api_base,
dynamic_api_key,
) = litellm.MoonshotChatConfig()._get_openai_compatible_provider_info(api_base, api_key)
) = litellm.MoonshotChatConfig().get_openai_compatible_provider_info(api_base, api_key)
# publicai is now handled by JSON config (see litellm/llms/openai_like/providers.json)
elif custom_llm_provider == "docker_model_runner":
(
api_base,
dynamic_api_key,
) = litellm.DockerModelRunnerChatConfig()._get_openai_compatible_provider_info(api_base, api_key)
) = litellm.DockerModelRunnerChatConfig().get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "v0":
(
api_base,
dynamic_api_key,
) = litellm.V0ChatConfig()._get_openai_compatible_provider_info(api_base, api_key)
) = litellm.V0ChatConfig().get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "morph":
(
api_base,
dynamic_api_key,
) = litellm.MorphChatConfig()._get_openai_compatible_provider_info(api_base, api_key)
) = litellm.MorphChatConfig().get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "lambda_ai":
(
api_base,
dynamic_api_key,
) = litellm.LambdaAIChatConfig()._get_openai_compatible_provider_info(api_base, api_key)
) = litellm.LambdaAIChatConfig().get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "inception":
(
api_base,
dynamic_api_key,
) = litellm.InceptionChatConfig()._get_openai_compatible_provider_info(api_base, api_key)
) = litellm.InceptionChatConfig().get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "hyperbolic":
(
api_base,
dynamic_api_key,
) = litellm.HyperbolicChatConfig()._get_openai_compatible_provider_info(api_base, api_key)
) = litellm.HyperbolicChatConfig().get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "vercel_ai_gateway":
(
api_base,
dynamic_api_key,
) = litellm.VercelAIGatewayConfig()._get_openai_compatible_provider_info(api_base, api_key)
) = litellm.VercelAIGatewayConfig().get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "edenai":
api_base = litellm.EdenAIChatConfig.get_api_base(api_base) # rebind-ok: chain resolves in place
dynamic_api_key = litellm.EdenAIChatConfig.get_api_key(api_key) # rebind-ok: chain resolves in place
@ -896,7 +896,7 @@ def _get_openai_compatible_provider_info(
(
api_base,
dynamic_api_key,
) = litellm.AIMLChatConfig()._get_openai_compatible_provider_info(api_base, api_key)
) = litellm.AIMLChatConfig().get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "wandb":
api_base = api_base or get_secret("WANDB_API_BASE") or "https://api.inference.wandb.ai/v1"
dynamic_api_key = api_key or get_secret_str("WANDB_API_KEY")
@ -904,19 +904,19 @@ def _get_openai_compatible_provider_info(
(
api_base,
dynamic_api_key,
) = litellm.LemonadeChatConfig()._get_openai_compatible_provider_info(api_base, api_key)
) = litellm.LemonadeChatConfig().get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "clarifai":
(
api_base,
dynamic_api_key,
) = litellm.ClarifaiConfig()._get_openai_compatible_provider_info(api_base, api_key)
) = litellm.ClarifaiConfig().get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "ragflow":
full_model: Final = f"ragflow/{model}"
(
api_base,
dynamic_api_key,
_,
) = litellm.RAGFlowConfig()._get_openai_compatible_provider_info(full_model, api_base, api_key, "ragflow")
) = litellm.RAGFlowConfig().get_openai_compatible_provider_info(full_model, api_base, api_key, "ragflow")
model = full_model
elif custom_llm_provider == "langgraph":
# LangGraph is a custom provider, just need to set api_base

4
litellm/litellm_core_utils/litellm_logging.py
View file

@ -2116,7 +2116,7 @@ class Logging(LiteLLMLoggingBaseClass):
import httpx
completion_response = result.model_dump(by_alias=True) if isinstance(result, BaseModel) else dict(result)
return litellm.VertexGeminiConfig()._transform_google_generate_content_to_openai_model_response(
return litellm.VertexGeminiConfig().transform_google_generate_content_to_openai_model_response(
completion_response=completion_response,
model_response=ModelResponse(),
model=self.model or "",
@ -4361,7 +4361,7 @@ class Logging(LiteLLMLoggingBaseClass):
if httpx_response is None:
raise ValueError("Google GenAI Generate Content: httpx_response is None")
dict_result: Final = httpx_response.json()
result = litellm.VertexGeminiConfig()._transform_google_generate_content_to_openai_model_response(
result = litellm.VertexGeminiConfig().transform_google_generate_content_to_openai_model_response(
completion_response=dict_result,
model_response=litellm.ModelResponse(),
model=self.model,

6
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -1303,7 +1303,7 @@ def convert_to_gemini_tool_call_invoke(
VertexGeminiConfig,
)
needs_dummy_signature: Final = model is not None and VertexGeminiConfig._is_gemini_3_or_newer(model)
needs_dummy_signature: Final = model is not None and VertexGeminiConfig.is_gemini_3_or_newer(model)
if tool_calls is not None:
for tool in tool_calls:
@ -5312,7 +5312,7 @@ def prompt_factory(
if custom_llm_provider == "ollama":
return ollama_pt(model=model, messages=messages)
elif custom_llm_provider == "anthropic":
if litellm.AnthropicTextConfig._is_anthropic_text_model(model):
if litellm.AnthropicTextConfig.is_anthropic_text_model(model):
return anthropic_pt(messages=messages)
return anthropic_messages_pt(messages=messages, model=model, llm_provider=custom_llm_provider)
elif custom_llm_provider == "anthropic_xml":
@ -5327,7 +5327,7 @@ def prompt_factory(
else:
return gemini_text_image_pt(messages=messages)
elif custom_llm_provider == "mistral":
return litellm.MistralConfig()._transform_messages(messages=messages, model=model)
return litellm.MistralConfig().transform_messages(messages=messages, model=model)
elif custom_llm_provider == "bedrock":
if "amazon.titan-text" in model:
return amazon_titan_pt(messages=messages)

6
litellm/litellm_core_utils/prompt_templates/huggingface_template_handler.py
View file

@ -5,8 +5,8 @@ from typing import Any, Final, Literal
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm.llms.custom_httpx.http_handler import (
_get_httpx_client,
get_async_httpx_client,
get_httpx_client,
)
from litellm.types.llms.custom_http import httpxSpecialProvider
@ -50,7 +50,7 @@ def get_tokenizer_config(hf_model_name: str) -> _TokenizerConfigResult:
"""
try:
url: Final = f"https://huggingface.co/{hf_model_name}/raw/main/tokenizer_config.json"
client: Final = _get_httpx_client()
client: Final = get_httpx_client()
response: Final = client.get(url=url)
except Exception as e:
raise e
@ -103,7 +103,7 @@ def get_chat_template_file(hf_model_name: str) -> _ChatTemplateFileResult:
Dict with 'status' and optionally 'chat_template' keys
"""
template_filenames: Final = ["chat_template.jinja", "chat_template.jinja2"]
client: Final = _get_httpx_client()
client: Final = get_httpx_client()
for filename in template_filenames:
try:

2
litellm/litellm_core_utils/streaming_handler.py
View file

@ -1315,7 +1315,7 @@ class CustomStreamWrapper:
raise ValueError(f"chunk is not a string: {chunk}")
response_obj = cast(
dict[str, object],
litellm.CodestralTextCompletionConfig()._chunk_parser(chunk),
litellm.CodestralTextCompletionConfig().chunk_parser(chunk),
)
completion_obj["content"] = response_obj["text"]
print_verbose(f"completion obj content: {completion_obj['content']}")

4
litellm/litellm_core_utils/token_counter.py
View file

@ -32,7 +32,7 @@ from litellm.constants import (
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.tokenizer import Encoding, HuggingFace, HuggingFaceTokenizer, OpenAIEncoding
from litellm.litellm_core_utils.url_utils import safe_get
from litellm.llms.custom_httpx.http_handler import _get_httpx_client
from litellm.llms.custom_httpx.http_handler import get_httpx_client
from litellm.rust_bridge.tokenizer import get_encoding
from litellm.types.llms.anthropic import (
AnthropicContentParamSource,
@ -232,7 +232,7 @@ def get_image_dimensions(
img_data = None
if data.startswith(("http://", "https://")):
try:
client: Final = _get_httpx_client()
client: Final = get_httpx_client()
response: Final[httpx.Response] = safe_get(client, data)
max_bytes: Final = int(MAX_IMAGE_URL_DOWNLOAD_SIZE_MB * 1024 * 1024)
content_length: Final[str | None] = response.headers.get("Content-Length")

7
litellm/llms/aiml/chat/transformation.py
View file

@ -18,3 +18,10 @@ class AIMLChatConfig(OpenAIGPTConfig):
)
dynamic_api_key: Final = api_key or get_secret_str("AIML_API_KEY")
return api_base, dynamic_api_key
def get_openai_compatible_provider_info(
self,
api_base: str | None,
api_key: str | None,
) -> tuple[str | None, str | None]:
return self._get_openai_compatible_provider_info(api_base, api_key)

2
litellm/llms/anthropic/chat/guardrail_translation/handler.py
View file

@ -633,7 +633,7 @@ class AnthropicMessagesHandler(BaseTranslation):
anthropic_config: Final = AnthropicConfig()
anthropic_tools: Final[list[AllAnthropicToolsValues]] = []
for tool in guardrailed_tools:
converted_tool, mcp_server = anthropic_config._map_tool_helper(tool)
converted_tool, mcp_server = anthropic_config.map_tool_helper(tool)
if converted_tool is not None:
anthropic_tools.append(converted_tool)
# Note: MCP servers are handled separately in the main transformation

6
litellm/llms/anthropic/chat/handler.py
View file

@ -23,8 +23,8 @@ from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAcc
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
_get_httpx_client,
get_async_httpx_client,
get_httpx_client,
)
from litellm.types.llms.anthropic import (
ContentBlockDelta,
@ -496,7 +496,7 @@ class AnthropicChatCompletion(BaseLLM):
else:
if client is None or not isinstance(client, HTTPHandler):
client = _get_httpx_client(params={"timeout": timeout})
client = get_httpx_client(params={"timeout": timeout})
else:
client = client
@ -1074,7 +1074,7 @@ class ModelResponseIterator:
# Convert tool to content if we're tracking a response_format tool
if self.is_response_format_tool:
message: Final = AnthropicConfig._convert_tool_response_to_message(tool_calls=[tool_use])
message: Final = AnthropicConfig.convert_tool_response_to_message(tool_calls=[tool_use])
if message is not None:
text = message.content or ""
tool_use = None

77
litellm/llms/anthropic/chat/transformation.py
View file

@ -416,6 +416,15 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
return f"effort='xhigh' is not supported by this model. Got model: {model}"
return None
@classmethod
def validate_effort_for_model(
cls,
model: str,
effort: str | None,
custom_llm_provider: str,
) -> str | None:
return cls._validate_effort_for_model(model, effort, custom_llm_provider)
@staticmethod
def _model_supports_effort_param(model: str, custom_llm_provider: str) -> bool:
"""Whether the model accepts ``output_config.effort`` at all.
@ -432,6 +441,14 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
for level in ("low", "minimal", "medium", "high", "xhigh", "max")
)
@classmethod
def model_supports_effort_param(
cls,
model: str,
custom_llm_provider: str,
) -> bool:
return cls._model_supports_effort_param(model, custom_llm_provider)
@staticmethod
def _model_supports_speed_param(model: str, custom_llm_provider: str | None = None) -> bool:
"""Whether the model accepts Anthropic's ``speed`` parameter (fast mode).
@ -472,6 +489,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
)
optional_params.pop("speed", None)
@classmethod
def maybe_drop_speed_param(
cls,
model: str,
optional_params: dict[str, object], # mutable-ok: mirrors override contract
drop_params: bool,
custom_llm_provider: str | None = None,
) -> None:
return cls._maybe_drop_speed_param(model, optional_params, drop_params, custom_llm_provider)
@staticmethod
def _raise_invalid_reasoning_effort(model: str, value: object, llm_provider: str) -> NoReturn:
"""Raise a ``BadRequestError`` for an unrecognised ``reasoning_effort``.
@ -495,6 +522,15 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
llm_provider=llm_provider,
)
@classmethod
def raise_invalid_reasoning_effort(
cls,
model: str,
value: object,
llm_provider: str,
) -> NoReturn:
return cls._raise_invalid_reasoning_effort(model, value, llm_provider)
def get_supported_openai_params(self, model: str):
params: Final = [
"stream",
@ -923,6 +959,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
return returned_tool, mcp_server
def map_tool_helper(
self,
tool: ChatCompletionToolParam,
) -> tuple[AllAnthropicToolsValues | None, AnthropicMcpServerTool | None]:
return self._map_tool_helper(tool)
def _map_openai_mcp_server_tool(self, tool: OpenAIMcpServerTool) -> AnthropicMcpServerTool:
from litellm.types.llms.anthropic import AnthropicMcpServerToolConfiguration
@ -998,6 +1040,14 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
mcp_servers.append(mcp_server_tool)
return anthropic_tools, mcp_servers
def map_tools(
self,
tools: list[ # mutable-ok: mirrors override contract
ChatCompletionToolParam | AllAnthropicToolsValues | dict[str, object]
],
) -> tuple[list[AllAnthropicToolsValues], list[AnthropicMcpServerTool]]: # mutable-ok: mirrors override contract
return self._map_tools(tools)
@staticmethod
def _rewrite_tool_names_in_messages(
messages: list[AllMessageValues],
@ -1249,6 +1299,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
new_stop = new_v
return new_stop
def map_stop_sequences(
self,
stop: str | list[str] | None, # mutable-ok: mirrors override contract
) -> list[str] | None: # mutable-ok: mirrors override contract
return self._map_stop_sequences(stop)
@staticmethod
def _map_reasoning_effort(
reasoning_effort: REASONING_EFFORT | str | None,
@ -1310,6 +1366,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
llm_provider=llm_provider,
)
@classmethod
def map_reasoning_effort(
cls,
reasoning_effort: REASONING_EFFORT | str | None,
model: str,
custom_llm_provider: str,
llm_provider: str = "anthropic",
) -> AnthropicThinkingParam | None:
return cls._map_reasoning_effort(reasoning_effort, model, custom_llm_provider, llm_provider)
@staticmethod
def cap_thinking_budget_to_max_tokens(
thinking: AnthropicThinkingParam, max_tokens: int | None
@ -2152,11 +2218,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
json_tool: Final = tool_calls[json_indices[0]]
if json_tool.get("function", {}).get("arguments") is None:
return None, tool_calls, None
_message: Final = AnthropicConfig._convert_tool_response_to_message(tool_calls=[json_tool])
_message: Final = AnthropicConfig.convert_tool_response_to_message(tool_calls=[json_tool])
return _message, [], None
first_json: Final = tool_calls[json_indices[0]]
json_msg: Final = AnthropicConfig._convert_tool_response_to_message([first_json])
json_msg: Final = AnthropicConfig.convert_tool_response_to_message([first_json])
extra_content: Final[str | None] = json_msg.content if json_msg is not None else None
filtered_tools: Final = [t for i, t in enumerate(tool_calls) if i not in json_indices]
return None, filtered_tools, extra_content
@ -2786,6 +2852,13 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
return litellm.Message(content=json_mode_content_str)
return None
@classmethod
def convert_tool_response_to_message(
cls,
tool_calls: list[ChatCompletionToolCallChunk], # mutable-ok: mirrors override contract
) -> LitellmMessage | None:
return cls._convert_tool_response_to_message(tool_calls)
def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException:
return AnthropicError(
status_code=status_code,

29
litellm/llms/anthropic/common_utils.py
View file

@ -664,6 +664,18 @@ class AnthropicModelInfo(BaseLLMModelInfo):
status_code=400,
)
@classmethod
def apply_sampling_param(
cls,
optional_params: dict[str, object], # mutable-ok: mirrors override contract
model: str,
param: str,
value: object,
drop_params: bool,
output_key: str,
) -> None:
return cls._apply_sampling_param(optional_params, model, param, value, drop_params, output_key)
@staticmethod
def forced_tool_use_unsupported(model: str) -> bool:
return AnthropicModelInfo._get_model_capability(model, "supports_forced_tool_use") is False
@ -829,6 +841,15 @@ class AnthropicModelInfo(BaseLLMModelInfo):
pass
return AnthropicModelInfo._get_model_capability(model, key) is True
@classmethod
def supports_model_capability(
cls,
model: str,
key: str,
custom_llm_provider: str,
) -> bool:
return cls._supports_model_capability(model, key, custom_llm_provider)
@staticmethod
def _is_adaptive_thinking_model(model: str, custom_llm_provider: str) -> bool:
"""Whether ``model`` uses adaptive thinking (``output_config.effort``).
@ -841,6 +862,14 @@ class AnthropicModelInfo(BaseLLMModelInfo):
"""
return AnthropicModelInfo._supports_model_capability(model, "supports_adaptive_thinking", custom_llm_provider)
@classmethod
def is_adaptive_thinking_model(
cls,
model: str,
custom_llm_provider: str,
) -> bool:
return cls._is_adaptive_thinking_model(model, custom_llm_provider)
@staticmethod
def _is_always_on_thinking_model(model: str, custom_llm_provider: str) -> bool:
"""Whether ``model`` always thinks and rejects ``thinking.type=disabled``

9
litellm/llms/anthropic/completion/transformation.py
View file

@ -163,7 +163,7 @@ class AnthropicTextConfig(BaseConfig):
if param == "stream" and value is True:
optional_params["stream"] = value
if param == "stop" and (isinstance(value, str) or isinstance(value, list)):
_value = litellm.AnthropicConfig()._map_stop_sequences(value)
_value = litellm.AnthropicConfig().map_stop_sequences(value)
if _value is not None:
optional_params["stop_sequences"] = _value
if param == "temperature":
@ -232,6 +232,13 @@ class AnthropicTextConfig(BaseConfig):
def _is_anthropic_text_model(model: str) -> bool:
return model == "claude-2" or model == "claude-instant-1"
@classmethod
def is_anthropic_text_model(
cls,
model: str,
) -> bool:
return cls._is_anthropic_text_model(model)
def _get_anthropic_text_prompt_from_messages(self, messages: list[AllMessageValues], model: str) -> str:
custom_prompt_dict: Final = litellm.custom_prompt_dict
if model in custom_prompt_dict:

6
litellm/llms/anthropic/pass_through/adapters/handler.py
View file

@ -231,7 +231,7 @@ def _normalize_spec_edits(
) -> list[dict[str, object]] | None:
"""Return the normalized ``edits`` list, or ``None`` if the polyfill won't run.
Delegates spec-shape normalization to the dispatcher's ``_normalize_spec``
Delegates spec-shape normalization to the dispatcher's ``normalize_spec``
so the prediction here can't drift from what the dispatcher actually does.
"""
if not context_management_spec:
@ -241,11 +241,11 @@ def _normalize_spec_edits(
return None
from litellm.llms.anthropic.pass_through.context_management.dispatcher import (
_normalize_spec,
normalize_spec,
)
try:
return _normalize_spec(context_management_spec)
return normalize_spec(context_management_spec)
except Exception:
return None

4
litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py
View file

@ -396,7 +396,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
from .transformation import LiteLLMAnthropicMessagesAdapter
usage_dict: UsageDelta = LiteLLMAnthropicMessagesAdapter._translate_openai_usage_to_anthropic_usage_delta(
usage_dict: UsageDelta = LiteLLMAnthropicMessagesAdapter.translate_openai_usage_to_anthropic_usage_delta(
chunk.usage
)
if self.applied_edits and "context_management" not in merged_chunk:
@ -1163,7 +1163,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
(
block_type,
content_block_start,
) = LiteLLMAnthropicMessagesAdapter()._translate_streaming_openai_chunk_to_anthropic_content_block(
) = LiteLLMAnthropicMessagesAdapter().translate_streaming_openai_chunk_to_anthropic_content_block(
choices=chunk.choices
)

20
litellm/llms/anthropic/pass_through/adapters/transformation.py
View file

@ -1502,7 +1502,7 @@ class LiteLLMAnthropicMessagesAdapter:
return cls._first_positive_prompt_tokens_detail_value(usage, ("web_search_requests",))
@classmethod
def _translate_openai_usage_to_anthropic_usage_delta(cls, usage: Usage) -> UsageDelta:
def translate_openai_usage_to_anthropic_usage_delta(cls, usage: Usage) -> UsageDelta:
cache_read_input_tokens: Final = cls._get_cache_read_input_tokens(usage)
cache_creation_input_tokens: Final = cls._get_cache_creation_input_tokens(usage)
web_search_requests: Final = cls._get_web_search_request_count(usage)
@ -1526,13 +1526,17 @@ class LiteLLMAnthropicMessagesAdapter:
)
return usage_delta
_translate_openai_usage_to_anthropic_usage_delta = translate_openai_usage_to_anthropic_usage_delta
@classmethod
def _translate_openai_usage_to_anthropic_usage(cls, usage: Usage) -> AnthropicUsage:
def translate_openai_usage_to_anthropic_usage(cls, usage: Usage) -> AnthropicUsage:
return cast(
AnthropicUsage,
cls._translate_openai_usage_to_anthropic_usage_delta(usage),
cls.translate_openai_usage_to_anthropic_usage_delta(usage),
)
_translate_openai_usage_to_anthropic_usage = translate_openai_usage_to_anthropic_usage
def translate_openai_response_to_anthropic(
self,
response: ModelResponse,
@ -1576,7 +1580,7 @@ class LiteLLMAnthropicMessagesAdapter:
)
# extract usage
usage: Final[Usage] = getattr(response, "usage")
message_usage: Final = self._translate_openai_usage_to_anthropic_usage(usage)
message_usage: Final = self.translate_openai_usage_to_anthropic_usage(usage)
polyfill_iterations: Final = polyfill_result.iterations_usage if polyfill_result is not None else None
anthropic_usage: Final[AnthropicUsage] = (
TypeAdapter(AnthropicUsage).validate_python(
@ -1616,7 +1620,7 @@ class LiteLLMAnthropicMessagesAdapter:
return translated_obj
def _translate_streaming_openai_chunk_to_anthropic_content_block(
def translate_streaming_openai_chunk_to_anthropic_content_block(
self, choices: list[OpenAIStreamingChoice | StreamingChoices]
) -> tuple[
Literal["text", "tool_use", "thinking"],
@ -1676,6 +1680,10 @@ class LiteLLMAnthropicMessagesAdapter:
return "text", TextBlock(type="text", text="")
_translate_streaming_openai_chunk_to_anthropic_content_block = (
translate_streaming_openai_chunk_to_anthropic_content_block
)
def _translate_streaming_openai_chunk_to_anthropic(
self, choices: list[OpenAIStreamingChoice | StreamingChoices]
) -> tuple[
@ -1748,7 +1756,7 @@ class LiteLLMAnthropicMessagesAdapter:
else None
)
if litellm_usage_chunk is not None:
usage_delta = self._translate_openai_usage_to_anthropic_usage_delta(litellm_usage_chunk)
usage_delta = self.translate_openai_usage_to_anthropic_usage_delta(litellm_usage_chunk)
else:
usage_delta = UsageDelta(input_tokens=0, output_tokens=0)
message_block: Final = MessageBlockDelta(

7
litellm/llms/anthropic/pass_through/context_management/dispatcher.py
View file

@ -33,7 +33,7 @@ def _edits_from(normalized: dict[str, object] | None) -> list[dict[str, object]]
return [edit for edit in edits if isinstance(edit, dict)]
def _normalize_spec(
def normalize_spec(
spec: dict[str, object] | list[dict[str, object]] | None,
) -> list[dict[str, object]] | None:
"""Accept Anthropic-native dict form or OpenAI list form; return edits list."""
@ -46,6 +46,9 @@ def _normalize_spec(
return _edits_from(spec)
_normalize_spec = normalize_spec
def _wrap_editor_return(
raw: EditorResult,
*,
@ -87,7 +90,7 @@ async def apply_context_management(
worker thread so their token counts stay off the event loop;
``inspect.iscoroutinefunction`` decides how each editor is invoked.
"""
edits: Final = _normalize_spec(context_management_spec)
edits: Final = normalize_spec(context_management_spec)
if not edits:
return PolyfillResult(messages=messages, system=system, applied_edits=[])

6
litellm/llms/anthropic/pass_through/messages/response_cache.py
View file

@ -9,9 +9,9 @@ from litellm.caching.caching_handler import create_cache_write_task, is_response
from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
AnthropicMessagesStreamingResponse,
BaseAnthropicMessagesStreamingIterator,
_is_message_stop_chunk,
_is_provider_error_chunk,
aclose_if_supported,
is_message_stop_chunk,
is_provider_error_chunk,
)
from litellm.types.caching import CACHED_STREAM_EVENTS_KEY
@ -90,7 +90,7 @@ class AnthropicMessagesStreamCacheWriter:
if self.persisted or cache is None:
return
collected_stream: Final = b"".join(self.collected_chunks)
if not _is_message_stop_chunk(collected_stream) or _is_provider_error_chunk(collected_stream):
if not is_message_stop_chunk(collected_stream) or is_provider_error_chunk(collected_stream):
return
self.persisted = True

12
litellm/llms/anthropic/pass_through/messages/streaming_iterator.py
View file

@ -30,7 +30,7 @@ _UPSTREAM_PUMP_TASKS: Final[set[asyncio.Task[None]]] = set() # mutable-ok: stdl
_DETACHED_STREAM_DRAINS: Final[set[asyncio.Task[None]]] = set() # mutable-ok: bounded strong-ref set, detached drains
def _is_message_stop_chunk(chunk: object) -> bool:
def is_message_stop_chunk(chunk: object) -> bool:
if isinstance(chunk, dict):
return chunk.get("type") == "message_stop"
if isinstance(chunk, (bytes, bytearray)):
@ -38,6 +38,9 @@ def _is_message_stop_chunk(chunk: object) -> bool:
return False
_is_message_stop_chunk = is_message_stop_chunk
def is_anthropic_ping_chunk(chunk: object) -> bool:
"""
Whether a chunk is made only of whole ``ping`` keepalive frames. A ping
@ -133,10 +136,13 @@ def _anthropic_error_body(chunk: object) -> Mapping[str, object] | None:
return error_body if isinstance(error_body, dict) else None
def _is_provider_error_chunk(chunk: object) -> bool:
def is_provider_error_chunk(chunk: object) -> bool:
return _anthropic_error_body(chunk) is not None
_is_provider_error_chunk = is_provider_error_chunk
def parse_anthropic_error_event(chunk: object) -> tuple[str, str, int] | None:
"""
Extract ``(error_type, message, http_status_code)`` from an Anthropic SSE
@ -161,7 +167,7 @@ def parse_anthropic_error_event(chunk: object) -> tuple[str, str, int] | None:
def _is_terminal_stream_chunk(chunk: object) -> bool:
return _is_message_stop_chunk(chunk) or _is_provider_error_chunk(chunk)
return is_message_stop_chunk(chunk) or is_provider_error_chunk(chunk)
def _try_claim_detached_drain_slot() -> bool:

25
litellm/llms/anthropic/pass_through/messages/transformation.py
View file

@ -62,10 +62,13 @@ DROP_UNFITTING_REASONING_EFFORT_WARNING: Final = (
)
def _messages_carry_output_config(messages: Sequence[object]) -> bool:
def messages_carry_output_config(messages: Sequence[object]) -> bool:
return any(isinstance(message, Mapping) and "output_config" in message for message in messages)
_messages_carry_output_config = messages_carry_output_config
class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
_workload_identity_eligible: ClassVar[bool] = True
@ -395,7 +398,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
return
try:
mapped_thinking: Final = AnthropicConfig._map_reasoning_effort(
mapped_thinking: Final = AnthropicConfig.map_reasoning_effort(
reasoning_effort=reasoning_effort,
model=model,
custom_llm_provider=custom_llm_provider,
@ -414,7 +417,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
return
optional_params.setdefault("thinking", fitted_thinking)
if AnthropicModelInfo._is_adaptive_thinking_model(model, custom_llm_provider):
if AnthropicModelInfo.is_adaptive_thinking_model(model, custom_llm_provider):
mapped_effort: Final = REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT.get(reasoning_effort)
if mapped_effort is None:
raise AnthropicError(
@ -425,7 +428,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
),
status_code=400,
)
gate_error: Final = AnthropicConfig._validate_effort_for_model(model, mapped_effort, custom_llm_provider)
gate_error: Final = AnthropicConfig.validate_effort_for_model(model, mapped_effort, custom_llm_provider)
if gate_error is not None:
raise AnthropicError(message=gate_error, status_code=400)
existing_output_config = optional_params.get("output_config")
@ -479,7 +482,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
from litellm.exceptions import BadRequestError as _BadRequestError
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
if AnthropicConfig._is_adaptive_thinking_model(model, custom_llm_provider):
if AnthropicConfig.is_adaptive_thinking_model(model, custom_llm_provider):
return
output_config: Final = optional_params.get("output_config")
@ -494,20 +497,20 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
# reject. Effort-only requests pass through so provider subclasses (bedrock/vertex) keep
# owning level clamping; an adaptive request only stays here when its effort level is one
# the model supports, otherwise it falls through to the legacy budget translation below.
if AnthropicConfig._model_supports_effort_param(model, custom_llm_provider) and (
if AnthropicConfig.model_supports_effort_param(model, custom_llm_provider) and (
not adaptive_thinking
or AnthropicConfig._validate_effort_for_model(model, effort, custom_llm_provider) is None
or AnthropicConfig.validate_effort_for_model(model, effort, custom_llm_provider) is None
):
if adaptive_thinking:
optional_params.pop("thinking", None)
return
supports_thinking: Final = AnthropicModelInfo._supports_model_capability(
supports_thinking: Final = AnthropicModelInfo.supports_model_capability(
model, "supports_reasoning", custom_llm_provider
)
try:
legacy_thinking: Final = (
AnthropicConfig._map_reasoning_effort(
AnthropicConfig.map_reasoning_effort(
reasoning_effort=effort or "medium",
model=model,
custom_llm_provider=custom_llm_provider,
@ -554,7 +557,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
Adaptive models (4.6+) own this natively and are left untouched.
"""
if AnthropicModelInfo._is_adaptive_thinking_model(model, custom_llm_provider):
if AnthropicModelInfo.is_adaptive_thinking_model(model, custom_llm_provider):
return
temperature: Final = optional_params.get("temperature")
if temperature is None or temperature == 1:
@ -770,7 +773,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
if optional_params.get("speed") == "fast":
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.FAST_MODE_2026_02_01.value)
if _messages_carry_output_config(messages):
if messages_carry_output_config(messages):
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value)
tools: Final = optional_params.get("tools")

2
litellm/llms/anthropic/pass_through/messages/utils.py
View file

@ -158,7 +158,7 @@ class AnthropicMessagesRequestUtils:
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
AnthropicConfig._maybe_drop_speed_param(
AnthropicConfig.maybe_drop_speed_param(
model=model,
optional_params=filtered_params,
drop_params=drop_params,

2
litellm/llms/anthropic/pass_through/responses_adapters/transformation.py
View file

@ -75,7 +75,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
from litellm.responses.utils import ResponseAPILoggingUtils
chat_usage = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(raw_usage)
return LiteLLMAnthropicMessagesAdapter._translate_openai_usage_to_anthropic_usage(chat_usage)
return LiteLLMAnthropicMessagesAdapter.translate_openai_usage_to_anthropic_usage(chat_usage)
# ------------------------------------------------------------------ #
# Request translation: Anthropic -> Responses API #

6
litellm/llms/azure/audio_transcriptions.py
View file

@ -78,7 +78,7 @@ class AzureAudioTranscription(AzureChatCompletion):
api_key=azure_client.api_key,
additional_args={
"headers": {"Authorization": f"Bearer {azure_client.api_key}"},
"api_base": azure_client._base_url._uri_reference,
"api_base": azure_client._base_url._uri_reference, # pyright: ignore[reportPrivateUsage] # SDK URL internals
"atranscription": True,
"complete_input_dict": data,
},
@ -149,7 +149,7 @@ class AzureAudioTranscription(AzureChatCompletion):
api_key=async_azure_client.api_key,
additional_args={
"headers": {"Authorization": f"Bearer {async_azure_client.api_key}"},
"api_base": async_azure_client._base_url._uri_reference,
"api_base": async_azure_client._base_url._uri_reference, # pyright: ignore[reportPrivateUsage] # SDK URL internals
"atranscription": True,
"complete_input_dict": data,
},
@ -175,7 +175,7 @@ class AzureAudioTranscription(AzureChatCompletion):
api_key=api_key,
additional_args={
"headers": {"Authorization": f"Bearer {async_azure_client.api_key}"},
"api_base": async_azure_client._base_url._uri_reference,
"api_base": async_azure_client._base_url._uri_reference, # pyright: ignore[reportPrivateUsage] # SDK URL internals
"atranscription": True,
"complete_input_dict": data,
},

7
litellm/llms/azure/azure.py
View file

@ -112,7 +112,7 @@ class AzureOpenAIAssistantsAPIConfig:
return optional_params
def _check_dynamic_azure_params(
def check_dynamic_azure_params(
azure_client_params: dict,
azure_client: AzureOpenAI | AsyncAzureOpenAI | None,
) -> bool:
@ -127,12 +127,15 @@ def _check_dynamic_azure_params(
dynamic_params: Final = ["api_version"]
for k, v in azure_client_params.items():
if k in dynamic_params and k == "api_version":
if v is not None and v != azure_client._custom_query["api-version"]:
if v is not None and v != azure_client._custom_query["api-version"]: # pyright: ignore[reportPrivateUsage] # SDK query internals
return True
return False
_check_dynamic_azure_params = check_dynamic_azure_params
class AzureChatCompletion(BaseAzureLLM, BaseLLM):
def __init__(self) -> None:
super().__init__()

10
litellm/llms/azure/chat/gpt_5_transformation.py
View file

@ -6,7 +6,7 @@ import litellm
from litellm.exceptions import UnsupportedParamsError
from litellm.llms.openai.chat.gpt_5_transformation import (
OpenAIGPT5Config,
_get_effort_level,
get_effort_level,
is_gpt_reasoning_series_name,
)
from litellm.types.llms.openai import AllMessageValues
@ -76,7 +76,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
api_version: str = "",
) -> dict:
reasoning_effort_value = non_default_params.get("reasoning_effort") or optional_params.get("reasoning_effort")
effective_effort: Final = _get_effort_level(reasoning_effort_value)
effective_effort: Final = get_effort_level(reasoning_effort_value)
# gpt-5.1/5.2/5.4 support reasoning_effort='none', but other gpt-5 models don't
# See: https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/reasoning
@ -86,9 +86,9 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
if litellm.drop_params is True or (drop_params is not None and drop_params is True):
non_default_params = non_default_params.copy()
optional_params = optional_params.copy()
if _get_effort_level(non_default_params.get("reasoning_effort")) == "none":
if get_effort_level(non_default_params.get("reasoning_effort")) == "none":
non_default_params.pop("reasoning_effort")
if _get_effort_level(optional_params.get("reasoning_effort")) == "none":
if get_effort_level(optional_params.get("reasoning_effort")) == "none":
optional_params.pop("reasoning_effort")
else:
raise UnsupportedParamsError(
@ -111,7 +111,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
)
# Only drop reasoning_effort='none' for models that don't support it
result_effort: Final = _get_effort_level(result.get("reasoning_effort"))
result_effort: Final = get_effort_level(result.get("reasoning_effort"))
if result_effort == "none" and not supports_none:
result.pop("reasoning_effort")

33
litellm/llms/azure/common_utils.py
View file

@ -388,7 +388,7 @@ def get_azure_ad_token(
# try to get DefaultAzureCredential provider
#########################################################
if azure_ad_token_provider is None and azure_ad_token is None:
azure_ad_token_provider = BaseAzureLLM._try_get_default_azure_credential_provider(
azure_ad_token_provider = BaseAzureLLM.try_get_default_azure_credential_provider(
scope=scope,
)
@ -477,6 +477,13 @@ class BaseAzureLLM(BaseOpenAILLM):
verbose_logger.debug("DefaultAzureCredential failed: %s", e)
return None
@classmethod
def try_get_default_azure_credential_provider(
cls,
scope: str,
) -> Callable[[], str] | None:
return cls._try_get_default_azure_credential_provider(scope)
def get_azure_openai_client(
self,
api_key: str | None,
@ -511,10 +518,12 @@ class BaseAzureLLM(BaseOpenAILLM):
if (
api_version is not None
and isinstance(client, (AzureOpenAI, AsyncAzureOpenAI))
and isinstance(client._custom_query, dict)
and isinstance(client._custom_query, dict) # pyright: ignore[reportPrivateUsage] # SDK query internals
):
# set api_version to version passed by user
client._custom_query.setdefault("api-version", api_version)
client._custom_query.setdefault( # pyright: ignore[reportPrivateUsage] # SDK query internals
"api-version", api_version
)
self.set_cached_openai_client(
openai_client=client,
client_initialization_params=client_initialization_params,
@ -777,6 +786,14 @@ class BaseAzureLLM(BaseOpenAILLM):
return headers
@classmethod
def base_validate_azure_environment(
cls,
headers: dict[str, str], # mutable-ok: mirrors override contract
litellm_params: GenericLiteLLMParams | None,
) -> dict[str, str]: # mutable-ok: mirrors override contract
return cls._base_validate_azure_environment(headers, litellm_params)
@staticmethod
def _get_base_azure_url(
api_base: str | None,
@ -829,6 +846,16 @@ class BaseAzureLLM(BaseOpenAILLM):
return str(final_url)
@classmethod
def get_base_azure_url(
cls,
api_base: str | None,
litellm_params: GenericLiteLLMParams | Mapping[str, object] | None,
route: Literal["/openai/responses", "/openai/vector_stores"] | str,
default_api_version: str | Literal["latest", "preview"] | None = None,
) -> str:
return cls._get_base_azure_url(api_base, litellm_params, route, default_api_version)
@staticmethod
def get_azure_v1_image_url(api_base: str, api_version: str | None, route: str) -> str | None:
"""

6
litellm/llms/azure/completion/handler.py
View file

@ -225,7 +225,7 @@ class AzureTextCompletion(BaseAzureLLM):
api_key=azure_client.api_key,
additional_args={
"headers": {"Authorization": f"Bearer {azure_client.api_key}"},
"api_base": azure_client._base_url._uri_reference,
"api_base": azure_client._base_url._uri_reference, # pyright: ignore[reportPrivateUsage] # SDK URL internals
"acompletion": True,
"complete_input_dict": data,
},
@ -284,7 +284,7 @@ class AzureTextCompletion(BaseAzureLLM):
api_key=azure_client.api_key,
additional_args={
"headers": {"Authorization": f"Bearer {azure_client.api_key}"},
"api_base": azure_client._base_url._uri_reference,
"api_base": azure_client._base_url._uri_reference, # pyright: ignore[reportPrivateUsage] # SDK URL internals
"acompletion": True,
"complete_input_dict": data,
},
@ -334,7 +334,7 @@ class AzureTextCompletion(BaseAzureLLM):
api_key=azure_client.api_key,
additional_args={
"headers": {"Authorization": f"Bearer {azure_client.api_key}"},
"api_base": azure_client._base_url._uri_reference,
"api_base": azure_client._base_url._uri_reference, # pyright: ignore[reportPrivateUsage] # SDK URL internals
"acompletion": True,
"complete_input_dict": data,
},

4
litellm/llms/azure/containers/transformation.py
View file

@ -29,7 +29,7 @@ class AzureContainerConfig(OpenAIContainerConfig):
headers: dict,
api_key: str | None = None,
) -> dict:
return BaseAzureLLM._base_validate_azure_environment(
return BaseAzureLLM.base_validate_azure_environment(
headers=headers,
litellm_params=GenericLiteLLMParams(api_key=api_key),
)
@ -75,7 +75,7 @@ class AzureContainerConfig(OpenAIContainerConfig):
api_version_from_base: Final = self._extract_api_version(api_base)
if api_version_from_base:
effective_params["api_version"] = api_version_from_base
return BaseAzureLLM._get_base_azure_url(
return BaseAzureLLM.get_base_azure_url(
api_base=self._normalize_api_base(api_base),
litellm_params=effective_params,
route="/openai/containers",

14
litellm/llms/azure/fine_tuning/handler.py
View file

@ -8,7 +8,7 @@ from litellm._logging import verbose_logger
from litellm.llms.azure.common_utils import BaseAzureLLM
from litellm.llms.openai.fine_tuning.handler import (
OpenAIFineTuningAPI,
_litellm_fine_tuning_job_from_response,
litellm_fine_tuning_job_from_response,
)
from litellm.types.utils import LiteLLMFineTuningJob
@ -37,7 +37,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
openai_client: AsyncOpenAI | AsyncAzureOpenAI,
) -> LiteLLMFineTuningJob:
response: Final = await openai_client.fine_tuning.jobs.create(**create_fine_tuning_job_data)
return _litellm_fine_tuning_job_from_response(response, is_azure=True)
return litellm_fine_tuning_job_from_response(response, is_azure=True)
async def acancel_fine_tuning_job(
self,
@ -45,7 +45,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
openai_client: AsyncOpenAI | AsyncAzureOpenAI,
) -> LiteLLMFineTuningJob:
response: Final = await openai_client.fine_tuning.jobs.cancel(fine_tuning_job_id=fine_tuning_job_id)
return _litellm_fine_tuning_job_from_response(response, is_azure=True)
return litellm_fine_tuning_job_from_response(response, is_azure=True)
async def aretrieve_fine_tuning_job(
self,
@ -53,7 +53,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
openai_client: AsyncOpenAI | AsyncAzureOpenAI,
) -> LiteLLMFineTuningJob:
response: Final = await openai_client.fine_tuning.jobs.retrieve(fine_tuning_job_id=fine_tuning_job_id)
return _litellm_fine_tuning_job_from_response(response, is_azure=True)
return litellm_fine_tuning_job_from_response(response, is_azure=True)
def create_fine_tuning_job(
self,
@ -96,7 +96,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
verbose_logger.debug("creating fine tuning job, args= %s", create_fine_tuning_job_data)
response: Final = cast(OpenAI, openai_client).fine_tuning.jobs.create(**create_fine_tuning_job_data)
return _litellm_fine_tuning_job_from_response(response, is_azure=True)
return litellm_fine_tuning_job_from_response(response, is_azure=True)
def cancel_fine_tuning_job(
self,
@ -136,7 +136,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
)
response: Final = cast(OpenAI, openai_client).fine_tuning.jobs.cancel(fine_tuning_job_id=fine_tuning_job_id)
return _litellm_fine_tuning_job_from_response(response, is_azure=True)
return litellm_fine_tuning_job_from_response(response, is_azure=True)
def retrieve_fine_tuning_job(
self,
@ -176,7 +176,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
)
response: Final = cast(OpenAI, openai_client).fine_tuning.jobs.retrieve(fine_tuning_job_id=fine_tuning_job_id)
return _litellm_fine_tuning_job_from_response(response, is_azure=True)
return litellm_fine_tuning_job_from_response(response, is_azure=True)
def get_openai_client(
self,

2
litellm/llms/azure/image_edit/transformation.py
View file

@ -65,7 +65,7 @@ class AzureImageEditConfig(OpenAIImageEditConfig):
params: Final = GenericLiteLLMParams(**(litellm_params or {}))
if api_key is not None and params.api_key is None:
params.api_key = api_key
return BaseAzureLLM._base_validate_azure_environment(headers=headers, litellm_params=params)
return BaseAzureLLM.base_validate_azure_environment(headers=headers, litellm_params=params)
def get_complete_url(
self,

4
litellm/llms/azure/passthrough/transformation.py
View file

@ -128,7 +128,7 @@ class AzurePassthroughConfig(BasePassthroughConfig):
caller_api_version: Final = request_query_params.get("api-version") if request_query_params else None
relay_base: Final = without_api_version(base_target_url) if caller_api_version else base_target_url
complete_url: Final = BaseAzureLLM._get_base_azure_url(
complete_url: Final = BaseAzureLLM.get_base_azure_url(
api_base=relay_base,
litellm_params=MappingProxyType(
{**litellm_params, "api_version": caller_api_version or litellm_params.get("api_version")}
@ -150,7 +150,7 @@ class AzurePassthroughConfig(BasePassthroughConfig):
api_key: str | None = None,
api_base: str | None = None,
) -> dict:
return BaseAzureLLM._base_validate_azure_environment(
return BaseAzureLLM.base_validate_azure_environment(
headers=headers,
litellm_params=GenericLiteLLMParams.model_validate({**litellm_params, "api_key": api_key}),
)

10
litellm/llms/azure/realtime/handler.py
View file

@ -130,6 +130,16 @@ class AzureOpenAIRealtime(AzureChatCompletion):
qs: Final = "&".join(query_parts)
return f"{api_base}{path}?{qs}" if qs else f"{api_base}{path}"
def construct_url(
self,
api_base: str,
model: str,
api_version: str | None,
realtime_protocol: str | None = None,
query_params: RealtimeQueryParams | None = None,
) -> str:
return self._construct_url(api_base, model, api_version, realtime_protocol, query_params)
async def async_realtime(
self,
model: str,

6
litellm/llms/azure/responses/transformation.py
View file

@ -32,7 +32,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
@staticmethod
def _supports_reasoning_effort_none(model: str) -> bool:
return AzureOpenAIGPT5Config._supports_reasoning_effort_level(model, "none")
return AzureOpenAIGPT5Config.supports_reasoning_effort_level(model, "none")
@staticmethod
def _effort_resolves_to_none(model: str, effort: str | None) -> bool:
@ -46,7 +46,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
return [param for param in base_supported_params if param not in self.AZURE_UNSUPPORTED_PARAMS]
def validate_environment(self, headers: dict, model: str, litellm_params: GenericLiteLLMParams | None) -> dict:
return BaseAzureLLM._base_validate_azure_environment(headers=headers, litellm_params=litellm_params)
return BaseAzureLLM.base_validate_azure_environment(headers=headers, litellm_params=litellm_params)
def get_stripped_model_name(self, model: str) -> str:
return model.replace("responses/", "").replace("o_series/", "").replace("azure_ai/", "")
@ -169,7 +169,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
"""
from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION
return BaseAzureLLM._get_base_azure_url(
return BaseAzureLLM.get_base_azure_url(
api_base=api_base,
litellm_params=litellm_params,
route="/openai/responses",

4
litellm/llms/azure/vector_stores/transformation.py
View file

@ -9,11 +9,11 @@ class AzureOpenAIVectorStoreConfig(OpenAIVectorStoreConfig):
api_base: str | None,
litellm_params: dict,
) -> str:
return BaseAzureLLM._get_base_azure_url(
return BaseAzureLLM.get_base_azure_url(
api_base=api_base,
litellm_params=litellm_params,
route="/openai/vector_stores",
)
def validate_environment(self, headers: dict, litellm_params: GenericLiteLLMParams | None) -> dict:
return BaseAzureLLM._base_validate_azure_environment(headers=headers, litellm_params=litellm_params)
return BaseAzureLLM.base_validate_azure_environment(headers=headers, litellm_params=litellm_params)

4
litellm/llms/azure/videos/transformation.py
View file

@ -72,7 +72,7 @@ class AzureVideoConfig(OpenAIVideoConfig):
# Use the base Azure validation method which properly handles:
# 1. Credentials from litellm_credential_name via litellm_params
# 2. Sets the correct "api-key" header (not "Authorization: Bearer")
return BaseAzureLLM._base_validate_azure_environment(headers=headers, litellm_params=litellm_params)
return BaseAzureLLM.base_validate_azure_environment(headers=headers, litellm_params=litellm_params)
def get_complete_url(
self,
@ -83,7 +83,7 @@ class AzureVideoConfig(OpenAIVideoConfig):
"""
Constructs a complete URL for the API request.
"""
return BaseAzureLLM._get_base_azure_url(
return BaseAzureLLM.get_base_azure_url(
api_base=api_base,
litellm_params=litellm_params,
route="/openai/v1/videos",

6
litellm/llms/azure_ai/agents/handler.py
View file

@ -279,7 +279,7 @@ class AzureAIAgentsHandler:
headers["Authorization"] = f"Bearer {api_key}"
api_version: Final = optional_params.get("api_version", self.config.DEFAULT_API_VERSION)
agent_id: Final = self.config._get_agent_id(model, optional_params)
agent_id: Final = self.config.get_agent_id(model, optional_params)
thread_id: Final = optional_params.get("thread_id")
api_base = api_base.rstrip("/")
@ -313,10 +313,10 @@ class AzureAIAgentsHandler:
headers: dict | None = None,
) -> ModelResponse:
"""Execute synchronous completion using Azure Agent Service."""
from litellm.llms.custom_httpx.http_handler import _get_httpx_client
from litellm.llms.custom_httpx.http_handler import get_httpx_client
if client is None:
client = _get_httpx_client(params={"ssl_verify": litellm_params.get("ssl_verify", None)})
client = get_httpx_client(params={"ssl_verify": litellm_params.get("ssl_verify", None)})
(
headers,

7
litellm/llms/azure_ai/agents/transformation.py
View file

@ -184,6 +184,13 @@ class AzureAIAgentsConfig(BaseConfig):
# Extract from model name using the static method
return self.get_agent_id_from_model(model)
def get_agent_id(
self,
model: str,
optional_params: dict[str, object], # mutable-ok: mirrors override contract
) -> str:
return self._get_agent_id(model, optional_params)
def transform_request(
self,
model: str,

2
litellm/llms/azure_ai/anthropic/count_tokens/transformation.py
View file

@ -56,7 +56,7 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
litellm_params_obj: Final = GenericLiteLLMParams.model_validate(litellm_params)
# Get Azure auth headers (api-key or Authorization)
azure_headers = BaseAzureLLM._base_validate_azure_environment(headers={}, litellm_params=litellm_params_obj)
azure_headers = BaseAzureLLM.base_validate_azure_environment(headers={}, litellm_params=litellm_params_obj)
# Merge Azure auth headers
headers.update(azure_headers)

4
litellm/llms/azure_ai/anthropic/handler.py
View file

@ -177,9 +177,9 @@ class AzureAnthropicChatCompletion(AnthropicChatCompletion):
else:
if client is None or not isinstance(client, HTTPHandler):
from litellm.llms.custom_httpx.http_handler import _get_httpx_client
from litellm.llms.custom_httpx.http_handler import get_httpx_client
client = _get_httpx_client(params={"timeout": timeout})
client = get_httpx_client(params={"timeout": timeout})
else:
client = client

2
litellm/llms/azure_ai/anthropic/messages_transformation.py
View file

@ -50,7 +50,7 @@ class AzureAnthropicMessagesConfig(AnthropicMessagesConfig):
litellm_params_obj.api_key = api_key
# Use Azure authentication logic
headers = BaseAzureLLM._base_validate_azure_environment(headers=headers, litellm_params=litellm_params_obj)
headers = BaseAzureLLM.base_validate_azure_environment(headers=headers, litellm_params=litellm_params_obj)
# Azure Anthropic uses x-api-key header (not api-key)
# Convert api-key to x-api-key if present

2
litellm/llms/azure_ai/anthropic/transformation.py
View file

@ -73,7 +73,7 @@ class AzureAnthropicConfig(AnthropicConfig):
litellm_params_obj.api_key = api_key
# Use Azure authentication logic
headers = BaseAzureLLM._base_validate_azure_environment(headers=headers, litellm_params=litellm_params_obj)
headers = BaseAzureLLM.base_validate_azure_environment(headers=headers, litellm_params=litellm_params_obj)
# Get tools and other anthropic-specific setup
tools: Final = optional_params.get("tools")

13
litellm/llms/azure_ai/chat/transformation.py
View file

@ -117,7 +117,7 @@ class AzureAIStudioConfig(OpenAIConfig):
if "grok" in model:
# Reuse Xai method for Grok model
xai_config: Final = XAIChatConfig()
return xai_config._supports_stop_reason(model)
return xai_config.supports_stop_reason(model)
return True
def validate_environment(
@ -138,7 +138,7 @@ class AzureAIStudioConfig(OpenAIConfig):
else:
# No api_key provided — fall back to Azure AD token-based auth
litellm_params_obj = GenericLiteLLMParams(**(litellm_params if isinstance(litellm_params, dict) else {}))
headers = BaseAzureLLM._base_validate_azure_environment(headers=headers, litellm_params=litellm_params_obj)
headers = BaseAzureLLM.base_validate_azure_environment(headers=headers, litellm_params=litellm_params_obj)
headers["Content-Type"] = "application/json"
@ -281,6 +281,15 @@ class AzureAIStudioConfig(OpenAIConfig):
custom_llm_provider = "azure"
return api_base, dynamic_api_key, custom_llm_provider
def get_openai_compatible_provider_info(
self,
model: str,
api_base: str | None,
api_key: str | None,
custom_llm_provider: str,
) -> tuple[str | None, str | None, str]:
return self._get_openai_compatible_provider_info(model, api_base, api_key, custom_llm_provider)
def transform_request(
self,
model: str,

14
litellm/llms/azure_ai/embed/cohere_transformation.py
View file

@ -67,6 +67,14 @@ class AzureAICohereConfig:
return image_embeddings_request, v1_embeddings_request, image_embedding_idx
def transform_request(
self,
input: list[str], # mutable-ok: mirrors override contract
optional_params: dict[str, object], # mutable-ok: mirrors override contract
model: str,
) -> tuple[ImageEmbeddingRequest, EmbeddingCreateParams, list[int]]: # mutable-ok: mirrors override contract
return self._transform_request(input, optional_params, model)
def _transform_response(self, response: EmbeddingResponse) -> EmbeddingResponse:
additional_headers: Final[dict | None] = response.hidden_params.get("additional_headers")
if additional_headers:
@ -84,3 +92,9 @@ class AzureAICohereConfig:
response.model = self._map_azure_model_group(base_model)
return response
def transform_response(
self,
response: EmbeddingResponse,
) -> EmbeddingResponse:
return self._transform_response(response)

6
litellm/llms/azure_ai/embed/handler.py
View file

@ -58,7 +58,7 @@ class AzureAIEmbedding(OpenAIChatCompletion):
elif text_embedding_responses is not None:
model_response.data = text_embedding_responses
response: Final = AzureAICohereConfig()._transform_response(response=model_response)
response: Final = AzureAICohereConfig().transform_response(response=model_response)
return response
@ -158,7 +158,7 @@ class AzureAIEmbedding(OpenAIChatCompletion):
image_embeddings_request,
v1_embeddings_request,
image_embeddings_idx,
) = AzureAICohereConfig()._transform_request(input=input, optional_params=optional_params, model=model)
) = AzureAICohereConfig().transform_request(input=input, optional_params=optional_params, model=model)
image_embedding_responses: list | None = None
text_embedding_responses: list | None = None
@ -246,7 +246,7 @@ class AzureAIEmbedding(OpenAIChatCompletion):
image_embeddings_request,
v1_embeddings_request,
image_embeddings_idx,
) = AzureAICohereConfig()._transform_request(input=input, optional_params=optional_params, model=model)
) = AzureAICohereConfig().transform_request(input=input, optional_params=optional_params, model=model)
image_embedding_responses: list | None = None
text_embedding_responses: list | None = None

7
litellm/llms/base_llm/base_model_iterator.py
View file

@ -108,6 +108,13 @@ class BaseModelResponseIterator:
stripped_json_chunk = None
return stripped_json_chunk
@classmethod
def string_to_dict_parser(
cls,
str_line: str,
) -> dict[str, object] | None: # mutable-ok: mirrors override contract
return cls._string_to_dict_parser(str_line)
def _handle_string_chunk(self, str_line: str) -> GenericStreamingChunk | ModelResponseStream:
# chunk is a str at this point
stripped_json_chunk: Final = BaseModelResponseIterator._string_to_dict_parser(str_line=str_line)

9
litellm/llms/base_llm/base_utils.py
View file

@ -8,7 +8,7 @@ from abc import ABC, abstractmethod
from collections.abc import Mapping
from typing import Any, Final
from openai.lib import _parsing, _pydantic
from openai.lib import _parsing, _pydantic # pyright: ignore[reportPrivateUsage] # SDK parser internals
from pydantic import BaseModel
from litellm._logging import verbose_logger
@ -128,7 +128,7 @@ class BaseLLMModelInfo(ABC):
return None
def _convert_tool_response_to_message(
def convert_tool_response_to_message(
tool_calls: list[ChatCompletionToolCallChunk],
) -> Message | None:
"""
@ -154,6 +154,9 @@ def _convert_tool_response_to_message(
return None
_convert_tool_response_to_message = convert_tool_response_to_message
def _dict_to_response_format_helper(response_format: dict, ref_template: str | None = None) -> dict:
if ref_template is not None and response_format.get("type") == "json_schema":
# Deep copy to avoid modifying original
@ -211,7 +214,7 @@ def type_to_response_format_param(
# type checkers don't narrow the negation of a `TypeGuard` as it isn't
# a safe default behaviour but we know that at this point the `response_format`
# can only be a `type`
if not _parsing._completions.is_basemodel_type(response_format):
if not _parsing._completions.is_basemodel_type(response_format): # pyright: ignore[reportPrivateUsage] # SDK parser internals
raise TypeError(f"Unsupported response_format type - {response_format}")
if ref_template is not None:

7
litellm/llms/base_llm/chat/transformation.py
View file

@ -146,6 +146,13 @@ class BaseConfig(ABC):
]
return optional_params
def add_tools_to_optional_params(
self,
optional_params: dict[str, object], # mutable-ok: mirrors override contract
tools: list[ChatCompletionToolParam], # mutable-ok: mirrors override contract
) -> dict[str, object]: # mutable-ok: mirrors override contract
return self._add_tools_to_optional_params(optional_params, tools)
def translate_developer_role_to_system_role(
self,
messages: list[AllMessageValues],

8
litellm/llms/base_llm/passthrough/transformation.py
View file

@ -122,7 +122,7 @@ class RawBytesStreamCollector:
self._raw_bytes.append(chunk)
def build_logged_response(self, litellm_logging_obj: LiteLLMLoggingObj) -> LoggedRelayResponse | None:
all_chunks: Final = self._provider_config._convert_raw_bytes_to_str_lines(self._raw_bytes)
all_chunks: Final = self._provider_config.convert_raw_bytes_to_str_lines(self._raw_bytes)
return self._provider_config.handle_logging_collected_chunks(
all_chunks=all_chunks,
litellm_logging_obj=litellm_logging_obj,
@ -258,3 +258,9 @@ class BasePassthroughConfig(BaseLLMModelInfo):
lines: Final = [line.strip() for line in combined_str.split("\n") if line.strip()]
return lines
def convert_raw_bytes_to_str_lines(
self,
raw_bytes: list[bytes], # mutable-ok: mirrors override contract
) -> list[str]: # mutable-ok: mirrors override contract
return self._convert_raw_bytes_to_str_lines(raw_bytes)

15
litellm/llms/bedrock/base_aws_llm.py
View file

@ -790,6 +790,14 @@ class BaseAWSLLM(SignsRequestsWithAWS):
self._validate_aws_region_name(aws_region_name)
return aws_region_name
def get_aws_region_name(
self,
optional_params: dict[str, object], # mutable-ok: mirrors override contract
model: str | None = None,
model_id: str | None = None,
) -> str:
return self._get_aws_region_name(optional_params, model, model_id)
@staticmethod
def _validate_aws_region_name(aws_region_name: str | None) -> None:
"""
@ -804,6 +812,13 @@ class BaseAWSLLM(SignsRequestsWithAWS):
"Region names must contain only lowercase letters, digits, and hyphens."
)
@classmethod
def validate_aws_region_name(
cls,
aws_region_name: str | None,
) -> None:
return cls._validate_aws_region_name(aws_region_name)
@staticmethod
def _parse_sts_region_from_endpoint(
aws_sts_endpoint: str | None,

12
litellm/llms/bedrock/batches/handler.py
View file

@ -167,7 +167,7 @@ class BedrockBatchesHandler:
)
def job_status() -> "LiteLLMBatch":
return BedrockBatchesHandler._handle_model_invocation_job_status(
return BedrockBatchesHandler.handle_model_invocation_job_status(
batch_id=batch_id,
aws_region_name=region,
logging_obj=logging_obj,
@ -187,7 +187,7 @@ class BedrockBatchesHandler:
return job_status()
@staticmethod
def _handle_async_invoke_status(batch_id: str, aws_region_name: str, logging_obj=None, **kwargs) -> "LiteLLMBatch":
def handle_async_invoke_status(batch_id: str, aws_region_name: str, logging_obj=None, **kwargs) -> "LiteLLMBatch":
"""
Handle async invoke status check for AWS Bedrock.
@ -210,7 +210,7 @@ class BedrockBatchesHandler:
embedding_handler: Final = BedrockEmbedding()
# Get the status of the async invoke job
status_response: Final = await embedding_handler._get_async_invoke_status(
status_response: Final = await embedding_handler.get_async_invoke_status(
invocation_arn=batch_id,
aws_region_name=aws_region_name,
logging_obj=logging_obj,
@ -263,8 +263,10 @@ class BedrockBatchesHandler:
future: Final = executor.submit(run_in_thread)
return future.result()
_handle_async_invoke_status = handle_async_invoke_status
@staticmethod
def _handle_model_invocation_job_status(
def handle_model_invocation_job_status(
batch_id: str,
aws_region_name: str | None = None,
logging_obj=None,
@ -406,3 +408,5 @@ class BedrockBatchesHandler:
input_file_id=input_uri,
output_file_id=output_file_uri if openai_status == "completed" else None,
)
_handle_model_invocation_job_status = handle_model_invocation_job_status

7
litellm/llms/bedrock/batches/transformation.py
View file

@ -507,6 +507,13 @@ class BedrockBatchesConfig(BaseAWSLLM, BaseBatchesConfig):
expires_at,
)
def parse_timestamps_and_status(
self,
response_data: Mapping[str, object],
status_str: str,
) -> tuple[int | None, int | None, int | None, int | None, int | None, int | None]:
return self._parse_timestamps_and_status(response_data, status_str)
def _extract_file_configs(self, response_data):
"""Helper to extract input and output file configurations."""
# Extract input file ID

16
litellm/llms/bedrock/chat/agentcore/transformation.py
View file

@ -193,6 +193,12 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
verbose_logger.debug("Generated new session ID: %s", generated_id)
return generated_id
def get_runtime_session_id(
self,
optional_params: dict[str, object], # mutable-ok: mirrors override contract
) -> str:
return self._get_runtime_session_id(optional_params)
def _get_runtime_user_id(self, optional_params: dict) -> str | None:
"""
Get runtime user ID if provided
@ -202,6 +208,12 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
verbose_logger.debug("Using provided runtimeUserId: %s", user_id)
return user_id
def get_runtime_user_id(
self,
optional_params: dict[str, object], # mutable-ok: mirrors override contract
) -> str | None:
return self._get_runtime_user_id(optional_params)
def transform_request(
self,
model: str,
@ -652,11 +664,11 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
"""
from litellm.llms.custom_httpx.http_handler import (
HTTPHandler,
_get_httpx_client,
get_httpx_client,
)
if client is None or not isinstance(client, HTTPHandler):
client = _get_httpx_client(params={})
client = get_httpx_client(params={})
verbose_logger.debug("Making sync streaming request to: %s", api_base)

10
litellm/llms/bedrock/chat/converse_handler.py
View file

@ -12,14 +12,14 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
_get_httpx_client,
get_async_httpx_client,
get_httpx_client,
)
from litellm.types.utils import ModelResponse
from litellm.utils import CustomStreamWrapper
from ..base_aws_llm import BaseAWSLLM, Credentials, bedrock_bearer_token, pop_aws_auth_params, run_aws_signing
from ..common_utils import BedrockError, _get_all_bedrock_regions, error_response_text
from ..common_utils import BedrockError, error_response_text, get_all_bedrock_regions
from .invoke_handler import AWSEventStreamDecoder, MockResponseIterator, make_call
@ -37,7 +37,7 @@ def make_sync_call(
timeout: float | httpx.Timeout | None = None,
) -> tuple[Any, httpx.Headers]:
if client is None:
client = _get_httpx_client() # Create a new client if none provided
client = get_httpx_client() # Create a new client if none provided
response: Final = client.post(
api_base,
@ -302,7 +302,7 @@ class BedrockConverseLLM(BaseAWSLLM):
# and capture it so it can be used as aws_region_name below.
_region_from_model: str | None = None
_potential_region: Final = _stripped.split("/", 1)[0]
if _potential_region in _get_all_bedrock_regions() and "/" in _stripped:
if _potential_region in get_all_bedrock_regions() and "/" in _stripped:
_region_from_model = _potential_region
_stripped = _stripped.split("/", 1)[1]
_model_for_id = _stripped
@ -443,7 +443,7 @@ class BedrockConverseLLM(BaseAWSLLM):
if isinstance(timeout, float) or isinstance(timeout, int):
timeout = httpx.Timeout(timeout)
_params["timeout"] = timeout
client = _get_httpx_client(_params)
client = get_httpx_client(_params)
else:
client = client

20
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -552,7 +552,7 @@ class AmazonConverseConfig(BaseConfig):
reasoning_config: Final = self._transform_reasoning_effort_to_reasoning_config(reasoning_effort)
optional_params.update(reasoning_config)
else:
mapped_thinking: Final = AnthropicConfig._map_reasoning_effort(
mapped_thinking: Final = AnthropicConfig.map_reasoning_effort(
reasoning_effort=reasoning_effort,
model=model,
custom_llm_provider="bedrock",
@ -563,10 +563,10 @@ class AmazonConverseConfig(BaseConfig):
optional_params.pop("output_config", None)
else:
optional_params["thinking"] = mapped_thinking
if AnthropicConfig._is_adaptive_thinking_model(model, "bedrock"):
if AnthropicConfig.is_adaptive_thinking_model(model, "bedrock"):
mapped_effort = REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT.get(reasoning_effort)
if mapped_effort is None:
AnthropicConfig._raise_invalid_reasoning_effort(
AnthropicConfig.raise_invalid_reasoning_effort(
model=model,
value=reasoning_effort,
llm_provider="bedrock_converse",
@ -598,7 +598,7 @@ class AmazonConverseConfig(BaseConfig):
model=model,
llm_provider="bedrock_converse",
)
error = AnthropicConfig._validate_effort_for_model(model=model, effort=effort, custom_llm_provider="bedrock")
error = AnthropicConfig.validate_effort_for_model(model=model, effort=effort, custom_llm_provider="bedrock")
if error is not None:
raise litellm.exceptions.BadRequestError(
message=error,
@ -1062,7 +1062,7 @@ class AmazonConverseConfig(BaseConfig):
optional_params["stopSequences"] = value
if param == "temperature" or param == "top_p":
if base_model.startswith("anthropic"):
AnthropicConfig._apply_sampling_param(
AnthropicConfig.apply_sampling_param(
optional_params=optional_params,
model=model,
param=param,
@ -1109,10 +1109,10 @@ class AmazonConverseConfig(BaseConfig):
if (
isinstance(value, dict)
and value.get("type") == "adaptive"
and not AnthropicConfig._is_adaptive_thinking_model(model, "bedrock")
and not AnthropicConfig.is_adaptive_thinking_model(model, "bedrock")
):
max_tokens = non_default_params.get("max_completion_tokens") or non_default_params.get("max_tokens")
legacy_thinking = AnthropicConfig._map_reasoning_effort(
legacy_thinking = AnthropicConfig.map_reasoning_effort(
reasoning_effort="medium",
model=model,
custom_llm_provider="bedrock",
@ -1557,7 +1557,7 @@ class AmazonConverseConfig(BaseConfig):
if val_top_k is not None:
if base_model.startswith("anthropic"):
top_k_params: Final[dict] = {}
AnthropicConfig._apply_sampling_param(
AnthropicConfig.apply_sampling_param(
optional_params=top_k_params,
model=model,
param="top_k",
@ -1695,7 +1695,7 @@ class AmazonConverseConfig(BaseConfig):
if is_bedrock_application_inference_profile_arn(model):
additional_request_params["output_config"] = anthropic_output_config
elif base_model.startswith("anthropic"):
if litellm.drop_params is True and not AnthropicConfig._model_supports_effort_param(model, "bedrock"):
if litellm.drop_params is True and not AnthropicConfig.model_supports_effort_param(model, "bedrock"):
litellm.verbose_logger.warning(
DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING,
model,
@ -1846,7 +1846,7 @@ class AmazonConverseConfig(BaseConfig):
if (
isinstance(output_config, dict)
and output_config.get("effort") is not None
and not AnthropicConfig._is_adaptive_thinking_model(model, "bedrock")
and not AnthropicConfig.is_adaptive_thinking_model(model, "bedrock")
):
from litellm.types.llms.anthropic import (
ANTHROPIC_EFFORT_BETA_HEADER,

16
litellm/llms/bedrock/chat/invoke_handler.py
View file

@ -18,8 +18,8 @@ from litellm.llms.anthropic.chat.handler import (
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
_get_httpx_client,
get_async_httpx_client,
get_httpx_client,
)
from litellm.types.llms.bedrock import *
from litellm.types.llms.openai import (
@ -297,7 +297,7 @@ def make_sync_call(
) -> "tuple[MockResponseIterator | Iterator[GChunk | ModelResponseStream | dict], httpx.Headers]":
try:
if client is None:
client = _get_httpx_client(
client = get_httpx_client(
params=(
{"ssl_verify": logging_obj.litellm_params.get("ssl_verify")}
if logging_obj and logging_obj.litellm_params and logging_obj.litellm_params.get("ssl_verify")
@ -777,6 +777,12 @@ class AWSEventStreamDecoder:
tool_use=None,
)
def chunk_parser(
self,
chunk_data: dict[str, object], # mutable-ok: mirrors override contract
) -> GChunk | ModelResponseStream | dict[str, object]: # mutable-ok: mirrors override contract
return self._chunk_parser(chunk_data)
def iter_bytes(
self, iterator: Iterator[bytes], *, response_headers: Mapping[str, str] | None = None
) -> Iterator[GChunk | ModelResponseStream | dict]:
@ -793,7 +799,7 @@ class AWSEventStreamDecoder:
if message:
# sse_event = ServerSentEvent(data=message, event="completion")
_data = json.loads(message)
yield self._chunk_parser(chunk_data=_data)
yield self.chunk_parser(chunk_data=_data)
undecoded_stream_error: Final = tally.undecoded_stream_error(response_headers)
if undecoded_stream_error is not None:
raise undecoded_stream_error
@ -813,7 +819,7 @@ class AWSEventStreamDecoder:
message = self._decode_event(event, tally)
if message:
_data = json.loads(message)
yield self._chunk_parser(chunk_data=_data)
yield self.chunk_parser(chunk_data=_data)
undecoded_stream_error: Final = tally.undecoded_stream_error(response_headers)
if undecoded_stream_error is not None:
raise undecoded_stream_error
@ -955,7 +961,7 @@ class MockResponseIterator: # for returning ai21 streaming responses
"""
tool_use: ChatCompletionToolCallChunk | None = None
if self.json_mode is True and tool_calls is not None:
message: Final = litellm.AnthropicConfig()._convert_tool_response_to_message(tool_calls=tool_calls)
message: Final = litellm.AnthropicConfig().convert_tool_response_to_message(tool_calls=tool_calls)
if message is not None:
text = message.content or ""
tool_use = None

4
litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py
View file

@ -28,7 +28,7 @@ from litellm.llms.bedrock.request_metadata import (
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
_get_httpx_client,
get_httpx_client,
)
from litellm.types.llms.bedrock import GuardrailConfigBlock
from litellm.types.llms.openai import AllMessageValues
@ -499,7 +499,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
timeout: float | httpx.Timeout | None = None,
) -> CustomStreamWrapper:
sync_client: Final = (
_get_httpx_client(params={}) if client is None or isinstance(client, AsyncHTTPHandler) else client
get_httpx_client(params={}) if client is None or isinstance(client, AsyncHTTPHandler) else client
)
chunk_size: Final = stored_control_options(litellm_params).stream_chunk_size
completion_stream, response_headers = make_sync_call(

15
litellm/llms/bedrock/common_utils.py
View file

@ -313,7 +313,7 @@ def strip_unsupported_bedrock_invoke_output_config_keys(
return
if all(key == "format" for key in output_config):
return
if _bedrock_model_supports(model, "supports_output_config") or AnthropicConfig._model_supports_effort_param(
if _bedrock_model_supports(model, "supports_output_config") or AnthropicConfig.model_supports_effort_param(
model, "bedrock"
):
return
@ -773,7 +773,7 @@ def get_bedrock_tool_name(response_tool_name: str) -> str:
_BEDROCK_GLOBAL_REGIONS: list[str] | None = None
def _get_all_bedrock_regions() -> list[str]:
def get_all_bedrock_regions() -> list[str]:
"""Get all Bedrock regions, cached at module level."""
global _BEDROCK_GLOBAL_REGIONS
if _BEDROCK_GLOBAL_REGIONS is None:
@ -781,6 +781,9 @@ def _get_all_bedrock_regions() -> list[str]:
return _BEDROCK_GLOBAL_REGIONS
_get_all_bedrock_regions = get_all_bedrock_regions
def get_bedrock_cross_region_inference_regions() -> list[str]:
"""Abbreviations of regions AWS Bedrock supports for cross region inference."""
return ["global", "us", "eu", "apac", "jp", "au", "us-gov"]
@ -829,7 +832,7 @@ def split_bedrock_region_path(model: str) -> tuple[str | None, str]:
"""
stripped: Final = strip_bedrock_routing_prefix(model)
region, separator, model_id = stripped.partition("/")
if separator and region in _get_all_bedrock_regions():
if separator and region in get_all_bedrock_regions():
return region, model_id
return None, stripped
@ -1104,7 +1107,7 @@ def get_bedrock_base_model(model: str) -> str:
if potential_region in get_bedrock_cross_region_inference_regions():
return model.split(".", 1)[1]
elif alt_potential_region in _get_all_bedrock_regions() and len(model.split("/", 1)) > 1:
elif alt_potential_region in get_all_bedrock_regions() and len(model.split("/", 1)) > 1:
return model.split("/", 1)[1]
return model
@ -1174,7 +1177,7 @@ def bedrock_supports_tool_search(model: str) -> bool:
"""
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
return AnthropicModelInfo._supports_model_capability(model, "supports_tool_search", "bedrock")
return AnthropicModelInfo.supports_model_capability(model, "supports_tool_search", "bedrock")
def is_claude_4_5_on_bedrock(model: str) -> bool:
@ -2055,7 +2058,7 @@ class CommonBatchFilesUtils:
except ImportError:
raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.")
aws_region_name: Final = self._base_aws._get_aws_region_name(optional_params=optional_params, model="")
aws_region_name: Final = self._base_aws.get_aws_region_name(optional_params=optional_params, model="")
credentials: Final = self._base_aws.resolve_credentials(
AwsAuthParams.model_validate(optional_params), aws_region_name
)

25
litellm/llms/bedrock/embed/amazon_nova_transformation.py
View file

@ -204,6 +204,16 @@ class AmazonNovaEmbeddingConfig:
return request
def transform_request(
self,
input: str,
inference_params: dict[str, object], # mutable-ok: mirrors override contract
async_invoke_route: bool = False,
model_id: str | None = None,
output_s3_uri: str | None = None,
) -> dict[str, object]: # mutable-ok: mirrors override contract
return self._transform_request(input, inference_params, async_invoke_route, model_id, output_s3_uri)
def _wrap_async_invoke_request(
self,
model_input: dict,
@ -316,6 +326,14 @@ class AmazonNovaEmbeddingConfig:
return EmbeddingResponse(data=embeddings, model=model, usage=usage)
def transform_response(
self,
response_list: list[dict[str, object]], # mutable-ok: mirrors override contract
model: str,
batch_data: list[dict[str, object]] | None = None, # mutable-ok: mirrors override contract
) -> EmbeddingResponse:
return self._transform_response(response_list, model, batch_data)
def _transform_async_invoke_response(self, response: dict, model: str) -> EmbeddingResponse:
"""
Transform async invoke response (invocation ARN) to OpenAI format.
@ -351,3 +369,10 @@ class AmazonNovaEmbeddingConfig:
usage=usage,
hidden_params=hidden_params,
)
def transform_async_invoke_response(
self,
response: dict[str, object], # mutable-ok: mirrors override contract
model: str,
) -> EmbeddingResponse:
return self._transform_async_invoke_response(response, model)

14
litellm/llms/bedrock/embed/amazon_titan_g1_transformation.py
View file

@ -60,6 +60,13 @@ class AmazonTitanG1Config:
def _transform_request(self, input: str, inference_params: dict) -> AmazonTitanG1EmbeddingRequest:
return AmazonTitanG1EmbeddingRequest(inputText=input)
def transform_request(
self,
input: str,
inference_params: dict[str, object], # mutable-ok: mirrors override contract
) -> AmazonTitanG1EmbeddingRequest:
return self._transform_request(input, inference_params)
def _transform_response(self, response_list: list[dict], model: str) -> EmbeddingResponse:
total_prompt_tokens = 0
@ -81,3 +88,10 @@ class AmazonTitanG1Config:
total_tokens=total_prompt_tokens,
)
return EmbeddingResponse(model=model, usage=usage, data=transformed_responses)
def transform_response(
self,
response_list: list[dict[str, object]], # mutable-ok: mirrors override contract
model: str,
) -> EmbeddingResponse:
return self._transform_response(response_list, model)

15
litellm/llms/bedrock/embed/amazon_titan_multimodal_transformation.py
View file

@ -52,6 +52,13 @@ class AmazonTitanMultimodalEmbeddingG1Config:
transformed_request[k] = v
return transformed_request
def transform_request(
self,
input: str,
inference_params: dict[str, object], # mutable-ok: mirrors override contract
) -> AmazonTitanMultimodalEmbeddingRequest:
return self._transform_request(input, inference_params)
def _transform_response(
self,
response_list: list[dict],
@ -91,3 +98,11 @@ class AmazonTitanMultimodalEmbeddingG1Config:
prompt_tokens_details=prompt_tokens_details,
)
return EmbeddingResponse(model=model, usage=usage, data=transformed_responses)
def transform_response(
self,
response_list: list[dict[str, object]], # mutable-ok: mirrors override contract
model: str,
batch_data: list[dict[str, object]] | None = None, # mutable-ok: mirrors override contract
) -> EmbeddingResponse:
return self._transform_response(response_list, model, batch_data)

14
litellm/llms/bedrock/embed/amazon_titan_v2_transformation.py
View file

@ -77,6 +77,13 @@ class AmazonTitanV2Config:
def _transform_request(self, input: str, inference_params: dict) -> AmazonTitanV2EmbeddingRequest:
return AmazonTitanV2EmbeddingRequest(inputText=input, **inference_params)
def transform_request(
self,
input: str,
inference_params: dict[str, object], # mutable-ok: mirrors override contract
) -> AmazonTitanV2EmbeddingRequest:
return self._transform_request(input, inference_params)
def _transform_response(self, response_list: list[dict], model: str) -> EmbeddingResponse:
total_prompt_tokens = 0
@ -116,3 +123,10 @@ class AmazonTitanV2Config:
total_tokens=total_prompt_tokens,
)
return EmbeddingResponse(model=model, usage=usage, data=transformed_responses)
def transform_response(
self,
response_list: list[dict[str, object]], # mutable-ok: mirrors override contract
model: str,
) -> EmbeddingResponse:
return self._transform_response(response_list, model)

8
litellm/llms/bedrock/embed/cohere_transformation.py
View file

@ -41,3 +41,11 @@ class BedrockCohereEmbeddingConfig:
new_transformed_request[k] = transformed_request[k]
return new_transformed_request
def transform_request(
self,
model: str,
input: list[str], # mutable-ok: mirrors override contract
inference_params: dict[str, object], # mutable-ok: mirrors override contract
) -> CohereEmbeddingRequest:
return self._transform_request(model, input, inference_params)

44
litellm/llms/bedrock/embed/embedding.py
View file

@ -16,13 +16,14 @@ from litellm.llms.cohere.embed.handler import embedding as cohere_embedding
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
_get_httpx_client,
get_async_httpx_client,
get_httpx_client,
)
from litellm.secret_managers.main import get_secret
from litellm.types.llms.bedrock import (
AmazonEmbeddingRequest,
CohereEmbeddingRequest,
TwelveLabsAsyncInvokeStatusResponse,
)
from litellm.types.utils import EmbeddingResponse, LlmProviders
@ -122,7 +123,7 @@ class BedrockEmbedding(BaseAWSLLM):
if isinstance(timeout, float) or isinstance(timeout, int):
timeout = httpx.Timeout(timeout)
_params["timeout"] = timeout
client = _get_httpx_client(_params)
client = get_httpx_client(_params)
else:
client = client
try:
@ -191,11 +192,11 @@ class BedrockEmbedding(BaseAWSLLM):
# Handle async invoke responses (single response with invocationArn)
if is_async_invoke and len(response_list) == 1 and "invocationArn" in response_list[0]:
if provider == "twelvelabs":
returned_response = TwelveLabsMarengoEmbeddingConfig()._transform_async_invoke_response(
returned_response = TwelveLabsMarengoEmbeddingConfig().transform_async_invoke_response(
response=response_list[0], model=model
)
elif provider == "nova":
returned_response = AmazonNovaEmbeddingConfig()._transform_async_invoke_response(
returned_response = AmazonNovaEmbeddingConfig().transform_async_invoke_response(
response=response_list[0], model=model
)
else:
@ -226,21 +227,21 @@ class BedrockEmbedding(BaseAWSLLM):
else:
# Handle regular invoke responses
if model == "amazon.titan-embed-image-v1":
returned_response = AmazonTitanMultimodalEmbeddingG1Config()._transform_response(
returned_response = AmazonTitanMultimodalEmbeddingG1Config().transform_response(
response_list=response_list, model=model, batch_data=batch_data
)
elif model == "amazon.titan-embed-text-v1":
returned_response = AmazonTitanG1Config()._transform_response(response_list=response_list, model=model)
returned_response = AmazonTitanG1Config().transform_response(response_list=response_list, model=model)
elif model == "amazon.titan-embed-text-v2:0":
returned_response = AmazonTitanV2Config()._transform_response(response_list=response_list, model=model)
returned_response = AmazonTitanV2Config().transform_response(response_list=response_list, model=model)
elif model == "amazon.titan-embed-g1-text-02":
returned_response = AmazonTitanG1Config()._transform_response(response_list=response_list, model=model)
returned_response = AmazonTitanG1Config().transform_response(response_list=response_list, model=model)
elif provider == "twelvelabs":
returned_response = TwelveLabsMarengoEmbeddingConfig()._transform_response(
returned_response = TwelveLabsMarengoEmbeddingConfig().transform_response(
response_list=response_list, model=model, batch_data=batch_data
)
elif provider == "nova":
returned_response = AmazonNovaEmbeddingConfig()._transform_response(
returned_response = AmazonNovaEmbeddingConfig().transform_response(
response_list=response_list, model=model, batch_data=batch_data
)
@ -438,7 +439,7 @@ class BedrockEmbedding(BaseAWSLLM):
data: CohereEmbeddingRequest | None = None
batch_data: list | None = None
if provider == "cohere":
data = BedrockCohereEmbeddingConfig()._transform_request(
data = BedrockCohereEmbeddingConfig().transform_request(
model=model, input=input, inference_params=inference_params
)
elif provider == "amazon" and model in [
@ -451,20 +452,20 @@ class BedrockEmbedding(BaseAWSLLM):
for i in input:
if model == "amazon.titan-embed-image-v1":
transformed_request: AmazonEmbeddingRequest = (
AmazonTitanMultimodalEmbeddingG1Config()._transform_request(
AmazonTitanMultimodalEmbeddingG1Config().transform_request(
input=i, inference_params=inference_params
)
)
elif model == "amazon.titan-embed-text-v1":
transformed_request = AmazonTitanG1Config()._transform_request(
transformed_request = AmazonTitanG1Config().transform_request(
input=i, inference_params=inference_params
)
elif model == "amazon.titan-embed-text-v2:0":
transformed_request = AmazonTitanV2Config()._transform_request(
transformed_request = AmazonTitanV2Config().transform_request(
input=i, inference_params=inference_params
)
elif model == "amazon.titan-embed-g1-text-02":
transformed_request = AmazonTitanG1Config()._transform_request(
transformed_request = AmazonTitanG1Config().transform_request(
input=i, inference_params=inference_params
)
else:
@ -483,7 +484,7 @@ class BedrockEmbedding(BaseAWSLLM):
elif provider == "twelvelabs":
batch_data = []
for i in input:
twelvelabs_request = TwelveLabsMarengoEmbeddingConfig(model=model)._transform_request(
twelvelabs_request = TwelveLabsMarengoEmbeddingConfig(model=model).transform_request(
input=i,
inference_params=inference_params,
async_invoke_route=has_async_invoke,
@ -495,7 +496,7 @@ class BedrockEmbedding(BaseAWSLLM):
elif provider == "nova":
batch_data = []
for i in input:
nova_request = AmazonNovaEmbeddingConfig()._transform_request(
nova_request = AmazonNovaEmbeddingConfig().transform_request(
input=i,
inference_params=inference_params,
async_invoke_route=has_async_invoke,
@ -665,3 +666,12 @@ class BedrockEmbedding(BaseAWSLLM):
return response.json()
else:
raise Exception(f"Failed to get async invoke status: {response.status_code} - {response.text}")
async def get_async_invoke_status(
self,
invocation_arn: str,
aws_region_name: str,
logging_obj: "LiteLLMLoggingObj | None" = None,
**kwargs: object, # kwargs-ok: mirrors private method extension kwargs
) -> TwelveLabsAsyncInvokeStatusResponse:
return await self._get_async_invoke_status(invocation_arn, aws_region_name, logging_obj, **kwargs)

28
litellm/llms/bedrock/embed/twelvelabs_marengo_transformation.py
View file

@ -272,6 +272,19 @@ class TwelveLabsMarengoEmbeddingConfig:
return transformed_request
def transform_request(
self,
input: str,
inference_params: dict[str, object], # mutable-ok: mirrors override contract
async_invoke_route: bool = False,
model_id: str | None = None,
output_s3_uri: str | None = None,
drop_params: bool = False,
) -> TwelveLabsMarengoEmbeddingRequest | TwelveLabsMarengo3EmbeddingRequest | TwelveLabsAsyncInvokeRequest:
return self._transform_request(
input, inference_params, async_invoke_route, model_id, output_s3_uri, drop_params
)
def _wrap_async_invoke_request(
self,
model_input: TwelveLabsMarengoEmbeddingRequest | TwelveLabsMarengo3EmbeddingRequest,
@ -319,6 +332,14 @@ class TwelveLabsMarengoEmbeddingConfig:
]
return EmbeddingResponse(data=embeddings, model=model, usage=_billed_usage(batch_data))
def transform_response(
self,
response_list: list[dict[str, object]], # mutable-ok: mirrors override contract
model: str,
batch_data: list[dict[str, object]] | None = None, # mutable-ok: mirrors override contract
) -> EmbeddingResponse:
return self._transform_response(response_list, model, batch_data)
def _transform_async_invoke_response(self, response: dict, model: str) -> EmbeddingResponse:
"""
Transform async invoke response (invocation ARN) to OpenAI format.
@ -366,3 +387,10 @@ class TwelveLabsMarengoEmbeddingConfig:
usage=usage,
hidden_params=hidden_params,
)
def transform_async_invoke_response(
self,
response: dict[str, object], # mutable-ok: mirrors override contract
model: str,
) -> EmbeddingResponse:
return self._transform_async_invoke_response(response, model)

11
litellm/llms/bedrock/files/transformation.py
View file

@ -817,7 +817,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
non_default_params=non_default_params,
optional_params={},
)
return dict(titan_config._transform_request(input=input_text, inference_params=inference_params))
return dict(titan_config.transform_request(input=input_text, inference_params=inference_params))
@staticmethod
def _transform_text_completion_body_to_chat_body(
@ -1024,6 +1024,13 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
bedrock_jsonl_content.append(bedrock_record)
return bedrock_jsonl_content
def transform_openai_jsonl_content_to_bedrock_jsonl_content(
self,
openai_jsonl_content: Sequence[_OpenAIBatchRecord],
target_model: str = "",
) -> list[_BedrockBatchRecord]: # mutable-ok: mirrors override contract
return self._transform_openai_jsonl_content_to_bedrock_jsonl_content(openai_jsonl_content, target_model)
def transform_create_file_request(
self,
model: str,
@ -1535,7 +1542,7 @@ class BedrockJsonlFilesTransformation:
Delegate to the main BedrockFilesConfig transformation method
"""
config: Final = BedrockFilesConfig()
return config._transform_openai_jsonl_content_to_bedrock_jsonl_content(openai_jsonl_content)
return config.transform_openai_jsonl_content_to_bedrock_jsonl_content(openai_jsonl_content)
def _get_s3_object_name(
self,

11
litellm/llms/bedrock/image_edit/amazon_nova_canvas_image_edit_transformation.py
View file

@ -229,6 +229,13 @@ class BedrockAmazonNovaCanvasImageEditConfig(BaseImageEditConfig):
"""
return _supports_nova_canvas_image_edit_from_model_cost(model or "")
@classmethod
def is_nova_canvas_image_edit_model(
cls,
model: str | None = None,
) -> bool:
return cls._is_nova_canvas_image_edit_model(model)
def get_error_class(
self,
error_message: str,
@ -509,9 +516,9 @@ def get_bedrock_image_edit_config_for_model(
BedrockStabilityImageEditConfig,
)
if BedrockStabilityImageEditConfig._is_stability_edit_model(model):
if BedrockStabilityImageEditConfig.is_stability_edit_model(model):
return BedrockStabilityImageEditConfig()
if BedrockAmazonNovaCanvasImageEditConfig._is_nova_canvas_image_edit_model(model):
if BedrockAmazonNovaCanvasImageEditConfig.is_nova_canvas_image_edit_model(model):
return BedrockAmazonNovaCanvasImageEditConfig()
raise ValueError(
f"Unsupported Bedrock image-edit model: {model!r}. "

8
litellm/llms/bedrock/image_edit/handler.py
View file

@ -23,8 +23,8 @@ from litellm.llms.bedrock.image_edit.stability_transformation import (
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
_get_httpx_client,
get_async_httpx_client,
get_httpx_client,
)
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.utils import ImageResponse
@ -56,9 +56,9 @@ class BedrockImageEdit(BaseAWSLLM):
@classmethod
def get_config_class(cls, model: str | None):
if BedrockStabilityImageEditConfig._is_stability_edit_model(model):
if BedrockStabilityImageEditConfig.is_stability_edit_model(model):
return BedrockStabilityImageEditConfig
if BedrockAmazonNovaCanvasImageEditConfig._is_nova_canvas_image_edit_model(model):
if BedrockAmazonNovaCanvasImageEditConfig.is_nova_canvas_image_edit_model(model):
return BedrockAmazonNovaCanvasImageEditConfig
raise ValueError(
f"Unsupported Bedrock image-edit model: {model!r}. "
@ -104,7 +104,7 @@ class BedrockImageEdit(BaseAWSLLM):
)
if client is None or not isinstance(client, HTTPHandler):
client = _get_httpx_client()
client = get_httpx_client()
try:
response: Final = client.post(
url=prepared_request.endpoint_url,

7
litellm/llms/bedrock/image_edit/stability_transformation.py
View file

@ -86,6 +86,13 @@ class BedrockStabilityImageEditConfig(BaseImageEditConfig):
return True
return False
@classmethod
def is_stability_edit_model(
cls,
model: str | None = None,
) -> bool:
return cls._is_stability_edit_model(model)
def get_error_class(
self,
error_message: str,

7
litellm/llms/bedrock/image_generation/amazon_nova_canvas_transformation.py
View file

@ -59,6 +59,13 @@ class AmazonNovaCanvasConfig:
return True
return False
@classmethod
def is_nova_model(
cls,
model: str | None = None,
) -> bool:
return cls._is_nova_model(model)
@classmethod
def transform_request_body(cls, text: str, optional_params: dict) -> AmazonNovaCanvasRequestBase:
"""

7
litellm/llms/bedrock/image_generation/amazon_stability3_transformation.py
View file

@ -65,6 +65,13 @@ class AmazonStability3Config:
return True
return False
@classmethod
def is_stability_3_model(
cls,
model: str | None = None,
) -> bool:
return cls._is_stability_3_model(model)
@classmethod
def transform_request_body(cls, text: str, optional_params: dict) -> AmazonStability3TextToImageRequest:
"""

7
litellm/llms/bedrock/image_generation/amazon_titan_transformation.py
View file

@ -70,6 +70,13 @@ class AmazonTitanImageGenerationConfig:
return True
return False
@classmethod
def is_titan_model(
cls,
model: str | None = None,
) -> bool:
return cls._is_titan_model(model)
@classmethod
def get_supported_openai_params(cls, model: str | None = None) -> list:
return ["size", "n", "quality"]

10
litellm/llms/bedrock/image_generation/image_handler.py
View file

@ -23,8 +23,8 @@ from litellm.llms.bedrock.image_generation.amazon_titan_transformation import (
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
_get_httpx_client,
get_async_httpx_client,
get_httpx_client,
)
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.utils import ImageResponse
@ -64,11 +64,11 @@ class BedrockImageGeneration(BaseAWSLLM):
@classmethod
def get_config_class(cls, model: str | None) -> BedrockImageConfigClass:
if AmazonTitanImageGenerationConfig._is_titan_model(model):
if AmazonTitanImageGenerationConfig.is_titan_model(model):
return AmazonTitanImageGenerationConfig
elif AmazonNovaCanvasConfig._is_nova_model(model):
elif AmazonNovaCanvasConfig.is_nova_model(model):
return AmazonNovaCanvasConfig
elif AmazonStability3Config._is_stability_3_model(model):
elif AmazonStability3Config.is_stability_3_model(model):
return AmazonStability3Config
else:
return litellm.AmazonStabilityConfig
@ -109,7 +109,7 @@ class BedrockImageGeneration(BaseAWSLLM):
)
if client is None or not isinstance(client, HTTPHandler):
client = _get_httpx_client()
client = get_httpx_client()
try:
response: Final = client.post(
url=prepared_request.endpoint_url,

8
litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -226,7 +226,7 @@ class AmazonAnthropicClaudeMessagesConfig(
Returns:
True if the model supports extended thinking on Bedrock
"""
if AnthropicModelInfo._is_adaptive_thinking_model(model, "bedrock"):
if AnthropicModelInfo.is_adaptive_thinking_model(model, "bedrock"):
return True
model_lower: Final = model.lower()
@ -276,7 +276,7 @@ class AmazonAnthropicClaudeMessagesConfig(
if not self._supports_extended_thinking_on_bedrock(model):
return False
is_adaptive_thinking_model: Final = AnthropicModelInfo._is_adaptive_thinking_model(model, "bedrock")
is_adaptive_thinking_model: Final = AnthropicModelInfo.is_adaptive_thinking_model(model, "bedrock")
thinking: Final = anthropic_messages_request.get("thinking")
if isinstance(thinking, dict):
@ -664,7 +664,7 @@ class AmazonAnthropicClaudeMessagesConfig(
path degrades ``xhigh`` -> ``max`` rather than 400-ing. Non-adaptive models
and models without a ceiling are left untouched.
"""
if not AnthropicModelInfo._is_adaptive_thinking_model(model, "bedrock"):
if not AnthropicModelInfo.is_adaptive_thinking_model(model, "bedrock"):
return
effort: Final = optional_params.get("reasoning_effort")
if not isinstance(effort, str):
@ -778,7 +778,7 @@ class AmazonAnthropicClaudeMessagesConfig(
litellm.drop_params is True
and isinstance(remaining_output_config, dict)
and any(key != "format" for key in remaining_output_config)
and not AnthropicConfig._model_supports_effort_param(model, "bedrock")
and not AnthropicConfig.model_supports_effort_param(model, "bedrock")
):
verbose_logger.warning(
DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING,

2
litellm/llms/bedrock/passthrough/transformation.py
View file

@ -74,7 +74,7 @@ def _translate_message(decoder: "AWSEventStreamDecoder", message: str) -> ModelR
)
from litellm.types.utils import GenericStreamingChunk
translated_chunk: Final = decoder._chunk_parser(chunk_data=json.loads(message))
translated_chunk: Final = decoder.chunk_parser(chunk_data=json.loads(message))
if isinstance(translated_chunk, ModelResponseStream):
return translated_chunk
if generic_chunk_has_all_required_fields(cast(dict, translated_chunk)):

10
litellm/llms/bedrock/rerank/handler.py
View file

@ -9,8 +9,8 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
_get_httpx_client,
get_async_httpx_client,
get_httpx_client,
)
from litellm.types.llms.bedrock import BedrockPreparedRequest
from litellm.types.rerank import RerankRequest
@ -58,7 +58,7 @@ class BedrockRerankHandler(BaseAWSLLM):
except httpx.TimeoutException:
raise BedrockError(status_code=408, message="Timeout error occurred.")
return BedrockRerankConfig()._transform_response(_JSON_DICT.validate_python(response.json()))
return BedrockRerankConfig().transform_response(_JSON_DICT.validate_python(response.json()))
def rerank(
self,
@ -85,7 +85,7 @@ class BedrockRerankHandler(BaseAWSLLM):
rank_fields=rank_fields,
return_documents=return_documents,
)
data: Final = BedrockRerankConfig()._transform_request(request_data)
data: Final = BedrockRerankConfig().transform_request(request_data)
prepared_request: Final = self._prepare_request(
model=model,
@ -114,7 +114,7 @@ class BedrockRerankHandler(BaseAWSLLM):
)
if client is None or not isinstance(client, HTTPHandler):
client = _get_httpx_client()
client = get_httpx_client()
try:
response: Final = client.post(
url=prepared_request["endpoint_url"],
@ -141,7 +141,7 @@ class BedrockRerankHandler(BaseAWSLLM):
response_json: Final = _JSON_DICT.validate_python(response.json())
return BedrockRerankConfig()._transform_response(response_json)
return BedrockRerankConfig().transform_response(response_json)
def _prepare_request(
self,

12
litellm/llms/bedrock/rerank/transformation.py
View file

@ -77,6 +77,12 @@ class BedrockRerankConfig:
sources=_sources,
)
def transform_request(
self,
request_data: RerankRequest,
) -> BedrockRerankRequest:
return self._transform_request(request_data)
def _transform_response(self, response: dict) -> RerankResponse:
"""
Transform the response from Bedrock into the RerankResponse format.
@ -108,3 +114,9 @@ class BedrockRerankConfig:
results=_results,
meta=rerank_meta,
) # Return response
def transform_response(
self,
response: dict[str, object], # mutable-ok: mirrors override contract
) -> RerankResponse:
return self._transform_response(response)

11
litellm/llms/bedrock_mantle/chat/transformation.py
View file

@ -71,7 +71,7 @@ class BedrockMantleChatConfig(BedrockMantleAuthMixin, OpenAILikeChatConfig):
or get_secret_str("AWS_REGION")
or BEDROCK_MANTLE_DEFAULT_REGION
)
BaseAWSLLM._validate_aws_region_name(region)
BaseAWSLLM.validate_aws_region_name(region)
# The base path segment is data-driven per model (use_openai_responses_path
# flag): gemma-4-* and gpt-5.x are served on /openai/v1, everything else on
# /v1. An explicit api_base still wins over the derived default.
@ -83,6 +83,15 @@ class BedrockMantleChatConfig(BedrockMantleAuthMixin, OpenAILikeChatConfig):
dynamic_api_key: Final = self._resolve_bearer_token(api_key)
return api_base, dynamic_api_key
def get_openai_compatible_provider_info(
self,
api_base: str | None,
api_key: str | None,
litellm_params: GenericLiteLLMParams | None = None,
model: str | None = None,
) -> tuple[str | None, str | None]:
return self._get_openai_compatible_provider_info(api_base, api_key, litellm_params, model)
def validate_environment(
self,
headers: dict,

2
litellm/llms/bedrock_mantle/common_utils.py
View file

@ -48,7 +48,7 @@ def split_mantle_region_prefix(model: str) -> tuple[str | None, str]:
def resolve_mantle_region(params: Mapping[str, object]) -> str:
region: Final = params.get("aws_region_name")
if isinstance(region, str) and region:
BaseAWSLLM._validate_aws_region_name(region)
BaseAWSLLM.validate_aws_region_name(region)
return region
api_base: Final = params.get("api_base")
base: Final = (api_base if isinstance(api_base, str) else None) or get_secret_str("BEDROCK_MANTLE_API_BASE")

4
litellm/llms/black_forest_labs/image_edit/handler.py
View file

@ -20,8 +20,8 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
_get_httpx_client,
get_async_httpx_client,
get_httpx_client,
)
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import FileTypes, ImageResponse
@ -138,7 +138,7 @@ class BlackForestLabsImageEdit:
# Sync version
if client is None or not isinstance(client, HTTPHandler):
sync_client = _get_httpx_client()
sync_client = get_httpx_client()
else:
sync_client = client

4
litellm/llms/black_forest_labs/image_generation/handler.py
View file

@ -20,8 +20,8 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
_get_httpx_client,
get_async_httpx_client,
get_httpx_client,
)
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import ImageResponse
@ -119,7 +119,7 @@ class BlackForestLabsImageGeneration:
# Sync version
if client is None or not isinstance(client, HTTPHandler):
sync_client = _get_httpx_client()
sync_client = get_httpx_client()
else:
sync_client = client

4
litellm/llms/bytez/chat/transformation.py
View file

@ -13,8 +13,8 @@ from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMExcepti
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
_get_httpx_client,
get_async_httpx_client,
get_httpx_client,
version,
)
from litellm.types.llms.openai import AllMessageValues
@ -264,7 +264,7 @@ class BytezChatConfig(BaseConfig):
timeout: float | httpx.Timeout | None = None,
) -> "BytezCustomStreamWrapper":
if client is None or isinstance(client, AsyncHTTPHandler):
client = _get_httpx_client(params={})
client = get_httpx_client(params={})
try:
response: Final = client.post(

Some files were not shown because too many files have changed in this diff Show more