From 557413fa96abca4f6f310b1dfeed23c74e81781c Mon Sep 17 00:00:00 2001 From: narmstrong-berri Date: Fri, 16 Oct 2026 20:01:06 +0000 Subject: [PATCH] refactor(native): move the chat provider helpers out of main.py --- litellm/images/main.py | 2 +- litellm/llms/azure/azure.py | 4 +- litellm/main.py | 4242 +---------------- litellm/native/chat_providers/__init__.py | 0 litellm/native/chat_providers/agents.py | 273 ++ litellm/native/chat_providers/anthropic.py | 131 + litellm/native/chat_providers/aws.py | 393 ++ litellm/native/chat_providers/azure.py | 406 ++ litellm/native/chat_providers/custom.py | 147 + litellm/native/chat_providers/gateways.py | 299 ++ litellm/native/chat_providers/google.py | 311 ++ litellm/native/chat_providers/handlers.py | 51 + litellm/native/chat_providers/model_labs.py | 905 ++++ litellm/native/chat_providers/openai.py | 388 ++ litellm/native/chat_providers/platforms.py | 623 +++ litellm/native/chat_providers/self_hosted.py | 325 ++ litellm/native/chat_providers/support.py | 47 + .../proxy/common_utils/load_config_utils.py | 2 +- litellm/responses/bridge_check.py | 6 +- litellm/types/completion.py | 2 +- ...test_azure_chat_o_series_transformation.py | 2 +- .../llms/azure/test_azure_common_utils.py | 2 +- .../test_main_azure_anthropic_timeout.py | 2 +- .../gdc/chat/test_gdc_chat_transformation.py | 2 +- .../llms/gemini/test_gemini_client_setup.py | 4 +- .../llms/github_copilot/test_per_user_auth.py | 2 +- tests/unit/llms/vertex_ai/test_vertex.py | 4 +- ..._vertex_ai_anthropic_image_url_handling.py | 3 +- ...partner_models_anthropic_transformation.py | 2 +- .../test_vertex_gemma_transformation.py | 2 +- .../common_utils/test_load_config_utils.py | 2 +- tests/unit/test_main.py | 2 +- 32 files changed, 4465 insertions(+), 4121 deletions(-) create mode 100644 litellm/native/chat_providers/__init__.py create mode 100644 litellm/native/chat_providers/agents.py create mode 100644 litellm/native/chat_providers/anthropic.py create mode 100644 litellm/native/chat_providers/aws.py create mode 100644 litellm/native/chat_providers/azure.py create mode 100644 litellm/native/chat_providers/custom.py create mode 100644 litellm/native/chat_providers/gateways.py create mode 100644 litellm/native/chat_providers/google.py create mode 100644 litellm/native/chat_providers/handlers.py create mode 100644 litellm/native/chat_providers/model_labs.py create mode 100644 litellm/native/chat_providers/openai.py create mode 100644 litellm/native/chat_providers/platforms.py create mode 100644 litellm/native/chat_providers/self_hosted.py create mode 100644 litellm/native/chat_providers/support.py diff --git a/litellm/images/main.py b/litellm/images/main.py index 7f94835d59b..060eccb1f69 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -42,13 +42,13 @@ from litellm.llms.black_forest_labs.image_generation.handler import bfl_image_ge from litellm.llms.openai.workload_identity import resolve_openai_workload_identity_config from litellm.main import ( azure_chat_completions, - base_llm_aiohttp_handler, base_llm_http_handler, bedrock_image_edit, bedrock_image_generation, openai_chat_completions, openai_image_variations, ) +from litellm.native.chat_providers.handlers import base_llm_aiohttp_handler ########################################### from litellm.secret_managers.main import get_secret_str diff --git a/litellm/llms/azure/azure.py b/litellm/llms/azure/azure.py index 67b25bf7d12..87d71a2bcbd 100644 --- a/litellm/llms/azure/azure.py +++ b/litellm/llms/azure/azure.py @@ -1,7 +1,7 @@ import asyncio import json import time -from collections.abc import Callable, Coroutine +from collections.abc import Callable, Coroutine, Mapping from typing import Final import httpx @@ -115,7 +115,7 @@ class AzureOpenAIAssistantsAPIConfig: def check_dynamic_azure_params( - azure_client_params: dict, + azure_client_params: Mapping[str, object], azure_client: AzureOpenAI | AsyncAzureOpenAI | None, ) -> bool: """ diff --git a/litellm/main.py b/litellm/main.py index ac0a2962259..92931433dbe 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -14,7 +14,6 @@ import contextvars import datetime import inspect import json -import os import random import sys import time @@ -61,13 +60,17 @@ if TYPE_CHECKING: from litellm.router import Router from litellm.types.utils import TokenCountResponse +import os as os # noqa: PLC0414 # litellm.* re-export + from litellm.constants import ( AZURE_OPENAI_AUDIO_PROVIDERS, DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT, - NADIR_DEFAULT_API_BASE, OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS, ) +from litellm.constants import ( + NADIR_DEFAULT_API_BASE as NADIR_DEFAULT_API_BASE, # noqa: PLC0414 # litellm.* re-export +) from litellm.exceptions import LiteLLMUnknownProvider from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.asyncify import asyncify, run_async_function @@ -118,24 +121,114 @@ from litellm.llms.base_llm import BaseConfig, BaseImageGenerationConfig from litellm.llms.base_llm.base_model_iterator import ( convert_model_response_to_streaming, ) -from litellm.llms.base_llm.chat.transformation import with_attribution_headers -from litellm.llms.bedrock.common_utils import ( - BedrockModelInfo, - NativeOutputFallback, - bedrock_route_for_request, - deployment_model_info, - without_bedrock_native_output_opt_in, - without_bedrock_route_prefix, +from litellm.llms.base_llm.chat.transformation import ( + with_attribution_headers as with_attribution_headers, # noqa: PLC0414 # litellm.* re-export +) +from litellm.llms.bedrock.common_utils import ( + BedrockModelInfo as BedrockModelInfo, # noqa: PLC0414 # litellm.* re-export +) +from litellm.llms.bedrock.common_utils import ( + bedrock_route_for_request as bedrock_route_for_request, # noqa: PLC0414 # litellm.* re-export +) +from litellm.llms.bedrock.common_utils import ( + without_bedrock_route_prefix as without_bedrock_route_prefix, # noqa: PLC0414 # litellm.* re-export +) +from litellm.llms.bedrock_mantle.chat.claude_transformation import ( + bedrock_mantle_chat_config as bedrock_mantle_chat_config, # noqa: PLC0414 # litellm.* re-export +) +from litellm.llms.cohere.common_utils import ( + CohereModelInfo as CohereModelInfo, # noqa: PLC0414 # litellm.* re-export +) +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, +) +from litellm.llms.custom_httpx.http_handler import ( + http2_enabled as http2_enabled, # noqa: PLC0414 # litellm.* re-export ) -from litellm.llms.bedrock_mantle.chat.claude_transformation import bedrock_mantle_chat_config -from litellm.llms.cohere.common_utils import CohereModelInfo -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, http2_enabled from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.llms.vertex_ai.common_utils import ( - VertexAIModelRoute, - get_vertex_ai_model_route, + VertexAIModelRoute as VertexAIModelRoute, # noqa: PLC0414 # litellm.* re-export ) +from litellm.llms.vertex_ai.common_utils import ( + get_vertex_ai_model_route as get_vertex_ai_model_route, # noqa: PLC0414 # litellm.* re-export +) +from litellm.native.chat_providers.agents import complete_a2a as _complete_a2a +from litellm.native.chat_providers.agents import complete_langflow as _complete_langflow +from litellm.native.chat_providers.agents import complete_langgraph as _complete_langgraph +from litellm.native.chat_providers.agents import complete_microsoft_365_copilot as _complete_microsoft_365_copilot +from litellm.native.chat_providers.agents import complete_ragflow as _complete_ragflow +from litellm.native.chat_providers.anthropic import complete_anthropic as _complete_anthropic +from litellm.native.chat_providers.anthropic import complete_anthropic_text as _complete_anthropic_text +from litellm.native.chat_providers.aws import complete_amazon_nova as _complete_amazon_nova +from litellm.native.chat_providers.aws import complete_bedrock as _complete_bedrock +from litellm.native.chat_providers.aws import complete_bedrock_mantle as _complete_bedrock_mantle +from litellm.native.chat_providers.aws import complete_sagemaker as _complete_sagemaker +from litellm.native.chat_providers.aws import complete_sagemaker_chat as _complete_sagemaker_chat +from litellm.native.chat_providers.azure import complete_azure as _complete_azure +from litellm.native.chat_providers.azure import complete_azure_ai as _complete_azure_ai +from litellm.native.chat_providers.azure import complete_azure_text as _complete_azure_text +from litellm.native.chat_providers.custom import complete_custom as _complete_custom +from litellm.native.chat_providers.custom import complete_custom_providers as _complete_custom_providers +from litellm.native.chat_providers.gateways import complete_cometapi as _complete_cometapi +from litellm.native.chat_providers.gateways import complete_edenai as _complete_edenai +from litellm.native.chat_providers.gateways import complete_fal_ai as _complete_fal_ai +from litellm.native.chat_providers.gateways import complete_nadir as _complete_nadir +from litellm.native.chat_providers.gateways import complete_openrouter as _complete_openrouter +from litellm.native.chat_providers.gateways import complete_vercel_ai_gateway as _complete_vercel_ai_gateway +from litellm.native.chat_providers.google import complete_gdc as _complete_gdc +from litellm.native.chat_providers.google import complete_vertex_ai as _complete_vertex_ai +from litellm.native.chat_providers.google import complete_vertex_ai_beta as _complete_vertex_ai_beta +from litellm.native.chat_providers.handlers import * # noqa: F403 # litellm.* re-export of the moved handlers +from litellm.native.chat_providers.model_labs import complete_aleph_alpha as _complete_aleph_alpha +from litellm.native.chat_providers.model_labs import complete_bytez as _complete_bytez +from litellm.native.chat_providers.model_labs import complete_cohere_chat as _complete_cohere_chat +from litellm.native.chat_providers.model_labs import complete_compactifai as _complete_compactifai +from litellm.native.chat_providers.model_labs import complete_deepseek as _complete_deepseek +from litellm.native.chat_providers.model_labs import complete_fireworks_ai as _complete_fireworks_ai +from litellm.native.chat_providers.model_labs import complete_groq as _complete_groq +from litellm.native.chat_providers.model_labs import complete_huggingface as _complete_huggingface +from litellm.native.chat_providers.model_labs import complete_maritalk as _complete_maritalk +from litellm.native.chat_providers.model_labs import complete_minimax as _complete_minimax +from litellm.native.chat_providers.model_labs import complete_mistral as _complete_mistral +from litellm.native.chat_providers.model_labs import complete_nlp_cloud as _complete_nlp_cloud +from litellm.native.chat_providers.model_labs import complete_predibase as _complete_predibase +from litellm.native.chat_providers.model_labs import complete_replicate as _complete_replicate +from litellm.native.chat_providers.model_labs import complete_scaledown as _complete_scaledown +from litellm.native.chat_providers.model_labs import ( + complete_text_completion_codestral as _complete_text_completion_codestral, +) +from litellm.native.chat_providers.model_labs import complete_together_ai as _complete_together_ai +from litellm.native.chat_providers.model_labs import complete_xai as _complete_xai +from litellm.native.chat_providers.openai import complete_aiohttp_openai as _complete_aiohttp_openai +from litellm.native.chat_providers.openai import complete_custom_openai as _complete_custom_openai +from litellm.native.chat_providers.openai import ( + complete_text_completion_inception as _complete_text_completion_inception, +) +from litellm.native.chat_providers.openai import complete_text_completion_openai as _complete_text_completion_openai +from litellm.native.chat_providers.platforms import complete_cloudflare as _complete_cloudflare +from litellm.native.chat_providers.platforms import complete_databricks as _complete_databricks +from litellm.native.chat_providers.platforms import complete_databricks_agent as _complete_databricks_agent +from litellm.native.chat_providers.platforms import complete_datarobot as _complete_datarobot +from litellm.native.chat_providers.platforms import complete_gigachat as _complete_gigachat +from litellm.native.chat_providers.platforms import complete_gradient_ai as _complete_gradient_ai +from litellm.native.chat_providers.platforms import complete_heroku as _complete_heroku +from litellm.native.chat_providers.platforms import complete_oci as _complete_oci +from litellm.native.chat_providers.platforms import complete_ovhcloud as _complete_ovhcloud +from litellm.native.chat_providers.platforms import complete_sap as _complete_sap +from litellm.native.chat_providers.platforms import complete_snowflake as _complete_snowflake +from litellm.native.chat_providers.platforms import complete_watsonx as _complete_watsonx +from litellm.native.chat_providers.platforms import complete_watsonx_text as _complete_watsonx_text +from litellm.native.chat_providers.self_hosted import complete_hosted_vllm as _complete_hosted_vllm +from litellm.native.chat_providers.self_hosted import complete_lemonade as _complete_lemonade +from litellm.native.chat_providers.self_hosted import complete_ollama as _complete_ollama +from litellm.native.chat_providers.self_hosted import complete_ollama_chat as _complete_ollama_chat +from litellm.native.chat_providers.self_hosted import complete_oobabooga as _complete_oobabooga +from litellm.native.chat_providers.self_hosted import complete_petals as _complete_petals +from litellm.native.chat_providers.self_hosted import complete_triton as _complete_triton +from litellm.native.chat_providers.self_hosted import complete_vllm as _complete_vllm +from litellm.native.chat_providers.support import print_verbose from litellm.native.route_plan import NativeResponses, plan_chat_route from litellm.realtime_api.main import realtime_health_check from litellm.responses.mcp.tool_precheck import may_name_litellm_mcp_gateway @@ -144,10 +237,14 @@ from litellm.router_utils.reasoning_effort_capability import ( reasoning_effort_needs_allowing, resolve_chat_reasoning, ) -from litellm.secret_managers.main import get_secret_bool, get_secret_str +from litellm.secret_managers.main import ( + get_secret_bool as get_secret_bool, # noqa: PLC0414 # litellm.* re-export +) +from litellm.secret_managers.main import ( + get_secret_str, +) from litellm.types.completion import ( CompletionDispatchContext, - _CompletionDispatchResult, ) from litellm.types.litellm_params import ControlOptions, RetryStrategy from litellm.types.llms.openai import AllMessageValues @@ -186,7 +283,6 @@ from litellm.utils import ( get_optional_params_embeddings, get_optional_params_image_gen, get_optional_params_transcription, - get_requester_metadata, get_secret, get_standard_openai_params, mock_completion_streaming_obj, @@ -201,13 +297,22 @@ from litellm.utils import ( validate_chat_completion_tool_choice, validate_openai_optional_params, ) +from litellm.utils import ( + get_requester_metadata as get_requester_metadata, # noqa: PLC0414 # litellm.* re-export +) from litellm.utils import ( token_counter as token_counter, # noqa: PLC0414 # public re-export ) from ._logging import verbose_logger from .caching.caching import disable_cache, enable_cache, update_cache -from .litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params, safe_deep_copy +from .litellm_core_utils.core_helpers import ( + as_str_mapping, + normalize_drop_params, +) +from .litellm_core_utils.core_helpers import ( + safe_deep_copy as safe_deep_copy, # noqa: PLC0414 # litellm.* re-export +) from .litellm_core_utils.fallback_utils import ( async_completion_with_fallbacks, completion_with_fallbacks, @@ -223,42 +328,38 @@ from .litellm_core_utils.prompt_templates.factory import ( map_system_message_pt, ollama_pt, prompt_factory, - stringify_json_tool_call_content, +) +from .litellm_core_utils.prompt_templates.factory import ( + stringify_json_tool_call_content as stringify_json_tool_call_content, # noqa: PLC0414 # litellm.* re-export ) from .litellm_core_utils.streaming_chunk_builder_utils import ( ChunkProcessor as ChunkProcessor, # noqa: PLC0414 # public re-export ) -from .llms.anthropic.chat import AnthropicChatCompletion from .llms.azure.audio_transcriptions import AzureAudioTranscription -from .llms.azure.azure import AzureChatCompletion, check_dynamic_azure_params -from .llms.azure.chat.o_series_handler import AzureOpenAIO1ChatCompletion -from .llms.azure.completion.handler import AzureTextCompletion -from .llms.azure_ai.anthropic.handler import AzureAnthropicChatCompletion +from .llms.azure.azure import ( + check_dynamic_azure_params as check_dynamic_azure_params, # noqa: PLC0414 # litellm.* re-export +) from .llms.azure_ai.embed import AzureAIEmbedding -from .llms.bedrock.chat import BedrockConverseLLM -from .llms.bedrock.chat.converse_transformation import AmazonConverseConfig -from .llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import AmazonAnthropicClaudeConfig -from .llms.bedrock.chat.native_output_fallback import NativeOutputAttemptResult, call_with_native_output_fallback from .llms.bedrock.embed.embedding import BedrockEmbedding from .llms.bedrock.image_edit.handler import BedrockImageEdit from .llms.bedrock.image_generation.image_handler import BedrockImageGeneration -from .llms.bytez.chat.transformation import BytezChatConfig from .llms.clarifai.chat.transformation import ClarifaiConfig -from .llms.codestral.completion.handler import CodestralTextCompletion from .llms.cohere.embed import handler as cohere_embed -from .llms.custom_httpx.aiohttp_handler import BaseLLMAIOHTTPHandler -from .llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler -from .llms.custom_llm import CustomLLM, custom_chat_llm_router +from .llms.custom_llm import ( + CustomLLM, +) +from .llms.custom_llm import ( + custom_chat_llm_router as custom_chat_llm_router, # noqa: PLC0414 # litellm.* re-export +) from .llms.databricks.common_utils import DatabricksBase from .llms.databricks.embed.handler import DatabricksEmbeddingHandler -from .llms.deprecated_providers import aleph_alpha -from .llms.gdc.chat.transformation import GDCGeminiConfig +from .llms.deprecated_providers import ( + aleph_alpha as aleph_alpha, # noqa: PLC0414 # litellm.* re-export +) from .llms.gemini.common_utils import get_api_key_from_env from .llms.groq.chat.handler import GroqChatCompletion from .llms.heroku.chat.transformation import HerokuChatConfig from .llms.huggingface.embedding.handler import HuggingFaceEmbedding -from .llms.lemonade.chat.transformation import LemonadeChatConfig -from .llms.nlp_cloud.chat.handler import completion as nlp_cloud_chat_completion from .llms.nvidia_riva.audio_transcription.handler import ( NvidiaRivaAudioTranscription, ) @@ -268,22 +369,15 @@ from .llms.nvidia_riva.audio_transcription.transformation import ( from .llms.oci.chat.transformation import OCIChatConfig from .llms.ollama.completion import handler as ollama from .llms.oobabooga.chat import oobabooga -from .llms.openai.completion.handler import OpenAITextCompletion from .llms.openai.image_variations.handler import OpenAIImageVariationsHandler -from .llms.openai.openai import OpenAIChatCompletion from .llms.openai.transcriptions.handler import OpenAIAudioTranscription from .llms.openai.workload_identity import build_openai_client -from .llms.openai_like.chat.handler import OpenAILikeChatHandler from .llms.openai_like.embedding.handler import OpenAILikeEmbeddingHandler -from .llms.ovhcloud.chat.transformation import OVHCloudChatConfig -from .llms.petals.completion import handler as petals_handler -from .llms.predibase.chat.handler import PredibaseChatCompletion -from .llms.replicate.chat.handler import completion as replicate_chat_completion from .llms.sagemaker.chat.handler import SagemakerChatHandler -from .llms.sagemaker.completion.handler import SagemakerLLM from .llms.sap.chat.handler import GenAIHubOrchestration -from .llms.vertex_ai import vertex_ai_non_gemini -from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM +from .llms.vertex_ai import ( + vertex_ai_non_gemini as vertex_ai_non_gemini, # noqa: PLC0414 # litellm.* re-export +) from .llms.vertex_ai.gemini_embeddings.batch_embed_content_handler import ( GoogleBatchEmbeddings, ) @@ -293,12 +387,7 @@ from .llms.vertex_ai.image_generation.image_generation_handler import ( from .llms.vertex_ai.multimodal_embeddings.embedding_handler import ( VertexMultimodalEmbedding, ) -from .llms.vertex_ai.vertex_ai_partner_models.main import VertexAIPartnerModels from .llms.vertex_ai.vertex_embeddings.embedding_handler import VertexEmbedding -from .llms.vertex_ai.vertex_gemma_models.main import VertexAIGemmaModels -from .llms.vertex_ai.vertex_model_garden.main import VertexAIModelGardenModels -from .llms.vllm.completion import handler as vllm_handler -from .llms.watsonx.chat.handler import WatsonXChatHandler from .llms.watsonx.common_utils import IBMWatsonXMixin from .types.llms.anthropic import AnthropicThinkingParam from .types.llms.openai import ( @@ -323,51 +412,27 @@ from .types.utils import ( ) ####### ENVIRONMENT VARIABLES ################### -openai_chat_completions = OpenAIChatCompletion() -openai_text_completions: Final = OpenAITextCompletion() openai_audio_transcriptions: Final = OpenAIAudioTranscription() nvidia_riva_audio_transcriptions: Final = NvidiaRivaAudioTranscription() openai_image_variations = OpenAIImageVariationsHandler() groq_chat_completions: Final = GroqChatCompletion() -sap_gen_ai_hub_chat_completions: Final = GenAIHubOrchestration() sap_gen_ai_hub_emb: Final = GenAIHubOrchestration() azure_ai_embedding: Final = AzureAIEmbedding() -anthropic_chat_completions: Final = AnthropicChatCompletion() -azure_anthropic_chat_completions: Final = AzureAnthropicChatCompletion() -azure_chat_completions = AzureChatCompletion() -azure_o1_chat_completions: Final = AzureOpenAIO1ChatCompletion() -azure_text_completions: Final = AzureTextCompletion() azure_audio_transcriptions: Final = AzureAudioTranscription() huggingface_embed: Final = HuggingFaceEmbedding() -predibase_chat_completions: Final = PredibaseChatCompletion() -codestral_text_completions: Final = CodestralTextCompletion() -bedrock_converse_chat_completion: Final = BedrockConverseLLM() bedrock_embedding: Final = BedrockEmbedding() bedrock_image_generation = BedrockImageGeneration() bedrock_image_edit = BedrockImageEdit() -vertex_chat_completion: Final = VertexLLM() vertex_embedding: Final = VertexEmbedding() vertex_multimodal_embedding: Final = VertexMultimodalEmbedding() vertex_image_generation: Final = VertexImageGeneration() google_batch_embeddings: Final = GoogleBatchEmbeddings() -vertex_partner_models_chat_completion: Final = VertexAIPartnerModels() -vertex_gemma_chat_completion: Final = VertexAIGemmaModels() -vertex_model_garden_chat_completion: Final = VertexAIModelGardenModels() -gdc_transformation: Final = GDCGeminiConfig() # vertex_text_to_speech is now replaced by VertexAITextToSpeechConfig -sagemaker_llm: Final = SagemakerLLM() -watsonx_chat_completion: Final = WatsonXChatHandler() openai_like_embedding: Final = OpenAILikeEmbeddingHandler() -openai_like_chat_completion: Final = OpenAILikeChatHandler() databricks_embedding: Final = DatabricksEmbeddingHandler() -base_llm_http_handler = BaseLLMHTTPHandler() -base_llm_aiohttp_handler = BaseLLMAIOHTTPHandler() sagemaker_chat_completion: Final = SagemakerChatHandler() -bytez_transformation: Final = BytezChatConfig() heroku_transformation: Final = HerokuChatConfig() oci_transformation: Final = OCIChatConfig() -ovhcloud_transformation: Final = OVHCloudChatConfig() -lemonade_transformation: Final = LemonadeChatConfig() MOCK_RESPONSE_TYPE = str | Exception | dict | ModelResponse | ModelResponseStream @@ -573,7 +638,6 @@ async def acompletion( from litellm.integrations.anthropic_cache_control_hook import ( AnthropicCacheControlHook, ) - from litellm.types.llms.openai import AllMessageValues AnthropicCacheControlHook.maybe_seed_default_injection_points( non_default_params=kwargs, @@ -1105,10 +1169,7 @@ def mock_completion( raise Exception(f"Mock completion response failed - {e}") -from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the old definition site - _resolve_openai_api_base as _resolve_openai_api_base, # pyright: ignore[reportPrivateUsage] # noqa: PLC0414 # re-export of moved helper -) -from litellm.responses.bridge_check import ( +from litellm.responses.bridge_check import ( # noqa: E402 # used by completion() and register_model bridge_reasoning_effort, keep_registered_mode, str_keyed_mapping, @@ -1219,10 +1280,6 @@ def _proxy_auth_headers(proxy_auth: _ProxyAuthHeadersProvider) -> Mapping[str, s return proxy_auth.get_auth_headers() -def _provider_config_items(config: Mapping[str, object]) -> Iterable[tuple[str, object]]: - return config.items() - - def _locals_snapshot(values: Mapping[str, object]) -> Mapping[str, object]: return values @@ -1314,2894 +1371,6 @@ def _register_custom_pricing_for_request( ) -def _dispatch_metadata(ctx: CompletionDispatchContext) -> Mapping[str, object] | None: - return ctx.metadata - - -def _dispatch_client_http(ctx: CompletionDispatchContext) -> HTTPHandler | AsyncHTTPHandler | None: - return ctx.client - - -def _dispatch_client_azure( - ctx: CompletionDispatchContext, -) -> openai.AzureOpenAI | openai.AsyncAzureOpenAI | HTTPHandler | AsyncHTTPHandler | None: - return ctx.client - - -def _dispatch_client_openai(ctx: CompletionDispatchContext) -> openai.OpenAI | openai.AsyncOpenAI | None: - return ctx.client - - -def _complete_azure(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - _azure_detection_model: Final = ctx._azure_detection_model - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - api_version = ctx.api_version - client: Final = _dispatch_client_azure(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - extra_headers: Final = ctx.extra_headers - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - max_retries: Final = ctx.max_retries - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - timeout: Final = ctx.timeout - - dynamic_params = False - if client is not None and (isinstance(client, openai.AzureOpenAI) or isinstance(client, openai.AsyncAzureOpenAI)): - dynamic_params = check_dynamic_azure_params( - azure_client_params={"api_version": api_version}, - azure_client=client, - ) - - api_type: Final = get_secret("AZURE_API_TYPE") or "azure" - - api_base = api_base or litellm.api_base or get_secret("AZURE_API_BASE") - - api_version = ( - api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") or litellm.AZURE_DEFAULT_API_VERSION - ) - - api_key = ( - api_key - or litellm.api_key - or litellm.azure_key - or get_secret_str("AZURE_OPENAI_API_KEY") - or get_secret_str("AZURE_API_KEY") - ) - - azure_ad_token: Final = optional_params.get("extra_body", {}).pop("azure_ad_token", None) or get_secret_str( - "AZURE_AD_TOKEN" - ) - - azure_ad_token_provider_value: Final = litellm_params.get("azure_ad_token_provider", None) - azure_ad_token_provider: Final = azure_ad_token_provider_value if callable(azure_ad_token_provider_value) else None - - headers = headers or litellm.headers - - if extra_headers is not None: - optional_params["extra_headers"] = extra_headers - if max_retries is not None: - optional_params["max_retries"] = max_retries - - if litellm.AzureOpenAIO1Config().is_o_series_model(model=_azure_detection_model): - ## LOAD CONFIG - if set - config = litellm.AzureOpenAIO1Config.get_config() - for k, v in _provider_config_items(config): - if ( - k not in optional_params - ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in - optional_params[k] = v - - response = azure_o1_chat_completions.completion( - model=model, - messages=messages, - headers=headers, - api_key=api_key, - api_base=api_base, - api_version=api_version, - dynamic_params=dynamic_params, - azure_ad_token=azure_ad_token, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - logging_obj=logging, - acompletion=acompletion, - timeout=timeout, - client=client, # pass AsyncAzureOpenAI, AzureOpenAI client - custom_llm_provider=custom_llm_provider, - ) - else: - ## LOAD CONFIG - if set - config = litellm.AzureOpenAIConfig.get_config() - for k, v in _provider_config_items(config): - if ( - k not in optional_params - ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in - optional_params[k] = v - - ## COMPLETION CALL - response = azure_chat_completions.completion( - model=model, - messages=messages, - headers=headers, - api_key=api_key, - api_base=api_base, - api_version=api_version, - api_type=api_type, - dynamic_params=dynamic_params, - azure_ad_token=azure_ad_token, - azure_ad_token_provider=azure_ad_token_provider, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - logging_obj=logging, - acompletion=acompletion, - timeout=timeout, - client=client, # pass AsyncAzureOpenAI, AzureOpenAI client - ) - - if optional_params.get("stream", False): - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=response, - additional_args={ - "headers": headers, - "api_version": api_version, - "api_base": api_base, - }, - ) - - return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract - - -def _complete_azure_text(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - api_version = ctx.api_version - client: Final = _dispatch_client_azure(ctx) - extra_headers: Final = ctx.extra_headers - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - timeout: Final = ctx.timeout - - api_type: Final = get_secret_str("AZURE_API_TYPE") or "azure" - - api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") - - if api_base is None: - raise ValueError( - "api_base is required for Azure OpenAI LLM provider. Either set it dynamically or set the AZURE_API_BASE environment variable." - ) - - api_version = api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") - - api_key = ( - api_key - or litellm.api_key - or litellm.azure_key - or get_secret_str("AZURE_OPENAI_API_KEY") - or get_secret_str("AZURE_API_KEY") - ) - - azure_ad_token: Final = optional_params.get("extra_body", {}).pop("azure_ad_token", None) or get_secret_str( - "AZURE_AD_TOKEN" - ) - - azure_ad_token_provider_value: Final = litellm_params.get("azure_ad_token_provider", None) - azure_ad_token_provider: Final = azure_ad_token_provider_value if callable(azure_ad_token_provider_value) else None - - headers = headers or litellm.headers - - if extra_headers is not None: - optional_params["extra_headers"] = extra_headers - - ## LOAD CONFIG - if set - config: Final = litellm.AzureOpenAIConfig.get_config() - for k, v in _provider_config_items(config): - if ( - k not in optional_params - ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in - optional_params[k] = v - - ## COMPLETION CALL - response: Final = azure_text_completions.completion( - model=model, - messages=messages, - headers=headers, - api_key=api_key, - api_base=api_base, - api_version=cast(str, api_version), - api_type=api_type, - azure_ad_token=azure_ad_token, - azure_ad_token_provider=azure_ad_token_provider, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - logging_obj=logging, - acompletion=acompletion, - timeout=timeout, - client=client, # pass AsyncAzureOpenAI, AzureOpenAI client - ) - - if optional_params.get("stream", False) or acompletion is True: - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=response, - additional_args={ - "headers": headers, - "api_version": api_version, - "api_base": api_base, - }, - ) - - return response - - -def _complete_deepseek(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key: Final = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - provider_config: Final = ctx.provider_config - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - try: - response: Final = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) - except Exception as e: - ## LOGGING - log the original exception returned - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e - - return response - - -def _complete_azure_ai(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - extra_headers: Final = ctx.extra_headers - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo - - azure_ai_route: Final = AzureFoundryModelInfo.get_azure_ai_route(model) - - # Check if this is an agents route - model format: azure_ai/agents/ - if azure_ai_route == "agents": - from litellm.llms.azure_ai.agents import AzureAIAgentsConfig - - api_base = AzureFoundryModelInfo.get_api_base(api_base) - if api_base is None: - raise ValueError( - "Azure AI Agents requests require an api_base. Set `api_base` or the AZURE_AI_API_BASE env var." - ) - api_key = AzureFoundryModelInfo.get_api_key(api_key) - - response = AzureAIAgentsConfig.completion( - model=model, - messages=messages, - api_base=api_base, - api_key=api_key, - model_response=model_response, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, - acompletion=acompletion, - stream=stream, - headers=headers or litellm.headers, - ) - - # Check if this is a Claude model - route to Azure Anthropic handler - elif "claude" in model.lower(): - # Use Azure Anthropic handler for Claude models - api_base = AzureFoundryModelInfo.get_api_base(api_base) - if api_base is None: - raise ValueError( - "Azure Anthropic requests require an api_base. Set `api_base` or the AZURE_AI_API_BASE env var." - ) - api_key = AzureFoundryModelInfo.get_api_key(api_key) - - # Ensure the URL ends with /v1/messages for Anthropic - if api_base: - api_base = api_base.rstrip("/") - if not api_base.endswith("/v1/messages"): - if "/anthropic" in api_base: - parts: Final = api_base.split("/anthropic", 1) - api_base = parts[0] + "/anthropic" - else: - api_base = api_base + "/anthropic" - api_base = api_base + "/v1/messages" - - response = azure_anthropic_chat_completions.completion( - model=model, - messages=messages, - api_base=api_base, - acompletion=acompletion, - custom_prompt_dict=litellm.custom_prompt_dict, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - headers=headers, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - ) - if optional_params.get("stream", False) or acompletion is True: - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=response, - ) - response = response - else: - # Non-Claude models use standard Azure AI flow - api_base = AzureFoundryModelInfo.get_api_base(api_base) - # set API KEY - api_key = AzureFoundryModelInfo.get_api_key(api_key) - - headers = headers or litellm.headers - - if extra_headers is not None: - optional_params["extra_headers"] = extra_headers - - ## FOR COHERE - if "command-r" in model: # make sure tool call in messages are str - messages = stringify_json_tool_call_content(messages=messages) - - ## COMPLETION CALL - try: - response = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, - client=client, # pass AsyncOpenAI, OpenAI client - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - ) - except Exception as e: - ## LOGGING - log the original exception returned - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e - - if optional_params.get("stream", False): - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=response, - additional_args={"headers": headers}, - ) - - return response - - -def _complete_text_completion_openai( - ctx: CompletionDispatchContext, -) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_openai(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - text_completion: Final = ctx.text_completion - timeout: Final = ctx.timeout - - openai.api_type = "openai" - - api_base = ( - api_base - or litellm.api_base - or get_secret("OPENAI_BASE_URL") - or get_secret("OPENAI_API_BASE") - or "https://api.openai.com/v1" - ) - - openai.api_version = None - # set API KEY - - api_key = api_key or litellm.api_key or litellm.openai_key or get_secret("OPENAI_API_KEY") - - headers = headers or litellm.headers - - ## LOAD CONFIG - if set - config: Final = litellm.OpenAITextCompletionConfig.get_config() - for k, v in _provider_config_items(config): - if ( - k not in optional_params - ): # completion(top_k=3) > openai_text_config(top_k=3) <- allows for dynamic variables to be passed in - optional_params[k] = v - if litellm.organization: - openai.organization = litellm.organization - - ## COMPLETION CALL - _response = openai_text_completions.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - print_verbose=print_verbose, - api_key=api_key, - custom_llm_provider=custom_llm_provider, - api_base=api_base, - acompletion=acompletion, - client=client, # pass AsyncOpenAI, OpenAI client - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - timeout=timeout, - ) - - if optional_params.get("stream", False) is False and acompletion is False and text_completion is False: - # convert to chat completion response - _response = litellm.OpenAITextCompletionConfig().convert_to_chat_model_response_object( - response_object=_response, model_response_object=model_response - ) - - if optional_params.get("stream", False) or acompletion is True: - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=_response, - additional_args={"headers": headers}, - ) - return _response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract - - -def _complete_fireworks_ai( - ctx: CompletionDispatchContext, -) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key: Final = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - provider_config: Final = ctx.provider_config - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - optional_params: Final = ( - provider_config.map_extra_body_params(optional_params=ctx.optional_params, model=model) - if isinstance(provider_config, litellm.FireworksAIConfig) - else ctx.optional_params - ) - - try: - response: Final = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) - except Exception as e: - ## LOGGING - log the original exception returned - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e - - return response - - -def _complete_together_ai(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key: Final = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - provider_config: Final = ctx.provider_config - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - try: - response: Final = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) - except Exception as e: - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args=MappingProxyType({"headers": headers}), - ) - raise - - return response - - -def _complete_heroku(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key: Final = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - provider_config: Final = ctx.provider_config - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - try: - response: Final = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) - except Exception as e: - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e - - return response - - -def _complete_ragflow(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key: Final = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - provider_config: Final = ctx.provider_config - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - try: - response: Final = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) - except Exception as e: - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e - - return response - - -def _complete_scaledown(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - try: - return base_llm_http_handler.completion( - model=ctx.model, - messages=ctx.messages, - headers=ctx.headers, - model_response=ctx.model_response, - api_key=ctx.api_key, - api_base=ctx.api_base, - acompletion=ctx.acompletion, - logging_obj=ctx.logging, - optional_params=ctx.optional_params, - litellm_params=ctx.litellm_params, - shared_session=ctx.shared_session, - timeout=cast( # cast-ok: CompletionTimeout.resolve normalizes this value - float | httpx.Timeout, ctx.timeout - ), - client=_dispatch_client_http(ctx), - custom_llm_provider=ctx.custom_llm_provider, - encoding=_get_encoding(), - stream=ctx.stream, - provider_config=ctx.provider_config, - ) - except Exception as exc: - ctx.logging.post_call( - input=ctx.messages, - api_key=ctx.api_key, - original_response=str(exc), - additional_args={"headers": ctx.headers}, - ) - raise - - -def _complete_xai(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key: Final = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - provider_config: Final = ctx.provider_config - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - try: - response: Final = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) - except Exception as e: - ## LOGGING - log the original exception returned - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e - - return response - - -def _complete_groq(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_base = ( - api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there - or litellm.api_base - or get_secret("GROQ_API_BASE") - or "https://api.groq.com/openai/v1" - ) - - # set API KEY - api_key = ( - api_key - or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there - or litellm.groq_key - or get_secret("GROQ_API_KEY") - ) - - headers = headers or litellm.headers - - ## LOAD CONFIG - if set - config: Final = litellm.GroqChatConfig.get_config() - for k, v in _provider_config_items(config): - if ( - k not in optional_params - ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in - optional_params[k] = v - - return base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider=custom_llm_provider, - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, - ) - - -def _complete_bedrock_mantle( - ctx: CompletionDispatchContext, -) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_base = api_base or litellm.api_base or get_secret("BEDROCK_MANTLE_API_BASE") - api_key = api_key or litellm.api_key or get_secret("BEDROCK_MANTLE_API_KEY") - headers = headers or litellm.headers - config: Final = bedrock_mantle_chat_config(model).get_config() - for k, v in _provider_config_items(config): - if k not in optional_params: - optional_params[k] = v - return base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider=custom_llm_provider, - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - client=client, - ) - - -def _complete_a2a(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - provider_config: Final = ctx.provider_config - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - ( - api_base, - api_key, - headers, - ) = litellm.A2AConfig.resolve_agent_config_from_registry( - agent_name=model, - api_base=api_base, - api_key=api_key, - headers=headers, - optional_params=optional_params, - ) - - # Fall back to environment variables and defaults - api_base = api_base or litellm.api_base or get_secret_str("A2A_API_BASE") - - if api_base is None: - raise Exception( - "api_base is required for A2A provider. " - "Either provide api_base parameter, set A2A_API_BASE environment variable, " - "or register the agent in the proxy with model='a2a/'." - ) - - headers = headers or litellm.headers - - return base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider=custom_llm_provider, - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - client=client, - provider_config=provider_config, - ) - - -def _complete_gigachat(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - provider_config: Final = ctx.provider_config - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_key = ( - api_key - or litellm.api_key - or litellm.gigachat_key - or get_secret("GIGACHAT_API_KEY") - or get_secret("GIGACHAT_CREDENTIALS") - ) - - headers = headers or litellm.headers or {} - - ## COMPLETION CALL - try: - response: Final = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) - except Exception as e: - ## LOGGING - log the original exception returned - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e - - return response - - -def _complete_sap(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key: Final = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - headers = headers or litellm.headers - ## LOAD CONFIG - if set - config: Final = litellm.GenAIHubOrchestrationConfig.get_config() - for k, v in _provider_config_items(config): - if ( - k not in optional_params - ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in - optional_params[k] = v - - return sap_gen_ai_hub_chat_completions.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, - shared_session=shared_session, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - api_key=api_key, - api_base=api_base, - stream=stream, - ) - - -def _complete_aiohttp_openai( - ctx: CompletionDispatchContext, -) -> _CompletionDispatchResult: - if http2_enabled(): - verbose_logger.warning( - "litellm.http2 is enabled but aiohttp_openai/ always uses aiohttp, which has no HTTP/2 client; this request stays on HTTP/1.1" - ) - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - extra_headers: Final = ctx.extra_headers - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_base = ( - api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there - or litellm.api_base - or get_secret("OPENAI_BASE_URL") - or get_secret("OPENAI_API_BASE") - or "https://api.openai.com/v1" - ) - # set API KEY - api_key = ( - api_key - or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there - or litellm.openai_key - or get_secret("OPENAI_API_KEY") - ) - - headers = headers or litellm.headers - - if extra_headers is not None: - optional_params["extra_headers"] = extra_headers - return base_llm_aiohttp_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - ) - - -def _complete_cometapi(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - provider_config: Final = ctx.provider_config - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_key = api_key or litellm.cometapi_key or get_secret_str("COMETAPI_KEY") or litellm.api_key - - api_base = api_base or litellm.api_base or get_secret_str("COMETAPI_API_BASE") or "https://api.cometapi.com/v1" - - ## COMPLETION CALL - response: Final = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) - - ## LOGGING - logging.post_call(input=messages, api_key=api_key, original_response=response) - - return response - - -def _complete_minimax(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - provider_config: Final = ctx.provider_config - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_key = api_key or get_secret_str("MINIMAX_API_KEY") or litellm.api_key - - api_base = api_base or litellm.api_base or get_secret_str("MINIMAX_API_BASE") or "https://api.minimax.io/v1" - - response: Final = base_llm_http_handler.completion( - model=model, - messages=messages, - api_base=api_base, - custom_llm_provider=custom_llm_provider, - model_response=model_response, - encoding=_get_encoding(), - logging_obj=logging, - optional_params=optional_params, - timeout=timeout, - litellm_params=litellm_params, - shared_session=shared_session, - acompletion=acompletion, - stream=stream, - api_key=api_key, - headers=headers, - client=client, - provider_config=provider_config, - ) - logging.post_call(input=messages, api_key=api_key, original_response=response) - - return response - - -def _complete_hosted_vllm(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key: Final = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - provider_config: Final = ctx.provider_config - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_base = api_base or litellm.api_base or get_secret_str("HOSTED_VLLM_API_BASE") - - response: Final = base_llm_http_handler.completion( - model=model, - messages=messages, - api_base=api_base, - custom_llm_provider=custom_llm_provider, - model_response=model_response, - encoding=_get_encoding(), - logging_obj=logging, - optional_params=optional_params, - timeout=timeout, - litellm_params=litellm_params, - shared_session=shared_session, - acompletion=acompletion, - stream=stream, - api_key=api_key, - headers=headers, - client=client, - provider_config=provider_config, - ) - logging.post_call(input=messages, api_key=api_key, original_response=response) - - return response - - -def _complete_custom_openai( - ctx: CompletionDispatchContext, -) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - custom_prompt_dict: Final = ctx.custom_prompt_dict - extra_headers = ctx.extra_headers - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - metadata: Final = _dispatch_metadata(ctx) - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - organization = ctx.organization - provider_config: Final = ctx.provider_config - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there - api_base = _resolve_openai_api_base(api_base) - organization = ( - organization - or litellm.organization - or get_secret("OPENAI_ORGANIZATION") - or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 - ) - openai.organization = organization - # set API KEY - api_key = ( - api_key - or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there - or litellm.openai_key - or get_secret("OPENAI_API_KEY") - ) - - headers = headers or litellm.headers - outbound_headers: Final = ( - headers - if provider_config is None - else with_attribution_headers(provider_config.get_attribution_headers(), headers) - ) - - # Add GitHub Copilot headers (same as /responses endpoint does) - if custom_llm_provider == "github_copilot": - from litellm.llms.github_copilot.authenticator import Authenticator - from litellm.llms.github_copilot.common_utils import ( - get_copilot_default_headers, - pin_session_authorization, - ) - from litellm.llms.github_copilot.per_user_auth import ( - require_github_copilot_user_session, - ) - - user_session: Final = require_github_copilot_user_session(litellm_params) - copilot_headers: Final = get_copilot_default_headers( - user_session.token if user_session is not None else Authenticator().get_api_key() - ) - if extra_headers: - copilot_headers.update( - cast("dict[str, str]", extra_headers) # cast-ok: extra_headers is a str-valued request dict - ) - if user_session is not None: - pin_session_authorization(copilot_headers, user_session.token) - extra_headers = copilot_headers - - use_base_llm_http_handler: Final = get_secret_bool("EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER") - - if extra_headers is not None and not use_base_llm_http_handler: - optional_params["extra_headers"] = extra_headers - - if litellm.enable_preview_features and metadata is not None: # [PREVIEW] allow metadata to be passed to OPENAI - openai_metadata: Final = get_requester_metadata(metadata) - if openai_metadata is not None: - optional_params["metadata"] = openai_metadata - - ## LOAD CONFIG - if set - config: Final = litellm.OpenAIConfig.get_config() - for k, v in _provider_config_items(config): - if ( - k not in optional_params - ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in - optional_params[k] = v - - ## COMPLETION CALL - try: - if use_base_llm_http_handler: - response = base_llm_http_handler.completion( - model=model, - messages=messages, - api_base=api_base, - custom_llm_provider=custom_llm_provider, - model_response=model_response, - encoding=_get_encoding(), - logging_obj=logging, - optional_params=optional_params, - timeout=timeout, - litellm_params=litellm_params, - shared_session=shared_session, - acompletion=acompletion, - stream=stream, - api_key=api_key, - headers=outbound_headers, - client=client, - provider_config=provider_config, - ) - else: - response = openai_chat_completions.completion( - model=model, - messages=messages, - headers=outbound_headers, - model_response=model_response, - print_verbose=print_verbose, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - timeout=timeout, - custom_prompt_dict=custom_prompt_dict, - client=client, # pass AsyncOpenAI, OpenAI client - organization=organization, - custom_llm_provider=custom_llm_provider, - shared_session=shared_session, - ) - except Exception as e: - ## LOGGING - log the original exception returned - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": outbound_headers}, - ) - raise e - - if optional_params.get("stream", False): - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=response, - additional_args={"headers": outbound_headers}, - ) - - return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract - - -def _complete_mistral(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - provider_config: Final = ctx.provider_config - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_key = api_key or litellm.api_key or get_secret("MISTRAL_API_KEY") - api_base = api_base or litellm.api_base or get_secret("MISTRAL_API_BASE") or "https://api.mistral.ai/v1" - - return base_llm_http_handler.completion( - model=model, - messages=messages, - api_base=api_base, - custom_llm_provider=custom_llm_provider, - model_response=model_response, - encoding=_get_encoding(), - logging_obj=logging, - optional_params=optional_params, - timeout=timeout, - litellm_params=litellm_params, - shared_session=shared_session, - acompletion=acompletion, - stream=stream, - api_key=api_key, - headers=headers, - client=client, - provider_config=provider_config, - ) - - -def _complete_replicate(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key: Final = ctx.api_key - custom_prompt_dict = ctx.custom_prompt_dict - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response = ctx.model_response - optional_params: Final = ctx.optional_params - - replicate_key: Final = ( - api_key - or litellm.replicate_key - or litellm.api_key - or get_secret("REPLICATE_API_KEY") - or get_secret("REPLICATE_API_TOKEN") - ) - - api_base = api_base or litellm.api_base or get_secret("REPLICATE_API_BASE") or "https://api.replicate.com/v1" - - custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict - - model_response = replicate_chat_completion( - model=model, - messages=messages, - api_base=api_base, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), # for calculating input/output tokens - api_key=replicate_key, - logging_obj=logging, - custom_prompt_dict=custom_prompt_dict, - acompletion=acompletion, - headers=headers, - ) - - if optional_params.get("stream", False) is True: - ## LOGGING - logging.post_call( - input=messages, - api_key=replicate_key, - original_response=model_response, - ) - - return model_response - - -def _complete_anthropic_text( - ctx: CompletionDispatchContext, -) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - custom_prompt_dict = ctx.custom_prompt_dict - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_key = api_key or litellm.anthropic_key or litellm.api_key or os.environ.get("ANTHROPIC_API_KEY") - custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict - api_base = cast( - str | None, - api_base - or litellm.api_base - or get_secret("ANTHROPIC_API_BASE") - or get_secret("ANTHROPIC_BASE_URL") - or "https://api.anthropic.com/v1/complete", - ) - - # Check if we should disable automatic URL suffix appending - disable_url_suffix: Final = get_secret_bool("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX") - if api_base is not None and not disable_url_suffix and not api_base.endswith("/v1/complete"): - api_base += "/v1/complete" - elif disable_url_suffix: - verbose_logger.debug("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX is set, skipping /v1/complete suffix") - - return base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="anthropic_text", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - ) - - -def _complete_anthropic(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - custom_prompt_dict = ctx.custom_prompt_dict - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - timeout: Final = ctx.timeout - - api_key = api_key or litellm.anthropic_key or litellm.api_key or os.environ.get("ANTHROPIC_API_KEY") - custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict - # call /messages - # default route for all anthropic models - api_base = cast( - str | None, - api_base - or litellm.api_base - or get_secret("ANTHROPIC_API_BASE") - or get_secret("ANTHROPIC_BASE_URL") - or "https://api.anthropic.com/v1/messages", - ) - - # Check if we should disable automatic URL suffix appending - disable_url_suffix: Final = get_secret_bool("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX") - if api_base is not None and not disable_url_suffix and not api_base.endswith("/v1/messages"): - api_base += "/v1/messages" - elif disable_url_suffix: - verbose_logger.debug("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX is set, skipping /v1/messages suffix") - - response: Final = anthropic_chat_completions.completion( - model=model, - messages=messages, - api_base=api_base, - acompletion=acompletion, - custom_prompt_dict=litellm.custom_prompt_dict, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), # for calculating input/output tokens - api_key=api_key, - logging_obj=logging, - headers=headers, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - ) - if optional_params.get("stream", False) or acompletion is True: - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=response, - ) - return response - - -def _complete_nlp_cloud(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key: Final = ctx.api_key - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - - nlp_cloud_key: Final = api_key or litellm.nlp_cloud_key or get_secret("NLP_CLOUD_API_KEY") or litellm.api_key - - api_base = api_base or litellm.api_base or get_secret("NLP_CLOUD_API_BASE") or "https://api.nlpcloud.io/v1/gpu/" - - response = nlp_cloud_chat_completion( - model=model, - messages=messages, - api_base=api_base, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - api_key=nlp_cloud_key, - logging_obj=logging, - ) - - if "stream" in optional_params and optional_params["stream"] is True: - # don't try to access stream object, - response = CustomStreamWrapper( - response, - model, - custom_llm_provider="nlp_cloud", - logging_obj=logging, - ) - - if optional_params.get("stream", False) or acompletion is True: - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=response, - ) - - return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract - - -def _complete_aleph_alpha(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - api_base = ctx.api_base - api_key: Final = ctx.api_key - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response = ctx.model_response - optional_params: Final = ctx.optional_params - - aleph_alpha_key: Final = ( - api_key - or litellm.aleph_alpha_key - or get_secret("ALEPH_ALPHA_API_KEY") - or get_secret("ALEPHALPHA_API_KEY") - or litellm.api_key - ) - - api_base = ( - api_base or litellm.api_base or get_secret("ALEPH_ALPHA_API_BASE") or "https://api.aleph-alpha.com/complete" - ) - - model_response = aleph_alpha.completion( - model=model, - messages=messages, - api_base=api_base, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - default_max_tokens_to_sample=litellm.max_tokens, - api_key=aleph_alpha_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - ) - - if "stream" in optional_params and optional_params["stream"] is True: - # don't try to access stream object, - return CustomStreamWrapper( - model_response, - model, - custom_llm_provider="aleph_alpha", - logging_obj=logging, - ) - return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract - - -def _complete_cohere_chat(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key: Final = ctx.api_key - extra_headers: Final = ctx.extra_headers - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - provider_config: Final = ctx.provider_config - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - cohere_key: Final = ( - api_key - or litellm.cohere_key - or get_secret_str("COHERE_API_KEY") - or get_secret_str("CO_API_KEY") - or litellm.api_key - ) - - cohere_route: Final = CohereModelInfo.get_cohere_route(model) - verbose_logger.debug("Cohere route: %s", cohere_route) - # Set API base based on route - if cohere_route == "v2": - api_base = api_base or litellm.api_base or get_secret_str("COHERE_API_BASE") or "https://api.cohere.com/v2/chat" - # Remove v2/ prefix from model name for the actual API call - if "v2/" in model: - model = model.replace("v2/", "") - else: - api_base = api_base or litellm.api_base or get_secret_str("COHERE_API_BASE") or "https://api.cohere.ai/v1/chat" - - headers = headers or litellm.headers or {} - if headers is None: - headers = {} - - if extra_headers is not None: - headers.update(extra_headers) - - verbose_logger.debug("Model: %s, API Base: %s", model, api_base) - verbose_logger.debug("Provider Config: %s", provider_config) - return base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="cohere_chat", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=cohere_key, - provider_config=provider_config, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - ) - - -def _complete_maritalk(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - api_base = ctx.api_base - api_key: Final = ctx.api_key - custom_prompt_dict: Final = ctx.custom_prompt_dict - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - - maritalk_key: Final = api_key or litellm.maritalk_key or get_secret("MARITALK_API_KEY") or litellm.api_key - - api_base = api_base or litellm.api_base or get_secret("MARITALK_API_BASE") or "https://chat.maritaca.ai/api" - - return openai_like_chat_completion.completion( - model=model, - messages=messages, - api_base=api_base, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - api_key=maritalk_key, - logging_obj=logging, - custom_llm_provider="maritalk", - custom_prompt_dict=custom_prompt_dict, - ) - - -def _complete_amazon_nova(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - api_base = ctx.api_base - api_key = ctx.api_key - custom_llm_provider: Final = ctx.custom_llm_provider - custom_prompt_dict: Final = ctx.custom_prompt_dict - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - timeout: Final = ctx.timeout - - api_key = api_key or litellm.amazon_nova_api_key or get_secret_str("AMAZON_NOVA_API_KEY") or litellm.api_key - api_base = ( - api_base or litellm.api_base or get_secret_str("AMAZON_NOVA_API_BASE") or "https://api.nova.amazon.com/v1" - ) - return openai_like_chat_completion.completion( - model=model, - messages=messages, - api_base=api_base, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - timeout=timeout, - custom_llm_provider=custom_llm_provider, - custom_prompt_dict=custom_prompt_dict, - ) - - -def _complete_huggingface(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key: Final = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - huggingface_key: Final = ( - api_key - or litellm.huggingface_key - or os.environ.get("HF_TOKEN") - or os.environ.get("HUGGINGFACE_API_KEY") - or litellm.api_key - ) - hf_headers: Final = headers or litellm.headers - return base_llm_http_handler.completion( - model=model, - messages=messages, - headers=hf_headers, - model_response=model_response, - api_key=huggingface_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - ) - - -def _complete_oci(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key: Final = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - return base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - ) - - -def _complete_compactifai(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - provider_config: Final = ctx.provider_config - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_key = api_key or get_secret_str("COMPACTIFAI_API_KEY") or litellm.api_key - - api_base = api_base or "https://api.compactif.ai/v1" - - ## COMPLETION CALL - return base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) - - -def _complete_oobabooga(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - api_base: Final = ctx.api_base - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response = ctx.model_response - optional_params: Final = ctx.optional_params - - model_response = oobabooga.completion( - model=model, - messages=messages, - model_response=model_response, - api_base=api_base, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - api_key=None, - logger_fn=logger_fn, - encoding=_get_encoding(), - logging_obj=logging, - ) - if "stream" in optional_params and optional_params["stream"] is True: - # don't try to access stream object, - return CustomStreamWrapper( - model_response, - model, - custom_llm_provider="oobabooga", - logging_obj=logging, - ) - return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract - - -def _complete_databricks(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = DatabricksBase.resolve_api_base(ctx.api_base) - api_key: Final = DatabricksBase.resolve_api_key(ctx.api_key) - client: Final = _dispatch_client_http(ctx) - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - headers = headers or litellm.headers - - ## COMPLETION CALL - try: - response: Final = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - custom_llm_provider="databricks", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, - ) - except Exception as e: - ## LOGGING - log the original exception returned - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e - - if optional_params.get("stream", False): - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=response, - additional_args={"headers": headers}, - ) - - return response - - -def _complete_datarobot(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key: Final = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - provider_config: Final = ctx.provider_config - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - return base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) - - -def _complete_openrouter(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_base = api_base or litellm.api_base or get_secret_str("OPENROUTER_API_BASE") or "https://openrouter.ai/api/v1" - - api_key = ( - api_key - or litellm.api_key - or litellm.openrouter_key - or get_secret_str("OPENROUTER_API_KEY") - or get_secret_str("OR_API_KEY") - ) - - openrouter_site_url: Final = get_secret("OR_SITE_URL") or "https://litellm.ai" - openrouter_app_name: Final = get_secret("OR_APP_NAME") or "liteLLM" - - openrouter_headers: Final = { - "HTTP-Referer": openrouter_site_url, - "X-Title": openrouter_app_name, - } - - _headers: Final = headers or litellm.headers - if _headers: - openrouter_headers.update(_headers) - - headers = openrouter_headers - - ## Load Config - config: Final = litellm.OpenrouterConfig.get_config() - for k, v in _provider_config_items(config): - if k == "extra_body": - # we use openai 'extra_body' to pass openrouter specific params - transforms, route, models - if "extra_body" in optional_params: - optional_params[k].update(v) - else: - optional_params[k] = v - elif k not in optional_params: - optional_params[k] = v - - ## COMPLETION CALL - response: Final = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="openrouter", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, - ) - ## LOGGING - logging.post_call(input=messages, api_key=openai.api_key, original_response=response) - - return response - - -def _complete_nadir(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - api_base: Final = ctx.api_base or litellm.api_base or get_secret_str("NADIR_API_BASE") or NADIR_DEFAULT_API_BASE - api_key: Final = ctx.api_key - - response: Final = base_llm_http_handler.completion( - model=ctx.model, - stream=ctx.stream, - messages=ctx.messages, - acompletion=ctx.acompletion, - api_base=api_base, - model_response=ctx.model_response, - optional_params=ctx.optional_params, - litellm_params=ctx.litellm_params, - shared_session=ctx.shared_session, - custom_llm_provider="nadir", - timeout=ctx.timeout, - headers=ctx.headers or litellm.headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=ctx.logging, - client=ctx.client, - ) - ctx.logging.post_call(input=ctx.messages, api_key=api_key, original_response=response) - - return response - - -def _complete_vercel_ai_gateway( - ctx: CompletionDispatchContext, -) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_base = ( - api_base - or litellm.api_base - or get_secret_str("VERCEL_AI_GATEWAY_API_BASE") - or "https://ai-gateway.vercel.sh/v1" - ) - - api_key = api_key or litellm.api_key or get_secret("VERCEL_AI_GATEWAY_API_KEY") - - vercel_site_url: Final = get_secret("VERCEL_SITE_URL") or "https://litellm.ai" - vercel_app_name: Final = get_secret("VERCEL_APP_NAME") or "liteLLM" - - vercel_headers: Final = { - "http-referer": vercel_site_url, - "x-title": vercel_app_name, - } - - _headers: Final = headers or litellm.headers - if _headers: - vercel_headers.update(_headers) - - headers = vercel_headers - - ## Load Config - config: Final = litellm.VercelAIGatewayConfig.get_config() - for k, v in _provider_config_items(config): - if k == "extra_body": - # we use openai 'extra_body' to pass vercel specific params - providerOptions - if "extra_body" in optional_params: - optional_params[k].update(v) - else: - optional_params[k] = v - elif k not in optional_params: - optional_params[k] = v - - ## COMPLETION CALL - response: Final = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="vercel_ai_gateway", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, - ) - ## LOGGING - logging.post_call(input=messages, api_key=openai.api_key, original_response=response) - - return response - - -def _complete_edenai(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - api_base: Final = litellm.EdenAIChatConfig.get_api_base(ctx.api_base) - api_key: Final = litellm.EdenAIChatConfig.get_api_key(ctx.api_key or litellm.api_key) - response: Final = base_llm_http_handler.completion( - model=ctx.model, - messages=ctx.messages, - api_base=api_base, - custom_llm_provider="edenai", - model_response=ctx.model_response, - encoding=_get_encoding(), - logging_obj=ctx.logging, - optional_params=ctx.optional_params, - timeout=ctx.timeout, - litellm_params=ctx.litellm_params, - shared_session=ctx.shared_session, - acompletion=ctx.acompletion, - stream=ctx.stream, - api_key=api_key, - headers=ctx.headers or litellm.headers, - client=_dispatch_client_http(ctx), - provider_config=ctx.provider_config, - ) - ctx.logging.post_call(input=ctx.messages, api_key=api_key, original_response=response) - return response - - -def _complete_fal_ai(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - if ctx.stream: - raise litellm.FalAIError( - status_code=400, - message="fal_ai chat completions do not support streaming", - ) - api_base: Final = litellm.FalAIChatConfig.get_api_base(ctx.api_base) - api_key: Final = litellm.FalAIChatConfig.get_api_key(ctx.api_key or litellm.api_key) - response: Final = base_llm_http_handler.completion( - model=ctx.model, - messages=ctx.messages, - api_base=api_base, - custom_llm_provider="fal_ai", - model_response=ctx.model_response, - encoding=_get_encoding(), - logging_obj=ctx.logging, - optional_params=ctx.optional_params, - timeout=ctx.timeout, - litellm_params=ctx.litellm_params, - shared_session=ctx.shared_session, - acompletion=ctx.acompletion, - stream=ctx.stream, - api_key=api_key, - headers=ctx.headers or litellm.headers, - client=_dispatch_client_http(ctx), - provider_config=ctx.provider_config, - ) - ctx.logging.post_call(input=ctx.messages, api_key=api_key, original_response=response) - return response - - -def _complete_vertex_ai_beta( - ctx: CompletionDispatchContext, -) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key: Final = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - timeout: Final = ctx.timeout - - vertex_ai_project: Final = ( - optional_params.pop("vertex_project", None) - or optional_params.pop("vertex_ai_project", None) - or litellm.vertex_project - or get_secret("VERTEXAI_PROJECT") - ) - vertex_ai_location: Final = ( - optional_params.pop("vertex_location", None) - or optional_params.pop("vertex_ai_location", None) - or litellm.vertex_location - or get_secret("VERTEXAI_LOCATION") - ) - vertex_credentials: Final = ( - optional_params.pop("vertex_credentials", None) - or optional_params.pop("vertex_ai_credentials", None) - or get_secret("VERTEXAI_CREDENTIALS") - ) - - gemini_api_key: Final = ( - api_key - or get_api_key_from_env() - or get_secret("PALM_API_KEY") # older palm api key should also work - or litellm.api_key - ) - - api_base = api_base or litellm.api_base or get_secret("GEMINI_API_BASE") - new_params: Final = safe_deep_copy(optional_params or {}) - return vertex_chat_completion.completion( - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - optional_params=new_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - vertex_location=vertex_ai_location, - vertex_project=vertex_ai_project, - vertex_credentials=vertex_credentials, - gemini_api_key=gemini_api_key, - logging_obj=logging, - acompletion=acompletion, - timeout=timeout, - custom_llm_provider=custom_llm_provider, - client=client, - api_base=api_base, - extra_headers=headers, - ) - - -def _complete_vertex_ai(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - custom_prompt_dict: Final = ctx.custom_prompt_dict - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response = ctx.model_response - optional_params: Final = ctx.optional_params - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - vertex_ai_project: Final = ( - optional_params.pop("vertex_project", None) - or optional_params.pop("vertex_ai_project", None) - or litellm.vertex_project - or get_secret("VERTEXAI_PROJECT") - ) - vertex_ai_location: Final = ( - optional_params.pop("vertex_location", None) - or optional_params.pop("vertex_ai_location", None) - or litellm.vertex_location - or get_secret("VERTEXAI_LOCATION") - ) - vertex_credentials: Final = ( - optional_params.pop("vertex_credentials", None) - or optional_params.pop("vertex_ai_credentials", None) - or get_secret("VERTEXAI_CREDENTIALS") - ) - - api_base = api_base or litellm.api_base or get_secret("VERTEXAI_API_BASE") - - new_params: Final = safe_deep_copy(optional_params or {}) - model_route: Final = get_vertex_ai_model_route(model=model, litellm_params=litellm_params) - - if model_route == VertexAIModelRoute.PARTNER_MODELS: - model_response = vertex_partner_models_chat_completion.completion( - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - optional_params=new_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - api_base=api_base, - vertex_location=vertex_ai_location, - vertex_project=vertex_ai_project, - vertex_credentials=vertex_credentials, - logging_obj=logging, - acompletion=acompletion, - headers=headers, - custom_prompt_dict=custom_prompt_dict, - timeout=timeout, - client=client, - ) - elif model_route == VertexAIModelRoute.GEMINI: - model_response = vertex_chat_completion.completion( - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - optional_params=new_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - vertex_location=vertex_ai_location, - vertex_project=vertex_ai_project, - vertex_credentials=vertex_credentials, - gemini_api_key=None, - logging_obj=logging, - acompletion=acompletion, - timeout=timeout, - custom_llm_provider=custom_llm_provider, - client=client, - api_base=api_base, - extra_headers=headers, - ) - elif model_route == VertexAIModelRoute.GEMMA: - # Vertex Gemma Models with custom prediction endpoint - model_response = vertex_gemma_chat_completion.completion( - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - optional_params=new_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - api_base=api_base, - vertex_location=vertex_ai_location, - vertex_project=vertex_ai_project, - vertex_credentials=vertex_credentials, - logging_obj=logging, - acompletion=acompletion, - headers=headers, - custom_prompt_dict=custom_prompt_dict, - timeout=timeout, - client=client, - ) - elif model_route == VertexAIModelRoute.MODEL_GARDEN: - # Vertex Model Garden - OpenAI compatible models - model_response = vertex_model_garden_chat_completion.completion( - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - optional_params=new_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - api_base=api_base, - vertex_location=vertex_ai_location, - vertex_project=vertex_ai_project, - vertex_credentials=vertex_credentials, - logging_obj=logging, - acompletion=acompletion, - headers=headers, - custom_prompt_dict=custom_prompt_dict, - timeout=timeout, - client=client, - ) - elif model_route == VertexAIModelRoute.AGENT_ENGINE: - # Vertex AI Agent Engine (Reasoning Engines) - from litellm.llms.vertex_ai.agent_engine.transformation import ( - VertexAgentEngineConfig, - ) - - vertex_agent_engine_config: Final = VertexAgentEngineConfig() - - # Update litellm_params with vertex credentials - litellm_params["vertex_project"] = vertex_ai_project - litellm_params["vertex_location"] = vertex_ai_location - litellm_params["vertex_credentials"] = vertex_credentials - - model_response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - model_response=model_response, - optional_params=new_params, - litellm_params=litellm_params, - encoding=_get_encoding(), - api_key=None, - api_base=api_base, - logging_obj=logging, - acompletion=acompletion, - timeout=timeout, - client=client, - custom_llm_provider="vertex_ai", - provider_config=vertex_agent_engine_config, - headers=headers or {}, - ) - else: # VertexAIModelRoute.NON_GEMINI - model_response = vertex_ai_non_gemini.completion( - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - optional_params=new_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - vertex_location=vertex_ai_location, - vertex_project=vertex_ai_project, - vertex_credentials=vertex_credentials, - logging_obj=logging, - acompletion=acompletion, - ) - - if "stream" in optional_params and optional_params["stream"] is True and acompletion is False: - return CustomStreamWrapper( - model_response, - model, - custom_llm_provider="vertex_ai", - logging_obj=logging, - ) - return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract - - -def _complete_predibase(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - custom_prompt_dict: Final = ctx.custom_prompt_dict - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - timeout: Final = ctx.timeout - - tenant_id: Final = ( - optional_params.pop("tenant_id", None) - or optional_params.pop("predibase_tenant_id", None) - or litellm.predibase_tenant_id - or get_secret("PREDIBASE_TENANT_ID") - ) - - if tenant_id is None: - raise ValueError( - "Missing Predibase Tenant ID - Required for making the request. Set dynamically (e.g. `completion(..tenant_id=)`) or in env - `PREDIBASE_TENANT_ID`." - ) - - api_base = ( - api_base - or optional_params.pop("api_base", None) - or optional_params.pop("base_url", None) - or litellm.api_base - or get_secret("PREDIBASE_API_BASE") - ) - - api_key = api_key or litellm.api_key or litellm.predibase_key or get_secret("PREDIBASE_API_KEY") - - _model_response: Final = predibase_chat_completions.completion( - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - logging_obj=logging, - acompletion=acompletion, - api_base=api_base, - custom_prompt_dict=custom_prompt_dict, - api_key=api_key, - tenant_id=tenant_id, - timeout=timeout, - ) - - if "stream" in optional_params and optional_params["stream"] is True and acompletion is False: - return _model_response - return _model_response - - -def _complete_text_completion_codestral( - ctx: CompletionDispatchContext, -) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - custom_prompt_dict: Final = ctx.custom_prompt_dict - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - optional_params: Final = ctx.optional_params - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_base = ( - api_base - or optional_params.pop("api_base", None) - or optional_params.pop("base_url", None) - or litellm.api_base - or "https://codestral.mistral.ai/v1/fim/completions" - ) - - api_key = api_key or litellm.api_key or get_secret("CODESTRAL_API_KEY") - - text_completion_model_response: Final = litellm.TextCompletionResponse(stream=stream) - - _model_response: Final = codestral_text_completions.completion( - model=model, - messages=messages, - model_response=text_completion_model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - logging_obj=logging, - acompletion=acompletion, - api_base=api_base, - custom_prompt_dict=custom_prompt_dict, - api_key=api_key, - timeout=timeout, - ) - - if "stream" in optional_params and optional_params["stream"] is True and acompletion is False: - return _model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract - return _model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract - - -def _complete_text_completion_inception( - ctx: CompletionDispatchContext, -) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_openai(ctx) - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - text_completion: Final = ctx.text_completion - timeout: Final = ctx.timeout - - passed_api_base: Final = api_base or optional_params.pop("api_base", None) or optional_params.pop("base_url", None) - api_base = passed_api_base or get_secret_str("INCEPTION_API_BASE") or "https://api.inceptionlabs.ai/v1" - # FIM is served at `/v1/fim/completions`; the OpenAI client appends - # `/completions`, so point it at the `/v1/fim` base. - api_base = api_base.rstrip("/") - if not api_base.endswith("/fim"): - api_base += "/fim" - - # Don't forward the server-managed Inception key to a caller-supplied - # api_base; only resolve it for the default/server base, or when the - # caller passes their own key. - if passed_api_base is None or api_key: - api_key = api_key or litellm.inception_key or get_secret_str("INCEPTION_API_KEY") - - _response = openai_text_completions.completion( - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - api_key=api_key, - custom_llm_provider="text-completion-inception", - api_base=api_base, - acompletion=acompletion, - client=client, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - timeout=timeout, - ) - - if optional_params.get("stream", False) is False and acompletion is False and text_completion is False: - _response = litellm.OpenAITextCompletionConfig().convert_to_chat_model_response_object( - response_object=_response, model_response_object=model_response - ) - - if optional_params.get("stream", False) or acompletion is True: - logging.post_call( - input=messages, - api_key=api_key, - original_response=_response, - additional_args={"headers": headers}, - ) - return _response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract - - -def _complete_sagemaker_chat( - ctx: CompletionDispatchContext, -) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key: Final = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - return base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - custom_llm_provider=custom_llm_provider, - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, - ) - - -def _complete_sagemaker(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - custom_prompt_dict: Final = ctx.custom_prompt_dict - hf_model_name: Final = ctx.hf_model_name - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - - return sagemaker_llm.completion( - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - custom_prompt_dict=custom_prompt_dict, - hf_model_name=hf_model_name, - logger_fn=logger_fn, - encoding=_get_encoding(), - logging_obj=logging, - acompletion=acompletion, - ) - - -_ADDITIONAL_DROP_PARAMS_ADAPTER: Final = TypeAdapter(list[str]) _OPTIONAL_PARAMS_ADAPTER: Final = TypeAdapter(dict[str, object]) _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object]) @@ -4209,1123 +1378,6 @@ _PROVIDER_ROUTING_HINTS: Final = TypeAdapter(ProviderRoutingHints) _BASE_URL_HINT: Final = TypeAdapter(BaseUrlRoutingHint) -_AWS_BEDROCK_CLIENT_CREDENTIAL_KEYS: Final = ( - "aws_access_key_id", - "aws_secret_access_key", - "aws_session_token", - "aws_region_name", -) - - -def _native_output_fallback( - ctx: CompletionDispatchContext, - first_attempt_params: Mapping[str, object], - first_attempt_litellm_params: Mapping[str, object], -) -> NativeOutputFallback: - litellm_params: Final = without_bedrock_native_output_opt_in(first_attempt_litellm_params) - remapped_params: Final = ctx.optional_params_for_model_info(deployment_model_info(litellm_params)) - return NativeOutputFallback( - optional_params={ - **{key: value for key, value in remapped_params.items() if key != "aws_bedrock_client"}, - **{ - key: first_attempt_params[key] - for key in _AWS_BEDROCK_CLIENT_CREDENTIAL_KEYS - if key in first_attempt_params - }, - }, - litellm_params=litellm_params, - ) - - -def _converse_native_output_fallback( - ctx: CompletionDispatchContext, - first_attempt_params: Mapping[str, object], - first_attempt_litellm_params: Mapping[str, object], -) -> NativeOutputFallback | None: - fallback: Final = _native_output_fallback(ctx, first_attempt_params, first_attempt_litellm_params) - return None if AmazonConverseConfig.sends_native_text_format(fallback.optional_params) else fallback - - -def _native_output_attempt_timeout( - ctx: CompletionDispatchContext, fallback: NativeOutputFallback | None -) -> float | str | httpx.Timeout | None: - if fallback is None: - return ctx.timeout - retry_timeout: Final = ctx.timeout_for_optional_params(fallback.optional_params) - ctx.logging.record_applied_timeout(retry_timeout) - return retry_timeout - - -def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key: Final = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_prompt_dict = ctx.custom_prompt_dict - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - provider_config = ctx.provider_config - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict - - if "aws_bedrock_client" in optional_params: - verbose_logger.warning( - "'aws_bedrock_client' is a deprecated param. Please move to another auth method - https://docs.litellm.ai/docs/providers/bedrock#boto3---authentication." - ) - # Extract credentials for legacy boto3 client and pass thru to httpx - aws_bedrock_client: Final = optional_params.pop("aws_bedrock_client") - creds: Final = aws_bedrock_client._get_credentials().get_frozen_credentials() - - if creds.access_key: - optional_params["aws_access_key_id"] = creds.access_key - if creds.secret_key: - optional_params["aws_secret_access_key"] = creds.secret_key - if creds.token: - optional_params["aws_session_token"] = creds.token - if "aws_region_name" not in optional_params or optional_params["aws_region_name"] is None: - optional_params["aws_region_name"] = aws_bedrock_client.meta.region_name - - additional_drop_params: Final = ( - _ADDITIONAL_DROP_PARAMS_ADAPTER.validate_python(ctx.kwargs["additional_drop_params"]) - if ctx.kwargs.get("additional_drop_params") is not None - else None - ) - bedrock_route: Final = bedrock_route_for_request(model, ctx.request_params, additional_drop_params) - if bedrock_route == "claude_platform": - provider_config = ProviderConfigManager.get_provider_chat_config( - model=model, - provider=LlmProviders.BEDROCK, - ) - model = BedrockModelInfo.get_claude_platform_model(model) - return base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="bedrock", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - client=client, - provider_config=provider_config, - ) - elif bedrock_route == "converse": - model = without_bedrock_route_prefix(model) - converse_model: Final = model - - def converse_attempt(fallback: NativeOutputFallback | None) -> NativeOutputAttemptResult: - return bedrock_converse_chat_completion.completion( - model=converse_model, - messages=messages, - custom_prompt_dict=custom_prompt_dict, - model_response=model_response, - optional_params=optional_params if fallback is None else dict(fallback.optional_params), - litellm_params=litellm_params if fallback is None else dict(fallback.litellm_params), - logger_fn=logger_fn, - encoding=_get_encoding(), - logging_obj=logging, - extra_headers=headers, # Use merged headers instead of original extra_headers - timeout=_native_output_attempt_timeout(ctx, fallback), - acompletion=acompletion, - client=client, - api_base=api_base, - api_key=api_key, - ) - - converse_params: Final = _OPTIONAL_PARAMS_ADAPTER.validate_python(optional_params) - response = call_with_native_output_fallback( - converse_attempt, - ( - partial(_converse_native_output_fallback, ctx, converse_params, dict(litellm_params)) - if AmazonConverseConfig().uses_opt_in_native_output(converse_model, converse_params, litellm_params) - else None - ), - converse_model, - ) - elif bedrock_route == "converse_like": - model = model.replace("converse_like/", "") - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - custom_llm_provider="bedrock", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, - ) - else: - - def invoke_attempt(fallback: NativeOutputFallback | None) -> NativeOutputAttemptResult: - return base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params if fallback is None else dict(fallback.optional_params), - litellm_params=litellm_params if fallback is None else dict(fallback.litellm_params), - custom_llm_provider="bedrock", - timeout=_native_output_attempt_timeout(ctx, fallback), - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - client=client, - ) - - invoke_params: Final = _OPTIONAL_PARAMS_ADAPTER.validate_python(optional_params) - response = call_with_native_output_fallback( - invoke_attempt, - ( - partial(_native_output_fallback, ctx, invoke_params, dict(litellm_params)) - if isinstance(provider_config, AmazonAnthropicClaudeConfig) - and provider_config.uses_opt_in_native_output(model, invoke_params, litellm_params) - else None - ), - model, - ) - - return response - - -def _complete_watsonx(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key: Final = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_prompt_dict: Final = ctx.custom_prompt_dict - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - timeout: Final = ctx.timeout - - return watsonx_chat_completion.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - print_verbose=print_verbose, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - timeout=timeout, - custom_prompt_dict=custom_prompt_dict, - client=client, # pass AsyncOpenAI, OpenAI client - encoding=_get_encoding(), - custom_llm_provider="watsonx", - ) - - -def _complete_watsonx_text( - ctx: CompletionDispatchContext, -) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_key = ( - api_key - or optional_params.pop("apikey", None) - or get_secret_str("WATSONX_APIKEY") - or get_secret_str("WATSONX_API_KEY") - or get_secret_str("WX_API_KEY") - ) - - api_base = ( - api_base - or optional_params.pop( - "url", - optional_params.pop("api_base", optional_params.pop("base_url", None)), - ) - or get_secret_str("WATSONX_API_BASE") - or get_secret_str("WATSONX_URL") - or get_secret_str("WX_URL") - or get_secret_str("WML_URL") - ) - - wx_credentials: Final = optional_params.pop( - "wx_credentials", - optional_params.pop("watsonx_credentials", None), # follow {provider}_credentials, same as vertex ai - ) - - token: str | None = None - if wx_credentials is not None: - api_base = wx_credentials.get("url", api_base) - api_key = wx_credentials.get("apikey", wx_credentials.get("api_key", api_key)) - token = wx_credentials.get( - "token", - wx_credentials.get( - "watsonx_token", None - ), # follow format of {provider}_token, same as azure - e.g. 'azure_ad_token=..' - ) - - if token is not None: - optional_params["token"] = token - - return base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="watsonx_text", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, - ) - - -def _complete_vllm(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - custom_prompt_dict = ctx.custom_prompt_dict - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response = ctx.model_response - optional_params: Final = ctx.optional_params - - custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict - model_response = vllm_handler.completion( - model=model, - messages=messages, - custom_prompt_dict=custom_prompt_dict, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - logging_obj=logging, - ) - - if "stream" in optional_params and optional_params["stream"] is True: ## [BETA] - # don't try to access stream object, - return CustomStreamWrapper( - model_response, - model, - custom_llm_provider="vllm", - logging_obj=logging, - ) - - ## RESPONSE OBJECT - return model_response - - -def _complete_ollama(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key: Final = ctx.api_key - client: Final = _dispatch_client_http(ctx) - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_base = litellm.api_base or api_base or get_secret("OLLAMA_API_BASE") or "http://localhost:11434" - if api_key is not None and "Authorization" not in headers: - headers["Authorization"] = f"Bearer {api_key}" - - return base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="ollama", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, - ) - - -def _complete_ollama_chat(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_base = litellm.api_base or api_base or get_secret("OLLAMA_API_BASE") or "http://localhost:11434" - - api_key = api_key or litellm.ollama_key or os.environ.get("OLLAMA_API_KEY") or litellm.api_key - if api_key is not None and "Authorization" not in headers: - headers["Authorization"] = f"Bearer {api_key}" - - return base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="ollama_chat", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, - ) - - -def _complete_triton(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key: Final = ctx.api_key - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_base = litellm.api_base or api_base - return base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider=custom_llm_provider, - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - ) - - -def _complete_cloudflare(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - custom_prompt_dict = ctx.custom_prompt_dict - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_key = api_key or litellm.cloudflare_api_key or litellm.api_key or get_secret("CLOUDFLARE_API_KEY") - api_base = api_base or litellm.api_base or get_secret("CLOUDFLARE_API_BASE") - - custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict - return base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="cloudflare", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - ) - - -def _complete_petals(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - api_base = ctx.api_base - client: Final = _dispatch_client_http(ctx) - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response = ctx.model_response - optional_params: Final = ctx.optional_params - stream = ctx.stream - - api_base = api_base or litellm.api_base - - stream = optional_params.pop("stream", False) - model_response = petals_handler.completion( - model=model, - messages=messages, - api_base=api_base, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - logging_obj=logging, - client=client, - ) - if stream is True: ## [BETA] - # Fake streaming for petals - resp_string: Final = model_response["choices"][0]["message"]["content"] - return CustomStreamWrapper( - resp_string, - model, - custom_llm_provider="petals", - logging_obj=logging, - ) - return model_response - - -def _complete_snowflake(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key: Final = ctx.api_key - injected_client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - client: Final = ( - injected_client if injected_client is not None else (HTTPHandler(timeout=timeout) if stream is False else None) - ) # Keep this here, otherwise, the httpx.client closes and streaming is impossible - - try: - response: Final = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - ) - - except Exception as e: - ## LOGGING - log the original exception returned - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e - - return response - - -def _complete_gradient_ai(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key: Final = ctx.api_key - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_base = litellm.api_base or api_base - return base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="gradient_ai", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - ) - - -def _complete_gdc(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_key = api_key or litellm.gdc_key or get_secret_str("GDC_API_KEY") or litellm.api_key - api_base = api_base or litellm.gdc_api_base or get_secret_str("GDC_API_BASE") or litellm.api_base - - return base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=gdc_transformation, - ) - - -def _complete_bytez(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_key = api_key or litellm.bytez_key or get_secret_str("BYTEZ_API_KEY") or litellm.api_key - - response: Final = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=bytez_transformation, - ) - - return response - - -def _complete_lemonade(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_key = api_key or litellm.lemonade_key or get_secret_str("LEMONADE_API_KEY") or litellm.api_key - - response: Final = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=lemonade_transformation, - ) - - return response - - -def _complete_ovhcloud(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_key = api_key or litellm.ovhcloud_key or get_secret_str("OVHCLOUD_API_KEY") or litellm.api_key - - api_base = ( - api_base - or litellm.api_base - or get_secret_str("OVHCLOUD_API_BASE") - or "https://oai.endpoints.kepler.ai.cloud.ovh.net/v1" - ) - - response: Final = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=ovhcloud_transformation, - ) - - return response - - -def _custom_api_first_output(resp: httpx.Response | None) -> str: - return resp.json()["data"][0]["output"][0] - - -def _complete_custom(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - api_base: Final = ctx.api_base - headers: Final = ctx.headers - kwargs: Final = ctx.kwargs - max_tokens: Final = ctx.max_tokens - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - temperature: Final = ctx.temperature - top_p: Final = ctx.top_p - - url: Final = litellm.api_base or api_base or "" - if url is None or url == "": - raise ValueError("api_base not set. Set api_base or litellm.api_base for custom endpoints") - - """ - assume input to custom LLM api bases follow this format: - resp = litellm.module_level_client.post( - api_base, - json={ - 'model': 'meta-llama/Llama-2-13b-hf', # model name - 'params': { - 'prompt': ["The capital of France is P"], - 'max_tokens': 32, - 'temperature': 0.7, - 'top_p': 1.0, - 'top_k': 40, - } - } - ) - - """ - prompt: Final = " ".join([message["content"] for message in messages]) - resp: Final = litellm.module_level_client.post( - url, - headers=headers, - json={ - "model": model, - "params": { - "prompt": [prompt], - "max_tokens": max_tokens, - "temperature": temperature, - "top_p": top_p, - "top_k": kwargs.get("top_k"), - }, - **kwargs.get("extra_body", {}), - }, - ) - """ - assume all responses from custom api_bases of this format: - { - 'data': [ - { - 'prompt': 'The capital of France is P', - 'output': ['The capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France'], - 'params': {'temperature': 0.7, 'top_k': 40, 'top_p': 1}}], - 'message': 'ok' - } - ] - } - """ - string_response: Final = _custom_api_first_output(resp) - ## RESPONSE OBJECT - model_response.choices[0].message.content = string_response - model_response.created = int(time.time()) - model_response.model = model - return model_response - - -def _complete_custom_providers( - ctx: CompletionDispatchContext, -) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key: Final = ctx.api_key - client: Final = ctx.client - custom_llm_provider: Final = ctx.custom_llm_provider - custom_prompt_dict: Final = ctx.custom_prompt_dict - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - custom_handler: CustomLLM | None = None - for item in litellm.custom_provider_map: - if item["provider"] == custom_llm_provider: - custom_handler = item["custom_handler"] - - if custom_handler is None: - raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider) - - ## ROUTE LLM CALL ## - handler_fn: Final = custom_chat_llm_router(async_fn=acompletion, stream=stream, custom_llm=custom_handler) - - headers = headers or litellm.headers or {} - - ## CALL FUNCTION - response: Final = handler_fn( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - print_verbose=print_verbose, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - timeout=timeout, - custom_prompt_dict=custom_prompt_dict, - client=client, # pass AsyncOpenAI, OpenAI client - encoding=_get_encoding(), - ) - if stream is True: - return CustomStreamWrapper( - completion_stream=response, - model=model, - custom_llm_provider=custom_llm_provider, - logging_obj=logging, - ) - - return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract - - -def _complete_langgraph(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - from litellm.llms.langgraph.chat.transformation import LangGraphConfig - - ( - api_base, - api_key, - ) = LangGraphConfig().get_openai_compatible_provider_info( - api_base=api_base or litellm.api_base, - api_key=api_key or litellm.api_key, - ) - - headers = headers or litellm.headers - - return base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider=custom_llm_provider, - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - client=client, - ) - - -def _complete_microsoft_365_copilot(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - from litellm.llms.microsoft_365_copilot.chat.handler import acompletion, completion - from litellm.llms.microsoft_365_copilot.common_utils import as_secret_fields - - messages: Final[Sequence[AllMessageValues]] = cast( # cast-ok: dispatch messages are prevalidated - Sequence[AllMessageValues], - ctx.messages, - ) - optional_params_value: Final[object] = cast( # cast-ok: dispatch params are validated below - object, - ctx.optional_params, - ) - optional_params: Final = TypeAdapter(Mapping[str, object]).validate_python(optional_params_value) - kwargs_value: Final[object] = cast( # cast-ok: dispatch kwargs are validated below - object, - ctx.kwargs, - ) - kwargs: Final = TypeAdapter(Mapping[str, object]).validate_python(kwargs_value) - secret_fields: Final = as_secret_fields(kwargs.get("secret_fields")) - client: Final = _dispatch_client_http(ctx) - if ctx.acompletion: - async_client: Final = client if isinstance(client, AsyncHTTPHandler) else None - return acompletion( - model=ctx.model, - messages=messages, - api_key=ctx.api_key, - litellm_params=ctx.litellm_params, - optional_params=optional_params, - secret_fields=secret_fields, - stream=ctx.stream is True, - timeout=ctx.timeout, - logging_obj=ctx.logging, - client=async_client, - shared_session=ctx.shared_session, - ) - sync_client: Final = client if isinstance(client, HTTPHandler) else None - return completion( - model=ctx.model, - messages=messages, - api_key=ctx.api_key, - litellm_params=ctx.litellm_params, - optional_params=optional_params, - secret_fields=secret_fields, - stream=ctx.stream is True, - timeout=ctx.timeout, - logging_obj=ctx.logging, - client=sync_client, - ) - - -def _complete_langflow(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - from litellm.llms.langflow.chat.transformation import LangFlowConfig - - ( - api_base, - api_key, - ) = LangFlowConfig().get_openai_compatible_provider_info( - api_base=api_base or litellm.api_base, - api_key=api_key or litellm.api_key, - ) - - headers = headers or litellm.headers - - return base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider=custom_llm_provider, - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - client=client, - ) - - -def _complete_databricks_agent(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - from litellm.llms.databricks.agent.transformation import DatabricksAgentConfig - - api_base, api_key = DatabricksAgentConfig.resolve_api_base_and_key( - api_base=ctx.api_base or litellm.api_base, - api_key=ctx.api_key or litellm.api_key, - ) - return base_llm_http_handler.completion( - model=ctx.model, - stream=ctx.stream, - messages=ctx.messages, - acompletion=ctx.acompletion, - api_base=api_base, - model_response=ctx.model_response, - optional_params=ctx.optional_params, - litellm_params={**ctx.litellm_params, "api_base": api_base}, - shared_session=ctx.shared_session, - custom_llm_provider=ctx.custom_llm_provider, - timeout=ctx.timeout, - headers=ctx.headers or litellm.headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=ctx.logging, - client=_dispatch_client_http(ctx), - ) - - @tracer.wrap() @client def completion( @@ -5520,7 +1572,6 @@ def completion( from litellm.integrations.anthropic_cache_control_hook import ( AnthropicCacheControlHook, ) - from litellm.types.llms.openai import AllMessageValues AnthropicCacheControlHook.maybe_seed_default_injection_points( non_default_params=non_default_params, @@ -9521,13 +5572,6 @@ async def ahealth_check( ####### HELPER FUNCTIONS ################ ## Set verbose to true -> ```litellm.set_verbose = True``` -def print_verbose(print_statement: object): - try: - verbose_logger.debug(print_statement) - if litellm.set_verbose: - print(print_statement) # noqa: T201 - except Exception: - pass def config_completion(**kwargs): diff --git a/litellm/native/chat_providers/__init__.py b/litellm/native/chat_providers/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/native/chat_providers/agents.py b/litellm/native/chat_providers/agents.py new file mode 100644 index 00000000000..5523ba0e6dc --- /dev/null +++ b/litellm/native/chat_providers/agents.py @@ -0,0 +1,273 @@ +from collections.abc import Mapping, Sequence +from typing import Final, cast # noqa: TID251 # casts moved verbatim from litellm/main.py + +from pydantic import TypeAdapter + +import litellm +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.native.chat_providers.handlers import base_llm_http_handler +from litellm.native.chat_providers.support import dispatch_client_http, get_encoding +from litellm.secret_managers.main import get_secret_str +from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult +from litellm.types.llms.openai import AllMessageValues + + +def complete_ragflow(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key: Final = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + provider_config: Final = ctx.provider_config + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + try: + response: Final = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + provider_config=provider_config, + ) + except Exception as e: + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + + return response + + +def complete_a2a(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + provider_config: Final = ctx.provider_config + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + ( + api_base, + api_key, + headers, + ) = litellm.A2AConfig.resolve_agent_config_from_registry( + agent_name=model, + api_base=api_base, + api_key=api_key, + headers=headers, + optional_params=optional_params, + ) + + # Fall back to environment variables and defaults + api_base = api_base or litellm.api_base or get_secret_str("A2A_API_BASE") + + if api_base is None: + raise Exception( + "api_base is required for A2A provider. " + "Either provide api_base parameter, set A2A_API_BASE environment variable, " + "or register the agent in the proxy with model='a2a/'." + ) + + headers = headers or litellm.headers + + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, + client=client, + provider_config=provider_config, + ) + + +def complete_langgraph(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + from litellm.llms.langgraph.chat.transformation import LangGraphConfig + + ( + api_base, + api_key, + ) = LangGraphConfig().get_openai_compatible_provider_info( + api_base=api_base or litellm.api_base, + api_key=api_key or litellm.api_key, + ) + + headers = headers or litellm.headers + + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, + client=client, + ) + + +def complete_microsoft_365_copilot(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + from litellm.llms.microsoft_365_copilot.chat.handler import acompletion, completion + from litellm.llms.microsoft_365_copilot.common_utils import as_secret_fields + + messages: Final[Sequence[AllMessageValues]] = cast( # cast-ok: dispatch messages are prevalidated + Sequence[AllMessageValues], + ctx.messages, + ) + optional_params_value: Final[object] = cast( # cast-ok: dispatch params are validated below + object, + ctx.optional_params, + ) + optional_params: Final = TypeAdapter(Mapping[str, object]).validate_python(optional_params_value) + kwargs_value: Final[object] = cast( # cast-ok: dispatch kwargs are validated below + object, + ctx.kwargs, + ) + kwargs: Final = TypeAdapter(Mapping[str, object]).validate_python(kwargs_value) + secret_fields: Final = as_secret_fields(kwargs.get("secret_fields")) + client: Final = dispatch_client_http(ctx) + if ctx.acompletion: + async_client: Final = client if isinstance(client, AsyncHTTPHandler) else None + return acompletion( + model=ctx.model, + messages=messages, + api_key=ctx.api_key, + litellm_params=ctx.litellm_params, + optional_params=optional_params, + secret_fields=secret_fields, + stream=ctx.stream is True, + timeout=ctx.timeout, + logging_obj=ctx.logging, + client=async_client, + shared_session=ctx.shared_session, + ) + sync_client: Final = client if isinstance(client, HTTPHandler) else None + return completion( + model=ctx.model, + messages=messages, + api_key=ctx.api_key, + litellm_params=ctx.litellm_params, + optional_params=optional_params, + secret_fields=secret_fields, + stream=ctx.stream is True, + timeout=ctx.timeout, + logging_obj=ctx.logging, + client=sync_client, + ) + + +def complete_langflow(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + from litellm.llms.langflow.chat.transformation import LangFlowConfig + + ( + api_base, + api_key, + ) = LangFlowConfig().get_openai_compatible_provider_info( + api_base=api_base or litellm.api_base, + api_key=api_key or litellm.api_key, + ) + + headers = headers or litellm.headers + + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, + client=client, + ) diff --git a/litellm/native/chat_providers/anthropic.py b/litellm/native/chat_providers/anthropic.py new file mode 100644 index 00000000000..66725f1c145 --- /dev/null +++ b/litellm/native/chat_providers/anthropic.py @@ -0,0 +1,131 @@ +import os +from typing import Final, cast # noqa: TID251 # casts moved verbatim from litellm/main.py + +import litellm +from litellm._logging import verbose_logger +from litellm.native.chat_providers.handlers import anthropic_chat_completions, base_llm_http_handler +from litellm.native.chat_providers.support import dispatch_client_http, get_encoding, print_verbose +from litellm.secret_managers.main import get_secret_bool +from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult +from litellm.utils import get_secret + + +def complete_anthropic_text( + ctx: CompletionDispatchContext, +) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + custom_prompt_dict = ctx.custom_prompt_dict + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_key = api_key or litellm.anthropic_key or litellm.api_key or os.environ.get("ANTHROPIC_API_KEY") + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + api_base = cast( + str | None, + api_base + or litellm.api_base + or get_secret("ANTHROPIC_API_BASE") + or get_secret("ANTHROPIC_BASE_URL") + or "https://api.anthropic.com/v1/complete", + ) + + # Check if we should disable automatic URL suffix appending + disable_url_suffix: Final = get_secret_bool("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX") + if api_base is not None and not disable_url_suffix and not api_base.endswith("/v1/complete"): + api_base += "/v1/complete" + elif disable_url_suffix: + verbose_logger.debug("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX is set, skipping /v1/complete suffix") + + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="anthropic_text", + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + ) + + +def complete_anthropic(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + custom_prompt_dict = ctx.custom_prompt_dict + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + timeout: Final = ctx.timeout + + api_key = api_key or litellm.anthropic_key or litellm.api_key or os.environ.get("ANTHROPIC_API_KEY") + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + # call /messages + # default route for all anthropic models + api_base = cast( + str | None, + api_base + or litellm.api_base + or get_secret("ANTHROPIC_API_BASE") + or get_secret("ANTHROPIC_BASE_URL") + or "https://api.anthropic.com/v1/messages", + ) + + # Check if we should disable automatic URL suffix appending + disable_url_suffix: Final = get_secret_bool("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX") + if api_base is not None and not disable_url_suffix and not api_base.endswith("/v1/messages"): + api_base += "/v1/messages" + elif disable_url_suffix: + verbose_logger.debug("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX is set, skipping /v1/messages suffix") + + response: Final = anthropic_chat_completions.completion( + model=model, + messages=messages, + api_base=api_base, + acompletion=acompletion, + custom_prompt_dict=litellm.custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=get_encoding(), # for calculating input/output tokens + api_key=api_key, + logging_obj=logging, + headers=headers, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + ) + if optional_params.get("stream", False) or acompletion is True: + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + ) + return response diff --git a/litellm/native/chat_providers/aws.py b/litellm/native/chat_providers/aws.py new file mode 100644 index 00000000000..3e6ee89e4eb --- /dev/null +++ b/litellm/native/chat_providers/aws.py @@ -0,0 +1,393 @@ +from collections.abc import Mapping +from functools import partial +from typing import Final + +import httpx +from pydantic import TypeAdapter + +import litellm +from litellm._logging import verbose_logger +from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig +from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( + AmazonAnthropicClaudeConfig, +) +from litellm.llms.bedrock.chat.native_output_fallback import NativeOutputAttemptResult, call_with_native_output_fallback +from litellm.llms.bedrock.common_utils import ( + BedrockModelInfo, + NativeOutputFallback, + bedrock_route_for_request, + deployment_model_info, + without_bedrock_native_output_opt_in, + without_bedrock_route_prefix, +) +from litellm.llms.bedrock_mantle.chat.claude_transformation import bedrock_mantle_chat_config +from litellm.native.chat_providers.handlers import ( + base_llm_http_handler, + bedrock_converse_chat_completion, + openai_like_chat_completion, + sagemaker_llm, +) +from litellm.native.chat_providers.support import ( + dispatch_client_http, + get_encoding, + print_verbose, + provider_config_items, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager, get_secret + +_ADDITIONAL_DROP_PARAMS_ADAPTER: Final = TypeAdapter(list[str]) +_OPTIONAL_PARAMS_ADAPTER: Final = TypeAdapter(dict[str, object]) + + +def complete_bedrock_mantle( + ctx: CompletionDispatchContext, +) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_base = api_base or litellm.api_base or get_secret("BEDROCK_MANTLE_API_BASE") + api_key = api_key or litellm.api_key or get_secret("BEDROCK_MANTLE_API_KEY") + headers = headers or litellm.headers + config: Final = bedrock_mantle_chat_config(model).get_config() + for k, v in provider_config_items(config): + if k not in optional_params: + optional_params[k] = v + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, + client=client, + ) + + +def complete_amazon_nova(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + api_base = ctx.api_base + api_key = ctx.api_key + custom_llm_provider: Final = ctx.custom_llm_provider + custom_prompt_dict: Final = ctx.custom_prompt_dict + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + timeout: Final = ctx.timeout + + api_key = api_key or litellm.amazon_nova_api_key or get_secret_str("AMAZON_NOVA_API_KEY") or litellm.api_key + api_base = ( + api_base or litellm.api_base or get_secret_str("AMAZON_NOVA_API_BASE") or "https://api.nova.amazon.com/v1" + ) + return openai_like_chat_completion.completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + custom_prompt_dict=custom_prompt_dict, + ) + + +def complete_sagemaker_chat( + ctx: CompletionDispatchContext, +) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key: Final = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) + + +def complete_sagemaker(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + custom_prompt_dict: Final = ctx.custom_prompt_dict + hf_model_name: Final = ctx.hf_model_name + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + + return sagemaker_llm.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + custom_prompt_dict=custom_prompt_dict, + hf_model_name=hf_model_name, + logger_fn=logger_fn, + encoding=get_encoding(), + logging_obj=logging, + acompletion=acompletion, + ) + + +_AWS_BEDROCK_CLIENT_CREDENTIAL_KEYS: Final = ( + "aws_access_key_id", + "aws_secret_access_key", + "aws_session_token", + "aws_region_name", +) + + +def _native_output_fallback( + ctx: CompletionDispatchContext, + first_attempt_params: Mapping[str, object], + first_attempt_litellm_params: Mapping[str, object], +) -> NativeOutputFallback: + litellm_params: Final = without_bedrock_native_output_opt_in(first_attempt_litellm_params) + remapped_params: Final = ctx.optional_params_for_model_info(deployment_model_info(litellm_params)) + return NativeOutputFallback( + optional_params={ + **{key: value for key, value in remapped_params.items() if key != "aws_bedrock_client"}, + **{ + key: first_attempt_params[key] + for key in _AWS_BEDROCK_CLIENT_CREDENTIAL_KEYS + if key in first_attempt_params + }, + }, + litellm_params=litellm_params, + ) + + +def _converse_native_output_fallback( + ctx: CompletionDispatchContext, + first_attempt_params: Mapping[str, object], + first_attempt_litellm_params: Mapping[str, object], +) -> NativeOutputFallback | None: + fallback: Final = _native_output_fallback(ctx, first_attempt_params, first_attempt_litellm_params) + return None if AmazonConverseConfig.sends_native_text_format(fallback.optional_params) else fallback + + +def _native_output_attempt_timeout( + ctx: CompletionDispatchContext, fallback: NativeOutputFallback | None +) -> float | str | httpx.Timeout | None: + if fallback is None: + return ctx.timeout + retry_timeout: Final = ctx.timeout_for_optional_params(fallback.optional_params) + ctx.logging.record_applied_timeout(retry_timeout) + return retry_timeout + + +def complete_bedrock(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key: Final = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_prompt_dict = ctx.custom_prompt_dict + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + provider_config = ctx.provider_config + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + + if "aws_bedrock_client" in optional_params: + verbose_logger.warning( + "'aws_bedrock_client' is a deprecated param. Please move to another auth method - https://docs.litellm.ai/docs/providers/bedrock#boto3---authentication." + ) + # Extract credentials for legacy boto3 client and pass thru to httpx + aws_bedrock_client: Final = optional_params.pop("aws_bedrock_client") + creds: Final = aws_bedrock_client._get_credentials().get_frozen_credentials() + + if creds.access_key: + optional_params["aws_access_key_id"] = creds.access_key + if creds.secret_key: + optional_params["aws_secret_access_key"] = creds.secret_key + if creds.token: + optional_params["aws_session_token"] = creds.token + if "aws_region_name" not in optional_params or optional_params["aws_region_name"] is None: + optional_params["aws_region_name"] = aws_bedrock_client.meta.region_name + + additional_drop_params: Final = ( + _ADDITIONAL_DROP_PARAMS_ADAPTER.validate_python(ctx.kwargs["additional_drop_params"]) + if ctx.kwargs.get("additional_drop_params") is not None + else None + ) + bedrock_route: Final = bedrock_route_for_request(model, ctx.request_params, additional_drop_params) + if bedrock_route == "claude_platform": + provider_config = ProviderConfigManager.get_provider_chat_config( + model=model, + provider=LlmProviders.BEDROCK, + ) + model = BedrockModelInfo.get_claude_platform_model(model) + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="bedrock", + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, + client=client, + provider_config=provider_config, + ) + elif bedrock_route == "converse": + model = without_bedrock_route_prefix(model) + converse_model: Final = model + + def converse_attempt(fallback: NativeOutputFallback | None) -> NativeOutputAttemptResult: + return bedrock_converse_chat_completion.completion( + model=converse_model, + messages=messages, + custom_prompt_dict=custom_prompt_dict, + model_response=model_response, + optional_params=optional_params if fallback is None else dict(fallback.optional_params), + litellm_params=litellm_params if fallback is None else dict(fallback.litellm_params), + logger_fn=logger_fn, + encoding=get_encoding(), + logging_obj=logging, + extra_headers=headers, # Use merged headers instead of original extra_headers + timeout=_native_output_attempt_timeout(ctx, fallback), + acompletion=acompletion, + client=client, + api_base=api_base, + api_key=api_key, + ) + + converse_params: Final = _OPTIONAL_PARAMS_ADAPTER.validate_python(optional_params) + response = call_with_native_output_fallback( + converse_attempt, + ( + partial(_converse_native_output_fallback, ctx, converse_params, dict(litellm_params)) + if AmazonConverseConfig().uses_opt_in_native_output(converse_model, converse_params, litellm_params) + else None + ), + converse_model, + ) + elif bedrock_route == "converse_like": + model = model.replace("converse_like/", "") + response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + custom_llm_provider="bedrock", + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) + else: + + def invoke_attempt(fallback: NativeOutputFallback | None) -> NativeOutputAttemptResult: + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params if fallback is None else dict(fallback.optional_params), + litellm_params=litellm_params if fallback is None else dict(fallback.litellm_params), + custom_llm_provider="bedrock", + timeout=_native_output_attempt_timeout(ctx, fallback), + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, + client=client, + ) + + invoke_params: Final = _OPTIONAL_PARAMS_ADAPTER.validate_python(optional_params) + response = call_with_native_output_fallback( + invoke_attempt, + ( + partial(_native_output_fallback, ctx, invoke_params, dict(litellm_params)) + if isinstance(provider_config, AmazonAnthropicClaudeConfig) + and provider_config.uses_opt_in_native_output(model, invoke_params, litellm_params) + else None + ), + model, + ) + + return response diff --git a/litellm/native/chat_providers/azure.py b/litellm/native/chat_providers/azure.py new file mode 100644 index 00000000000..ff12ce76faa --- /dev/null +++ b/litellm/native/chat_providers/azure.py @@ -0,0 +1,406 @@ +from typing import Final, cast # noqa: TID251 # casts moved verbatim from litellm/main.py + +import openai + +import litellm +from litellm.litellm_core_utils.prompt_templates.factory import stringify_json_tool_call_content +from litellm.llms.azure.azure import check_dynamic_azure_params +from litellm.native.chat_providers.handlers import ( + azure_anthropic_chat_completions, + azure_chat_completions, + azure_o1_chat_completions, + azure_text_completions, + base_llm_http_handler, +) +from litellm.native.chat_providers.support import ( + dispatch_client_azure, + dispatch_client_http, + get_encoding, + print_verbose, + provider_config_items, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult +from litellm.utils import get_secret + + +def complete_azure(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + _azure_detection_model: Final = ctx._azure_detection_model + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + api_version = ctx.api_version + client: Final = dispatch_client_azure(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + extra_headers: Final = ctx.extra_headers + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + max_retries: Final = ctx.max_retries + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + timeout: Final = ctx.timeout + + dynamic_params = False + if client is not None and (isinstance(client, openai.AzureOpenAI) or isinstance(client, openai.AsyncAzureOpenAI)): + dynamic_params = check_dynamic_azure_params( + azure_client_params={"api_version": api_version}, + azure_client=client, + ) + + api_type: Final = get_secret("AZURE_API_TYPE") or "azure" + + api_base = api_base or litellm.api_base or get_secret("AZURE_API_BASE") + + api_version = ( + api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") or litellm.AZURE_DEFAULT_API_VERSION + ) + + api_key = ( + api_key + or litellm.api_key + or litellm.azure_key + or get_secret_str("AZURE_OPENAI_API_KEY") + or get_secret_str("AZURE_API_KEY") + ) + + azure_ad_token: Final = optional_params.get("extra_body", {}).pop("azure_ad_token", None) or get_secret_str( + "AZURE_AD_TOKEN" + ) + + azure_ad_token_provider_value: Final = litellm_params.get("azure_ad_token_provider", None) + azure_ad_token_provider: Final = azure_ad_token_provider_value if callable(azure_ad_token_provider_value) else None + + headers = headers or litellm.headers + + if extra_headers is not None: + optional_params["extra_headers"] = extra_headers + if max_retries is not None: + optional_params["max_retries"] = max_retries + + if litellm.AzureOpenAIO1Config().is_o_series_model(model=_azure_detection_model): + ## LOAD CONFIG - if set + config = litellm.AzureOpenAIO1Config.get_config() + for k, v in provider_config_items(config): + if ( + k not in optional_params + ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + + response = azure_o1_chat_completions.completion( + model=model, + messages=messages, + headers=headers, + api_key=api_key, + api_base=api_base, + api_version=api_version, + dynamic_params=dynamic_params, + azure_ad_token=azure_ad_token, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + logging_obj=logging, + acompletion=acompletion, + timeout=timeout, + client=client, # pass AsyncAzureOpenAI, AzureOpenAI client + custom_llm_provider=custom_llm_provider, + ) + else: + ## LOAD CONFIG - if set + config = litellm.AzureOpenAIConfig.get_config() + for k, v in provider_config_items(config): + if ( + k not in optional_params + ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + + ## COMPLETION CALL + response = azure_chat_completions.completion( + model=model, + messages=messages, + headers=headers, + api_key=api_key, + api_base=api_base, + api_version=api_version, + api_type=api_type, + dynamic_params=dynamic_params, + azure_ad_token=azure_ad_token, + azure_ad_token_provider=azure_ad_token_provider, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + logging_obj=logging, + acompletion=acompletion, + timeout=timeout, + client=client, # pass AsyncAzureOpenAI, AzureOpenAI client + ) + + if optional_params.get("stream", False): + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + additional_args={ + "headers": headers, + "api_version": api_version, + "api_base": api_base, + }, + ) + + return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + + +def complete_azure_text(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + api_version = ctx.api_version + client: Final = dispatch_client_azure(ctx) + extra_headers: Final = ctx.extra_headers + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + timeout: Final = ctx.timeout + + api_type: Final = get_secret_str("AZURE_API_TYPE") or "azure" + + api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") + + if api_base is None: + raise ValueError( + "api_base is required for Azure OpenAI LLM provider. Either set it dynamically or set the AZURE_API_BASE environment variable." + ) + + api_version = api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") + + api_key = ( + api_key + or litellm.api_key + or litellm.azure_key + or get_secret_str("AZURE_OPENAI_API_KEY") + or get_secret_str("AZURE_API_KEY") + ) + + azure_ad_token: Final = optional_params.get("extra_body", {}).pop("azure_ad_token", None) or get_secret_str( + "AZURE_AD_TOKEN" + ) + + azure_ad_token_provider_value: Final = litellm_params.get("azure_ad_token_provider", None) + azure_ad_token_provider: Final = azure_ad_token_provider_value if callable(azure_ad_token_provider_value) else None + + headers = headers or litellm.headers + + if extra_headers is not None: + optional_params["extra_headers"] = extra_headers + + ## LOAD CONFIG - if set + config: Final = litellm.AzureOpenAIConfig.get_config() + for k, v in provider_config_items(config): + if ( + k not in optional_params + ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + + ## COMPLETION CALL + response: Final = azure_text_completions.completion( + model=model, + messages=messages, + headers=headers, + api_key=api_key, + api_base=api_base, + api_version=cast(str, api_version), + api_type=api_type, + azure_ad_token=azure_ad_token, + azure_ad_token_provider=azure_ad_token_provider, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + logging_obj=logging, + acompletion=acompletion, + timeout=timeout, + client=client, # pass AsyncAzureOpenAI, AzureOpenAI client + ) + + if optional_params.get("stream", False) or acompletion is True: + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + additional_args={ + "headers": headers, + "api_version": api_version, + "api_base": api_base, + }, + ) + + return response + + +def complete_azure_ai(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + extra_headers: Final = ctx.extra_headers + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo + + azure_ai_route: Final = AzureFoundryModelInfo.get_azure_ai_route(model) + + # Check if this is an agents route - model format: azure_ai/agents/ + if azure_ai_route == "agents": + from litellm.llms.azure_ai.agents import AzureAIAgentsConfig + + api_base = AzureFoundryModelInfo.get_api_base(api_base) + if api_base is None: + raise ValueError( + "Azure AI Agents requests require an api_base. Set `api_base` or the AZURE_AI_API_BASE env var." + ) + api_key = AzureFoundryModelInfo.get_api_key(api_key) + + response = AzureAIAgentsConfig.completion( + model=model, + messages=messages, + api_base=api_base, + api_key=api_key, + model_response=model_response, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, + acompletion=acompletion, + stream=stream, + headers=headers or litellm.headers, + ) + + # Check if this is a Claude model - route to Azure Anthropic handler + elif "claude" in model.lower(): + # Use Azure Anthropic handler for Claude models + api_base = AzureFoundryModelInfo.get_api_base(api_base) + if api_base is None: + raise ValueError( + "Azure Anthropic requests require an api_base. Set `api_base` or the AZURE_AI_API_BASE env var." + ) + api_key = AzureFoundryModelInfo.get_api_key(api_key) + + # Ensure the URL ends with /v1/messages for Anthropic + if api_base: + api_base = api_base.rstrip("/") + if not api_base.endswith("/v1/messages"): + if "/anthropic" in api_base: + parts: Final = api_base.split("/anthropic", 1) + api_base = parts[0] + "/anthropic" + else: + api_base = api_base + "/anthropic" + api_base = api_base + "/v1/messages" + + response = azure_anthropic_chat_completions.completion( + model=model, + messages=messages, + api_base=api_base, + acompletion=acompletion, + custom_prompt_dict=litellm.custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, + headers=headers, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + ) + if optional_params.get("stream", False) or acompletion is True: + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + ) + response = response + else: + # Non-Claude models use standard Azure AI flow + api_base = AzureFoundryModelInfo.get_api_base(api_base) + # set API KEY + api_key = AzureFoundryModelInfo.get_api_key(api_key) + + headers = headers or litellm.headers + + if extra_headers is not None: + optional_params["extra_headers"] = extra_headers + + ## FOR COHERE + if "command-r" in model: # make sure tool call in messages are str + messages = stringify_json_tool_call_content(messages=messages) + + ## COMPLETION CALL + try: + response = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, + client=client, # pass AsyncOpenAI, OpenAI client + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + ) + except Exception as e: + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + + if optional_params.get("stream", False): + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + additional_args={"headers": headers}, + ) + + return response diff --git a/litellm/native/chat_providers/custom.py b/litellm/native/chat_providers/custom.py new file mode 100644 index 00000000000..f4bdd09d9c6 --- /dev/null +++ b/litellm/native/chat_providers/custom.py @@ -0,0 +1,147 @@ +import time +from typing import Final + +import httpx + +import litellm +from litellm.exceptions import LiteLLMUnknownProvider +from litellm.llms.custom_llm import CustomLLM, custom_chat_llm_router +from litellm.native.chat_providers.support import get_encoding, print_verbose +from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult +from litellm.utils import CustomStreamWrapper + + +def _custom_api_first_output(resp: httpx.Response | None) -> str: + return resp.json()["data"][0]["output"][0] + + +def complete_custom(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + api_base: Final = ctx.api_base + headers: Final = ctx.headers + kwargs: Final = ctx.kwargs + max_tokens: Final = ctx.max_tokens + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + temperature: Final = ctx.temperature + top_p: Final = ctx.top_p + + url: Final = litellm.api_base or api_base or "" + if url is None or url == "": + raise ValueError("api_base not set. Set api_base or litellm.api_base for custom endpoints") + + """ + assume input to custom LLM api bases follow this format: + resp = litellm.module_level_client.post( + api_base, + json={ + 'model': 'meta-llama/Llama-2-13b-hf', # model name + 'params': { + 'prompt': ["The capital of France is P"], + 'max_tokens': 32, + 'temperature': 0.7, + 'top_p': 1.0, + 'top_k': 40, + } + } + ) + + """ + prompt: Final = " ".join([message["content"] for message in messages]) + resp: Final = litellm.module_level_client.post( + url, + headers=headers, + json={ + "model": model, + "params": { + "prompt": [prompt], + "max_tokens": max_tokens, + "temperature": temperature, + "top_p": top_p, + "top_k": kwargs.get("top_k"), + }, + **kwargs.get("extra_body", {}), + }, + ) + """ + assume all responses from custom api_bases of this format: + { + 'data': [ + { + 'prompt': 'The capital of France is P', + 'output': ['The capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France'], + 'params': {'temperature': 0.7, 'top_k': 40, 'top_p': 1}}], + 'message': 'ok' + } + ] + } + """ + string_response: Final = _custom_api_first_output(resp) + ## RESPONSE OBJECT + model_response.choices[0].message.content = string_response + model_response.created = int(time.time()) + model_response.model = model + return model_response + + +def complete_custom_providers( + ctx: CompletionDispatchContext, +) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key: Final = ctx.api_key + client: Final = ctx.client + custom_llm_provider: Final = ctx.custom_llm_provider + custom_prompt_dict: Final = ctx.custom_prompt_dict + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + custom_handler: CustomLLM | None = None + for item in litellm.custom_provider_map: + if item["provider"] == custom_llm_provider: + custom_handler = item["custom_handler"] + + if custom_handler is None: + raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider) + + ## ROUTE LLM CALL ## + handler_fn: Final = custom_chat_llm_router(async_fn=acompletion, stream=stream, custom_llm=custom_handler) + + headers = headers or litellm.headers or {} + + ## CALL FUNCTION + response: Final = handler_fn( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + print_verbose=print_verbose, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + timeout=timeout, + custom_prompt_dict=custom_prompt_dict, + client=client, # pass AsyncOpenAI, OpenAI client + encoding=get_encoding(), + ) + if stream is True: + return CustomStreamWrapper( + completion_stream=response, + model=model, + custom_llm_provider=custom_llm_provider, + logging_obj=logging, + ) + + return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract diff --git a/litellm/native/chat_providers/gateways.py b/litellm/native/chat_providers/gateways.py new file mode 100644 index 00000000000..3cde7cf206c --- /dev/null +++ b/litellm/native/chat_providers/gateways.py @@ -0,0 +1,299 @@ +from typing import Final + +import openai + +import litellm +from litellm.constants import NADIR_DEFAULT_API_BASE +from litellm.native.chat_providers.handlers import base_llm_http_handler +from litellm.native.chat_providers.support import dispatch_client_http, get_encoding, provider_config_items +from litellm.secret_managers.main import get_secret_str +from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult +from litellm.utils import get_secret + + +def complete_cometapi(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + provider_config: Final = ctx.provider_config + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_key = api_key or litellm.cometapi_key or get_secret_str("COMETAPI_KEY") or litellm.api_key + + api_base = api_base or litellm.api_base or get_secret_str("COMETAPI_API_BASE") or "https://api.cometapi.com/v1" + + ## COMPLETION CALL + response: Final = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + provider_config=provider_config, + ) + + ## LOGGING + logging.post_call(input=messages, api_key=api_key, original_response=response) + + return response + + +def complete_openrouter(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_base = api_base or litellm.api_base or get_secret_str("OPENROUTER_API_BASE") or "https://openrouter.ai/api/v1" + + api_key = ( + api_key + or litellm.api_key + or litellm.openrouter_key + or get_secret_str("OPENROUTER_API_KEY") + or get_secret_str("OR_API_KEY") + ) + + openrouter_site_url: Final = get_secret("OR_SITE_URL") or "https://litellm.ai" + openrouter_app_name: Final = get_secret("OR_APP_NAME") or "liteLLM" + + openrouter_headers: Final = { + "HTTP-Referer": openrouter_site_url, + "X-Title": openrouter_app_name, + } + + _headers: Final = headers or litellm.headers + if _headers: + openrouter_headers.update(_headers) + + headers = openrouter_headers + + ## Load Config + config: Final = litellm.OpenrouterConfig.get_config() + for k, v in provider_config_items(config): + if k == "extra_body": + # we use openai 'extra_body' to pass openrouter specific params - transforms, route, models + if "extra_body" in optional_params: + optional_params[k].update(v) + else: + optional_params[k] = v + elif k not in optional_params: + optional_params[k] = v + + ## COMPLETION CALL + response: Final = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="openrouter", + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) + ## LOGGING + logging.post_call(input=messages, api_key=openai.api_key, original_response=response) + + return response + + +def complete_nadir(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + api_base: Final = ctx.api_base or litellm.api_base or get_secret_str("NADIR_API_BASE") or NADIR_DEFAULT_API_BASE + api_key: Final = ctx.api_key + + response: Final = base_llm_http_handler.completion( + model=ctx.model, + stream=ctx.stream, + messages=ctx.messages, + acompletion=ctx.acompletion, + api_base=api_base, + model_response=ctx.model_response, + optional_params=ctx.optional_params, + litellm_params=ctx.litellm_params, + shared_session=ctx.shared_session, + custom_llm_provider="nadir", + timeout=ctx.timeout, + headers=ctx.headers or litellm.headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=ctx.logging, + client=ctx.client, + ) + ctx.logging.post_call(input=ctx.messages, api_key=api_key, original_response=response) + + return response + + +def complete_vercel_ai_gateway( + ctx: CompletionDispatchContext, +) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_base = ( + api_base + or litellm.api_base + or get_secret_str("VERCEL_AI_GATEWAY_API_BASE") + or "https://ai-gateway.vercel.sh/v1" + ) + + api_key = api_key or litellm.api_key or get_secret("VERCEL_AI_GATEWAY_API_KEY") + + vercel_site_url: Final = get_secret("VERCEL_SITE_URL") or "https://litellm.ai" + vercel_app_name: Final = get_secret("VERCEL_APP_NAME") or "liteLLM" + + vercel_headers: Final = { + "http-referer": vercel_site_url, + "x-title": vercel_app_name, + } + + _headers: Final = headers or litellm.headers + if _headers: + vercel_headers.update(_headers) + + headers = vercel_headers + + ## Load Config + config: Final = litellm.VercelAIGatewayConfig.get_config() + for k, v in provider_config_items(config): + if k == "extra_body": + # we use openai 'extra_body' to pass vercel specific params - providerOptions + if "extra_body" in optional_params: + optional_params[k].update(v) + else: + optional_params[k] = v + elif k not in optional_params: + optional_params[k] = v + + ## COMPLETION CALL + response: Final = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="vercel_ai_gateway", + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) + ## LOGGING + logging.post_call(input=messages, api_key=openai.api_key, original_response=response) + + return response + + +def complete_edenai(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + api_base: Final = litellm.EdenAIChatConfig.get_api_base(ctx.api_base) + api_key: Final = litellm.EdenAIChatConfig.get_api_key(ctx.api_key or litellm.api_key) + response: Final = base_llm_http_handler.completion( + model=ctx.model, + messages=ctx.messages, + api_base=api_base, + custom_llm_provider="edenai", + model_response=ctx.model_response, + encoding=get_encoding(), + logging_obj=ctx.logging, + optional_params=ctx.optional_params, + timeout=ctx.timeout, + litellm_params=ctx.litellm_params, + shared_session=ctx.shared_session, + acompletion=ctx.acompletion, + stream=ctx.stream, + api_key=api_key, + headers=ctx.headers or litellm.headers, + client=dispatch_client_http(ctx), + provider_config=ctx.provider_config, + ) + ctx.logging.post_call(input=ctx.messages, api_key=api_key, original_response=response) + return response + + +def complete_fal_ai(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + if ctx.stream: + raise litellm.FalAIError( + status_code=400, + message="fal_ai chat completions do not support streaming", + ) + api_base: Final = litellm.FalAIChatConfig.get_api_base(ctx.api_base) + api_key: Final = litellm.FalAIChatConfig.get_api_key(ctx.api_key or litellm.api_key) + response: Final = base_llm_http_handler.completion( + model=ctx.model, + messages=ctx.messages, + api_base=api_base, + custom_llm_provider="fal_ai", + model_response=ctx.model_response, + encoding=get_encoding(), + logging_obj=ctx.logging, + optional_params=ctx.optional_params, + timeout=ctx.timeout, + litellm_params=ctx.litellm_params, + shared_session=ctx.shared_session, + acompletion=ctx.acompletion, + stream=ctx.stream, + api_key=api_key, + headers=ctx.headers or litellm.headers, + client=dispatch_client_http(ctx), + provider_config=ctx.provider_config, + ) + ctx.logging.post_call(input=ctx.messages, api_key=api_key, original_response=response) + return response diff --git a/litellm/native/chat_providers/google.py b/litellm/native/chat_providers/google.py new file mode 100644 index 00000000000..edb1924be08 --- /dev/null +++ b/litellm/native/chat_providers/google.py @@ -0,0 +1,311 @@ +from typing import Final + +import litellm +from litellm.litellm_core_utils.core_helpers import safe_deep_copy +from litellm.llms.gemini.common_utils import get_api_key_from_env +from litellm.llms.vertex_ai import vertex_ai_non_gemini +from litellm.llms.vertex_ai.common_utils import VertexAIModelRoute, get_vertex_ai_model_route +from litellm.native.chat_providers.handlers import ( + base_llm_http_handler, + gdc_transformation, + vertex_chat_completion, + vertex_gemma_chat_completion, + vertex_model_garden_chat_completion, + vertex_partner_models_chat_completion, +) +from litellm.native.chat_providers.support import dispatch_client_http, get_encoding, print_verbose +from litellm.secret_managers.main import get_secret_str +from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult +from litellm.utils import CustomStreamWrapper, get_secret + + +def complete_vertex_ai_beta( + ctx: CompletionDispatchContext, +) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key: Final = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + timeout: Final = ctx.timeout + + vertex_ai_project: Final = ( + optional_params.pop("vertex_project", None) + or optional_params.pop("vertex_ai_project", None) + or litellm.vertex_project + or get_secret("VERTEXAI_PROJECT") + ) + vertex_ai_location: Final = ( + optional_params.pop("vertex_location", None) + or optional_params.pop("vertex_ai_location", None) + or litellm.vertex_location + or get_secret("VERTEXAI_LOCATION") + ) + vertex_credentials: Final = ( + optional_params.pop("vertex_credentials", None) + or optional_params.pop("vertex_ai_credentials", None) + or get_secret("VERTEXAI_CREDENTIALS") + ) + + gemini_api_key: Final = ( + api_key + or get_api_key_from_env() + or get_secret("PALM_API_KEY") # older palm api key should also work + or litellm.api_key + ) + + api_base = api_base or litellm.api_base or get_secret("GEMINI_API_BASE") + new_params: Final = safe_deep_copy(optional_params or {}) + return vertex_chat_completion.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=new_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=get_encoding(), + vertex_location=vertex_ai_location, + vertex_project=vertex_ai_project, + vertex_credentials=vertex_credentials, + gemini_api_key=gemini_api_key, + logging_obj=logging, + acompletion=acompletion, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + client=client, + api_base=api_base, + extra_headers=headers, + ) + + +def complete_vertex_ai(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + custom_prompt_dict: Final = ctx.custom_prompt_dict + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response = ctx.model_response + optional_params: Final = ctx.optional_params + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + vertex_ai_project: Final = ( + optional_params.pop("vertex_project", None) + or optional_params.pop("vertex_ai_project", None) + or litellm.vertex_project + or get_secret("VERTEXAI_PROJECT") + ) + vertex_ai_location: Final = ( + optional_params.pop("vertex_location", None) + or optional_params.pop("vertex_ai_location", None) + or litellm.vertex_location + or get_secret("VERTEXAI_LOCATION") + ) + vertex_credentials: Final = ( + optional_params.pop("vertex_credentials", None) + or optional_params.pop("vertex_ai_credentials", None) + or get_secret("VERTEXAI_CREDENTIALS") + ) + + api_base = api_base or litellm.api_base or get_secret("VERTEXAI_API_BASE") + + new_params: Final = safe_deep_copy(optional_params or {}) + model_route: Final = get_vertex_ai_model_route(model=model, litellm_params=litellm_params) + + if model_route == VertexAIModelRoute.PARTNER_MODELS: + model_response = vertex_partner_models_chat_completion.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=new_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=get_encoding(), + api_base=api_base, + vertex_location=vertex_ai_location, + vertex_project=vertex_ai_project, + vertex_credentials=vertex_credentials, + logging_obj=logging, + acompletion=acompletion, + headers=headers, + custom_prompt_dict=custom_prompt_dict, + timeout=timeout, + client=client, + ) + elif model_route == VertexAIModelRoute.GEMINI: + model_response = vertex_chat_completion.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=new_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=get_encoding(), + vertex_location=vertex_ai_location, + vertex_project=vertex_ai_project, + vertex_credentials=vertex_credentials, + gemini_api_key=None, + logging_obj=logging, + acompletion=acompletion, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + client=client, + api_base=api_base, + extra_headers=headers, + ) + elif model_route == VertexAIModelRoute.GEMMA: + # Vertex Gemma Models with custom prediction endpoint + model_response = vertex_gemma_chat_completion.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=new_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=get_encoding(), + api_base=api_base, + vertex_location=vertex_ai_location, + vertex_project=vertex_ai_project, + vertex_credentials=vertex_credentials, + logging_obj=logging, + acompletion=acompletion, + headers=headers, + custom_prompt_dict=custom_prompt_dict, + timeout=timeout, + client=client, + ) + elif model_route == VertexAIModelRoute.MODEL_GARDEN: + # Vertex Model Garden - OpenAI compatible models + model_response = vertex_model_garden_chat_completion.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=new_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=get_encoding(), + api_base=api_base, + vertex_location=vertex_ai_location, + vertex_project=vertex_ai_project, + vertex_credentials=vertex_credentials, + logging_obj=logging, + acompletion=acompletion, + headers=headers, + custom_prompt_dict=custom_prompt_dict, + timeout=timeout, + client=client, + ) + elif model_route == VertexAIModelRoute.AGENT_ENGINE: + # Vertex AI Agent Engine (Reasoning Engines) + from litellm.llms.vertex_ai.agent_engine.transformation import ( + VertexAgentEngineConfig, + ) + + vertex_agent_engine_config: Final = VertexAgentEngineConfig() + + # Update litellm_params with vertex credentials + litellm_params["vertex_project"] = vertex_ai_project + litellm_params["vertex_location"] = vertex_ai_location + litellm_params["vertex_credentials"] = vertex_credentials + + model_response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + model_response=model_response, + optional_params=new_params, + litellm_params=litellm_params, + encoding=get_encoding(), + api_key=None, + api_base=api_base, + logging_obj=logging, + acompletion=acompletion, + timeout=timeout, + client=client, + custom_llm_provider="vertex_ai", + provider_config=vertex_agent_engine_config, + headers=headers or {}, + ) + else: # VertexAIModelRoute.NON_GEMINI + model_response = vertex_ai_non_gemini.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=new_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=get_encoding(), + vertex_location=vertex_ai_location, + vertex_project=vertex_ai_project, + vertex_credentials=vertex_credentials, + logging_obj=logging, + acompletion=acompletion, + ) + + if "stream" in optional_params and optional_params["stream"] is True and acompletion is False: + return CustomStreamWrapper( + model_response, + model, + custom_llm_provider="vertex_ai", + logging_obj=logging, + ) + return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + + +def complete_gdc(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_key = api_key or litellm.gdc_key or get_secret_str("GDC_API_KEY") or litellm.api_key + api_base = api_base or litellm.gdc_api_base or get_secret_str("GDC_API_BASE") or litellm.api_base + + return base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + provider_config=gdc_transformation, + ) diff --git a/litellm/native/chat_providers/handlers.py b/litellm/native/chat_providers/handlers.py new file mode 100644 index 00000000000..773def0c8e0 --- /dev/null +++ b/litellm/native/chat_providers/handlers.py @@ -0,0 +1,51 @@ +from typing import Final + +from litellm.llms.anthropic.chat import AnthropicChatCompletion +from litellm.llms.azure.azure import AzureChatCompletion +from litellm.llms.azure.chat.o_series_handler import AzureOpenAIO1ChatCompletion +from litellm.llms.azure.completion.handler import AzureTextCompletion +from litellm.llms.azure_ai.anthropic.handler import AzureAnthropicChatCompletion +from litellm.llms.bedrock.chat import BedrockConverseLLM +from litellm.llms.bytez.chat.transformation import BytezChatConfig +from litellm.llms.codestral.completion.handler import CodestralTextCompletion +from litellm.llms.custom_httpx.aiohttp_handler import BaseLLMAIOHTTPHandler +from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.llms.gdc.chat.transformation import GDCGeminiConfig +from litellm.llms.lemonade.chat.transformation import LemonadeChatConfig +from litellm.llms.openai.completion.handler import OpenAITextCompletion +from litellm.llms.openai.openai import OpenAIChatCompletion +from litellm.llms.openai_like.chat.handler import OpenAILikeChatHandler +from litellm.llms.ovhcloud.chat.transformation import OVHCloudChatConfig +from litellm.llms.predibase.chat.handler import PredibaseChatCompletion +from litellm.llms.sagemaker.completion.handler import SagemakerLLM +from litellm.llms.sap.chat.handler import GenAIHubOrchestration +from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM +from litellm.llms.vertex_ai.vertex_ai_partner_models.main import VertexAIPartnerModels +from litellm.llms.vertex_ai.vertex_gemma_models.main import VertexAIGemmaModels +from litellm.llms.vertex_ai.vertex_model_garden.main import VertexAIModelGardenModels +from litellm.llms.watsonx.chat.handler import WatsonXChatHandler + +openai_chat_completions = OpenAIChatCompletion() +openai_text_completions: Final = OpenAITextCompletion() +sap_gen_ai_hub_chat_completions: Final = GenAIHubOrchestration() +anthropic_chat_completions: Final = AnthropicChatCompletion() +azure_anthropic_chat_completions: Final = AzureAnthropicChatCompletion() +azure_chat_completions = AzureChatCompletion() +azure_o1_chat_completions: Final = AzureOpenAIO1ChatCompletion() +azure_text_completions: Final = AzureTextCompletion() +predibase_chat_completions: Final = PredibaseChatCompletion() +codestral_text_completions: Final = CodestralTextCompletion() +bedrock_converse_chat_completion: Final = BedrockConverseLLM() +vertex_chat_completion: Final = VertexLLM() +vertex_partner_models_chat_completion: Final = VertexAIPartnerModels() +vertex_gemma_chat_completion: Final = VertexAIGemmaModels() +vertex_model_garden_chat_completion: Final = VertexAIModelGardenModels() +gdc_transformation: Final = GDCGeminiConfig() +sagemaker_llm: Final = SagemakerLLM() +watsonx_chat_completion: Final = WatsonXChatHandler() +openai_like_chat_completion: Final = OpenAILikeChatHandler() +base_llm_http_handler = BaseLLMHTTPHandler() +base_llm_aiohttp_handler = BaseLLMAIOHTTPHandler() +bytez_transformation: Final = BytezChatConfig() +ovhcloud_transformation: Final = OVHCloudChatConfig() +lemonade_transformation: Final = LemonadeChatConfig() diff --git a/litellm/native/chat_providers/model_labs.py b/litellm/native/chat_providers/model_labs.py new file mode 100644 index 00000000000..36a3a2a975a --- /dev/null +++ b/litellm/native/chat_providers/model_labs.py @@ -0,0 +1,905 @@ +import os +from types import MappingProxyType +from typing import Final, cast # noqa: TID251 # casts moved verbatim from litellm/main.py + +import httpx + +import litellm +from litellm._logging import verbose_logger +from litellm.llms.cohere.common_utils import CohereModelInfo +from litellm.llms.deprecated_providers import aleph_alpha +from litellm.llms.nlp_cloud.chat.handler import completion as nlp_cloud_chat_completion +from litellm.llms.replicate.chat.handler import completion as replicate_chat_completion +from litellm.native.chat_providers.handlers import ( + base_llm_http_handler, + bytez_transformation, + codestral_text_completions, + openai_like_chat_completion, + predibase_chat_completions, +) +from litellm.native.chat_providers.support import ( + dispatch_client_http, + get_encoding, + print_verbose, + provider_config_items, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult +from litellm.utils import CustomStreamWrapper, get_secret + + +def complete_deepseek(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key: Final = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + provider_config: Final = ctx.provider_config + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + try: + response: Final = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + provider_config=provider_config, + ) + except Exception as e: + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + + return response + + +def complete_fireworks_ai( + ctx: CompletionDispatchContext, +) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key: Final = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + provider_config: Final = ctx.provider_config + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + optional_params: Final = ( + provider_config.map_extra_body_params(optional_params=ctx.optional_params, model=model) + if isinstance(provider_config, litellm.FireworksAIConfig) + else ctx.optional_params + ) + + try: + response: Final = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + provider_config=provider_config, + ) + except Exception as e: + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + + return response + + +def complete_together_ai(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key: Final = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + provider_config: Final = ctx.provider_config + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + try: + response: Final = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + provider_config=provider_config, + ) + except Exception as e: + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args=MappingProxyType({"headers": headers}), + ) + raise + + return response + + +def complete_scaledown(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + try: + return base_llm_http_handler.completion( + model=ctx.model, + messages=ctx.messages, + headers=ctx.headers, + model_response=ctx.model_response, + api_key=ctx.api_key, + api_base=ctx.api_base, + acompletion=ctx.acompletion, + logging_obj=ctx.logging, + optional_params=ctx.optional_params, + litellm_params=ctx.litellm_params, + shared_session=ctx.shared_session, + timeout=cast( # cast-ok: CompletionTimeout.resolve normalizes this value + float | httpx.Timeout, ctx.timeout + ), + client=dispatch_client_http(ctx), + custom_llm_provider=ctx.custom_llm_provider, + encoding=get_encoding(), + stream=ctx.stream, + provider_config=ctx.provider_config, + ) + except Exception as exc: + ctx.logging.post_call( + input=ctx.messages, + api_key=ctx.api_key, + original_response=str(exc), + additional_args={"headers": ctx.headers}, + ) + raise + + +def complete_xai(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key: Final = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + provider_config: Final = ctx.provider_config + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + try: + response: Final = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + provider_config=provider_config, + ) + except Exception as e: + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + + return response + + +def complete_groq(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_base = ( + api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there + or litellm.api_base + or get_secret("GROQ_API_BASE") + or "https://api.groq.com/openai/v1" + ) + + # set API KEY + api_key = ( + api_key + or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there + or litellm.groq_key + or get_secret("GROQ_API_KEY") + ) + + headers = headers or litellm.headers + + ## LOAD CONFIG - if set + config: Final = litellm.GroqChatConfig.get_config() + for k, v in provider_config_items(config): + if ( + k not in optional_params + ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) + + +def complete_minimax(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + provider_config: Final = ctx.provider_config + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_key = api_key or get_secret_str("MINIMAX_API_KEY") or litellm.api_key + + api_base = api_base or litellm.api_base or get_secret_str("MINIMAX_API_BASE") or "https://api.minimax.io/v1" + + response: Final = base_llm_http_handler.completion( + model=model, + messages=messages, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + model_response=model_response, + encoding=get_encoding(), + logging_obj=logging, + optional_params=optional_params, + timeout=timeout, + litellm_params=litellm_params, + shared_session=shared_session, + acompletion=acompletion, + stream=stream, + api_key=api_key, + headers=headers, + client=client, + provider_config=provider_config, + ) + logging.post_call(input=messages, api_key=api_key, original_response=response) + + return response + + +def complete_mistral(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + provider_config: Final = ctx.provider_config + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_key = api_key or litellm.api_key or get_secret("MISTRAL_API_KEY") + api_base = api_base or litellm.api_base or get_secret("MISTRAL_API_BASE") or "https://api.mistral.ai/v1" + + return base_llm_http_handler.completion( + model=model, + messages=messages, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + model_response=model_response, + encoding=get_encoding(), + logging_obj=logging, + optional_params=optional_params, + timeout=timeout, + litellm_params=litellm_params, + shared_session=shared_session, + acompletion=acompletion, + stream=stream, + api_key=api_key, + headers=headers, + client=client, + provider_config=provider_config, + ) + + +def complete_replicate(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key: Final = ctx.api_key + custom_prompt_dict = ctx.custom_prompt_dict + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response = ctx.model_response + optional_params: Final = ctx.optional_params + + replicate_key: Final = ( + api_key + or litellm.replicate_key + or litellm.api_key + or get_secret("REPLICATE_API_KEY") + or get_secret("REPLICATE_API_TOKEN") + ) + + api_base = api_base or litellm.api_base or get_secret("REPLICATE_API_BASE") or "https://api.replicate.com/v1" + + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + + model_response = replicate_chat_completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=get_encoding(), # for calculating input/output tokens + api_key=replicate_key, + logging_obj=logging, + custom_prompt_dict=custom_prompt_dict, + acompletion=acompletion, + headers=headers, + ) + + if optional_params.get("stream", False) is True: + ## LOGGING + logging.post_call( + input=messages, + api_key=replicate_key, + original_response=model_response, + ) + + return model_response + + +def complete_nlp_cloud(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key: Final = ctx.api_key + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + + nlp_cloud_key: Final = api_key or litellm.nlp_cloud_key or get_secret("NLP_CLOUD_API_KEY") or litellm.api_key + + api_base = api_base or litellm.api_base or get_secret("NLP_CLOUD_API_BASE") or "https://api.nlpcloud.io/v1/gpu/" + + response = nlp_cloud_chat_completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=get_encoding(), + api_key=nlp_cloud_key, + logging_obj=logging, + ) + + if "stream" in optional_params and optional_params["stream"] is True: + # don't try to access stream object, + response = CustomStreamWrapper( + response, + model, + custom_llm_provider="nlp_cloud", + logging_obj=logging, + ) + + if optional_params.get("stream", False) or acompletion is True: + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + ) + + return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + + +def complete_aleph_alpha(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + api_base = ctx.api_base + api_key: Final = ctx.api_key + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response = ctx.model_response + optional_params: Final = ctx.optional_params + + aleph_alpha_key: Final = ( + api_key + or litellm.aleph_alpha_key + or get_secret("ALEPH_ALPHA_API_KEY") + or get_secret("ALEPHALPHA_API_KEY") + or litellm.api_key + ) + + api_base = ( + api_base or litellm.api_base or get_secret("ALEPH_ALPHA_API_BASE") or "https://api.aleph-alpha.com/complete" + ) + + model_response = aleph_alpha.completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=get_encoding(), + default_max_tokens_to_sample=litellm.max_tokens, + api_key=aleph_alpha_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + ) + + if "stream" in optional_params and optional_params["stream"] is True: + # don't try to access stream object, + return CustomStreamWrapper( + model_response, + model, + custom_llm_provider="aleph_alpha", + logging_obj=logging, + ) + return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + + +def complete_cohere_chat(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key: Final = ctx.api_key + extra_headers: Final = ctx.extra_headers + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + provider_config: Final = ctx.provider_config + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + cohere_key: Final = ( + api_key + or litellm.cohere_key + or get_secret_str("COHERE_API_KEY") + or get_secret_str("CO_API_KEY") + or litellm.api_key + ) + + cohere_route: Final = CohereModelInfo.get_cohere_route(model) + verbose_logger.debug("Cohere route: %s", cohere_route) + # Set API base based on route + if cohere_route == "v2": + api_base = api_base or litellm.api_base or get_secret_str("COHERE_API_BASE") or "https://api.cohere.com/v2/chat" + # Remove v2/ prefix from model name for the actual API call + if "v2/" in model: + model = model.replace("v2/", "") + else: + api_base = api_base or litellm.api_base or get_secret_str("COHERE_API_BASE") or "https://api.cohere.ai/v1/chat" + + headers = headers or litellm.headers or {} + if headers is None: + headers = {} + + if extra_headers is not None: + headers.update(extra_headers) + + verbose_logger.debug("Model: %s, API Base: %s", model, api_base) + verbose_logger.debug("Provider Config: %s", provider_config) + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="cohere_chat", + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=cohere_key, + provider_config=provider_config, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + ) + + +def complete_maritalk(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + api_base = ctx.api_base + api_key: Final = ctx.api_key + custom_prompt_dict: Final = ctx.custom_prompt_dict + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + + maritalk_key: Final = api_key or litellm.maritalk_key or get_secret("MARITALK_API_KEY") or litellm.api_key + + api_base = api_base or litellm.api_base or get_secret("MARITALK_API_BASE") or "https://chat.maritaca.ai/api" + + return openai_like_chat_completion.completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=get_encoding(), + api_key=maritalk_key, + logging_obj=logging, + custom_llm_provider="maritalk", + custom_prompt_dict=custom_prompt_dict, + ) + + +def complete_huggingface(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key: Final = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + huggingface_key: Final = ( + api_key + or litellm.huggingface_key + or os.environ.get("HF_TOKEN") + or os.environ.get("HUGGINGFACE_API_KEY") + or litellm.api_key + ) + hf_headers: Final = headers or litellm.headers + return base_llm_http_handler.completion( + model=model, + messages=messages, + headers=hf_headers, + model_response=model_response, + api_key=huggingface_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + ) + + +def complete_compactifai(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + provider_config: Final = ctx.provider_config + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_key = api_key or get_secret_str("COMPACTIFAI_API_KEY") or litellm.api_key + + api_base = api_base or "https://api.compactif.ai/v1" + + ## COMPLETION CALL + return base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + provider_config=provider_config, + ) + + +def complete_predibase(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + custom_prompt_dict: Final = ctx.custom_prompt_dict + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + timeout: Final = ctx.timeout + + tenant_id: Final = ( + optional_params.pop("tenant_id", None) + or optional_params.pop("predibase_tenant_id", None) + or litellm.predibase_tenant_id + or get_secret("PREDIBASE_TENANT_ID") + ) + + if tenant_id is None: + raise ValueError( + "Missing Predibase Tenant ID - Required for making the request. Set dynamically (e.g. `completion(..tenant_id=)`) or in env - `PREDIBASE_TENANT_ID`." + ) + + api_base = ( + api_base + or optional_params.pop("api_base", None) + or optional_params.pop("base_url", None) + or litellm.api_base + or get_secret("PREDIBASE_API_BASE") + ) + + api_key = api_key or litellm.api_key or litellm.predibase_key or get_secret("PREDIBASE_API_KEY") + + _model_response: Final = predibase_chat_completions.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=get_encoding(), + logging_obj=logging, + acompletion=acompletion, + api_base=api_base, + custom_prompt_dict=custom_prompt_dict, + api_key=api_key, + tenant_id=tenant_id, + timeout=timeout, + ) + + if "stream" in optional_params and optional_params["stream"] is True and acompletion is False: + return _model_response + return _model_response + + +def complete_text_completion_codestral( + ctx: CompletionDispatchContext, +) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + custom_prompt_dict: Final = ctx.custom_prompt_dict + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + optional_params: Final = ctx.optional_params + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_base = ( + api_base + or optional_params.pop("api_base", None) + or optional_params.pop("base_url", None) + or litellm.api_base + or "https://codestral.mistral.ai/v1/fim/completions" + ) + + api_key = api_key or litellm.api_key or get_secret("CODESTRAL_API_KEY") + + text_completion_model_response: Final = litellm.TextCompletionResponse(stream=stream) + + _model_response: Final = codestral_text_completions.completion( + model=model, + messages=messages, + model_response=text_completion_model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=get_encoding(), + logging_obj=logging, + acompletion=acompletion, + api_base=api_base, + custom_prompt_dict=custom_prompt_dict, + api_key=api_key, + timeout=timeout, + ) + + if "stream" in optional_params and optional_params["stream"] is True and acompletion is False: + return _model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + return _model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + + +def complete_bytez(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_key = api_key or litellm.bytez_key or get_secret_str("BYTEZ_API_KEY") or litellm.api_key + + response: Final = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + provider_config=bytez_transformation, + ) + + return response diff --git a/litellm/native/chat_providers/openai.py b/litellm/native/chat_providers/openai.py new file mode 100644 index 00000000000..94b8fa64278 --- /dev/null +++ b/litellm/native/chat_providers/openai.py @@ -0,0 +1,388 @@ +from typing import Final, cast # noqa: TID251 # casts moved verbatim from litellm/main.py + +import openai + +import litellm +from litellm._logging import verbose_logger +from litellm.llms.base_llm.chat.transformation import with_attribution_headers +from litellm.llms.custom_httpx.http_handler import http2_enabled +from litellm.native.chat_providers.handlers import ( + base_llm_aiohttp_handler, + base_llm_http_handler, + openai_chat_completions, + openai_text_completions, +) +from litellm.native.chat_providers.support import ( + dispatch_client_http, + dispatch_client_openai, + dispatch_metadata, + get_encoding, + print_verbose, + provider_config_items, +) +from litellm.responses.bridge_check import resolve_openai_api_base +from litellm.secret_managers.main import get_secret_bool, get_secret_str +from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult +from litellm.utils import get_requester_metadata, get_secret + + +def complete_text_completion_openai( + ctx: CompletionDispatchContext, +) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_openai(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + text_completion: Final = ctx.text_completion + timeout: Final = ctx.timeout + + openai.api_type = "openai" + + api_base = ( + api_base + or litellm.api_base + or get_secret("OPENAI_BASE_URL") + or get_secret("OPENAI_API_BASE") + or "https://api.openai.com/v1" + ) + + openai.api_version = None + # set API KEY + + api_key = api_key or litellm.api_key or litellm.openai_key or get_secret("OPENAI_API_KEY") + + headers = headers or litellm.headers + + ## LOAD CONFIG - if set + config: Final = litellm.OpenAITextCompletionConfig.get_config() + for k, v in provider_config_items(config): + if ( + k not in optional_params + ): # completion(top_k=3) > openai_text_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + if litellm.organization: + openai.organization = litellm.organization + + ## COMPLETION CALL + _response = openai_text_completions.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + print_verbose=print_verbose, + api_key=api_key, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + acompletion=acompletion, + client=client, # pass AsyncOpenAI, OpenAI client + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + timeout=timeout, + ) + + if optional_params.get("stream", False) is False and acompletion is False and text_completion is False: + # convert to chat completion response + _response = litellm.OpenAITextCompletionConfig().convert_to_chat_model_response_object( + response_object=_response, model_response_object=model_response + ) + + if optional_params.get("stream", False) or acompletion is True: + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=_response, + additional_args={"headers": headers}, + ) + return _response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + + +def complete_aiohttp_openai( + ctx: CompletionDispatchContext, +) -> CompletionDispatchResult: + if http2_enabled(): + verbose_logger.warning( + "litellm.http2 is enabled but aiohttp_openai/ always uses aiohttp, which has no HTTP/2 client; this request stays on HTTP/1.1" + ) + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + extra_headers: Final = ctx.extra_headers + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_base = ( + api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there + or litellm.api_base + or get_secret("OPENAI_BASE_URL") + or get_secret("OPENAI_API_BASE") + or "https://api.openai.com/v1" + ) + # set API KEY + api_key = ( + api_key + or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there + or litellm.openai_key + or get_secret("OPENAI_API_KEY") + ) + + headers = headers or litellm.headers + + if extra_headers is not None: + optional_params["extra_headers"] = extra_headers + return base_llm_aiohttp_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + ) + + +def complete_custom_openai( + ctx: CompletionDispatchContext, +) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + custom_prompt_dict: Final = ctx.custom_prompt_dict + extra_headers = ctx.extra_headers + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + metadata: Final = dispatch_metadata(ctx) + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + organization = ctx.organization + provider_config: Final = ctx.provider_config + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there + api_base = resolve_openai_api_base(api_base) + organization = ( + organization + or litellm.organization + or get_secret("OPENAI_ORGANIZATION") + or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 + ) + openai.organization = organization + # set API KEY + api_key = ( + api_key + or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there + or litellm.openai_key + or get_secret("OPENAI_API_KEY") + ) + + headers = headers or litellm.headers + outbound_headers: Final = ( + headers + if provider_config is None + else with_attribution_headers(provider_config.get_attribution_headers(), headers) + ) + + # Add GitHub Copilot headers (same as /responses endpoint does) + if custom_llm_provider == "github_copilot": + from litellm.llms.github_copilot.authenticator import Authenticator + from litellm.llms.github_copilot.common_utils import ( + get_copilot_default_headers, + pin_session_authorization, + ) + from litellm.llms.github_copilot.per_user_auth import ( + require_github_copilot_user_session, + ) + + user_session: Final = require_github_copilot_user_session(litellm_params) + copilot_headers: Final = get_copilot_default_headers( + user_session.token if user_session is not None else Authenticator().get_api_key() + ) + if extra_headers: + copilot_headers.update( + cast("dict[str, str]", extra_headers) # cast-ok: extra_headers is a str-valued request dict + ) + if user_session is not None: + pin_session_authorization(copilot_headers, user_session.token) + extra_headers = copilot_headers + + use_base_llm_http_handler: Final = get_secret_bool("EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER") + + if extra_headers is not None and not use_base_llm_http_handler: + optional_params["extra_headers"] = extra_headers + + if litellm.enable_preview_features and metadata is not None: # [PREVIEW] allow metadata to be passed to OPENAI + openai_metadata: Final = get_requester_metadata(metadata) + if openai_metadata is not None: + optional_params["metadata"] = openai_metadata + + ## LOAD CONFIG - if set + config: Final = litellm.OpenAIConfig.get_config() + for k, v in provider_config_items(config): + if ( + k not in optional_params + ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + + ## COMPLETION CALL + try: + if use_base_llm_http_handler: + response = base_llm_http_handler.completion( + model=model, + messages=messages, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + model_response=model_response, + encoding=get_encoding(), + logging_obj=logging, + optional_params=optional_params, + timeout=timeout, + litellm_params=litellm_params, + shared_session=shared_session, + acompletion=acompletion, + stream=stream, + api_key=api_key, + headers=outbound_headers, + client=client, + provider_config=provider_config, + ) + else: + response = openai_chat_completions.completion( + model=model, + messages=messages, + headers=outbound_headers, + model_response=model_response, + print_verbose=print_verbose, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + timeout=timeout, + custom_prompt_dict=custom_prompt_dict, + client=client, # pass AsyncOpenAI, OpenAI client + organization=organization, + custom_llm_provider=custom_llm_provider, + shared_session=shared_session, + ) + except Exception as e: + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": outbound_headers}, + ) + raise e + + if optional_params.get("stream", False): + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + additional_args={"headers": outbound_headers}, + ) + + return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + + +def complete_text_completion_inception( + ctx: CompletionDispatchContext, +) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_openai(ctx) + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + text_completion: Final = ctx.text_completion + timeout: Final = ctx.timeout + + passed_api_base: Final = api_base or optional_params.pop("api_base", None) or optional_params.pop("base_url", None) + api_base = passed_api_base or get_secret_str("INCEPTION_API_BASE") or "https://api.inceptionlabs.ai/v1" + # FIM is served at `/v1/fim/completions`; the OpenAI client appends + # `/completions`, so point it at the `/v1/fim` base. + api_base = api_base.rstrip("/") + if not api_base.endswith("/fim"): + api_base += "/fim" + + # Don't forward the server-managed Inception key to a caller-supplied + # api_base; only resolve it for the default/server base, or when the + # caller passes their own key. + if passed_api_base is None or api_key: + api_key = api_key or litellm.inception_key or get_secret_str("INCEPTION_API_KEY") + + _response = openai_text_completions.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + api_key=api_key, + custom_llm_provider="text-completion-inception", + api_base=api_base, + acompletion=acompletion, + client=client, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + timeout=timeout, + ) + + if optional_params.get("stream", False) is False and acompletion is False and text_completion is False: + _response = litellm.OpenAITextCompletionConfig().convert_to_chat_model_response_object( + response_object=_response, model_response_object=model_response + ) + + if optional_params.get("stream", False) or acompletion is True: + logging.post_call( + input=messages, + api_key=api_key, + original_response=_response, + additional_args={"headers": headers}, + ) + return _response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract diff --git a/litellm/native/chat_providers/platforms.py b/litellm/native/chat_providers/platforms.py new file mode 100644 index 00000000000..7d793d5ab3b --- /dev/null +++ b/litellm/native/chat_providers/platforms.py @@ -0,0 +1,623 @@ +from typing import Final + +import litellm +from litellm.llms.custom_httpx.http_handler import HTTPHandler +from litellm.llms.databricks.common_utils import DatabricksBase +from litellm.native.chat_providers.handlers import ( + base_llm_http_handler, + ovhcloud_transformation, + sap_gen_ai_hub_chat_completions, + watsonx_chat_completion, +) +from litellm.native.chat_providers.support import ( + dispatch_client_http, + get_encoding, + print_verbose, + provider_config_items, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult +from litellm.utils import get_secret + + +def complete_heroku(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key: Final = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + provider_config: Final = ctx.provider_config + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + try: + response: Final = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + provider_config=provider_config, + ) + except Exception as e: + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + + return response + + +def complete_gigachat(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + provider_config: Final = ctx.provider_config + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_key = ( + api_key + or litellm.api_key + or litellm.gigachat_key + or get_secret("GIGACHAT_API_KEY") + or get_secret("GIGACHAT_CREDENTIALS") + ) + + headers = headers or litellm.headers or {} + + ## COMPLETION CALL + try: + response: Final = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + provider_config=provider_config, + ) + except Exception as e: + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + + return response + + +def complete_sap(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key: Final = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + headers = headers or litellm.headers + ## LOAD CONFIG - if set + config: Final = litellm.GenAIHubOrchestrationConfig.get_config() + for k, v in provider_config_items(config): + if ( + k not in optional_params + ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + + return sap_gen_ai_hub_chat_completions.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, + shared_session=shared_session, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + api_key=api_key, + api_base=api_base, + stream=stream, + ) + + +def complete_oci(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key: Final = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + return base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + ) + + +def complete_databricks(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = DatabricksBase.resolve_api_base(ctx.api_base) + api_key: Final = DatabricksBase.resolve_api_key(ctx.api_key) + client: Final = dispatch_client_http(ctx) + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + headers = headers or litellm.headers + + ## COMPLETION CALL + try: + response: Final = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + custom_llm_provider="databricks", + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) + except Exception as e: + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + + if optional_params.get("stream", False): + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + additional_args={"headers": headers}, + ) + + return response + + +def complete_databricks_agent(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + from litellm.llms.databricks.agent.transformation import DatabricksAgentConfig + + api_base, api_key = DatabricksAgentConfig.resolve_api_base_and_key( + api_base=ctx.api_base or litellm.api_base, + api_key=ctx.api_key or litellm.api_key, + ) + return base_llm_http_handler.completion( + model=ctx.model, + stream=ctx.stream, + messages=ctx.messages, + acompletion=ctx.acompletion, + api_base=api_base, + model_response=ctx.model_response, + optional_params=ctx.optional_params, + litellm_params={**ctx.litellm_params, "api_base": api_base}, + shared_session=ctx.shared_session, + custom_llm_provider=ctx.custom_llm_provider, + timeout=ctx.timeout, + headers=ctx.headers or litellm.headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=ctx.logging, + client=dispatch_client_http(ctx), + ) + + +def complete_datarobot(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key: Final = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + provider_config: Final = ctx.provider_config + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + return base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + provider_config=provider_config, + ) + + +def complete_watsonx(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key: Final = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_prompt_dict: Final = ctx.custom_prompt_dict + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + timeout: Final = ctx.timeout + + return watsonx_chat_completion.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + print_verbose=print_verbose, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + timeout=timeout, + custom_prompt_dict=custom_prompt_dict, + client=client, # pass AsyncOpenAI, OpenAI client + encoding=get_encoding(), + custom_llm_provider="watsonx", + ) + + +def complete_watsonx_text( + ctx: CompletionDispatchContext, +) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_key = ( + api_key + or optional_params.pop("apikey", None) + or get_secret_str("WATSONX_APIKEY") + or get_secret_str("WATSONX_API_KEY") + or get_secret_str("WX_API_KEY") + ) + + api_base = ( + api_base + or optional_params.pop( + "url", + optional_params.pop("api_base", optional_params.pop("base_url", None)), + ) + or get_secret_str("WATSONX_API_BASE") + or get_secret_str("WATSONX_URL") + or get_secret_str("WX_URL") + or get_secret_str("WML_URL") + ) + + wx_credentials: Final = optional_params.pop( + "wx_credentials", + optional_params.pop("watsonx_credentials", None), # follow {provider}_credentials, same as vertex ai + ) + + token: str | None = None + if wx_credentials is not None: + api_base = wx_credentials.get("url", api_base) + api_key = wx_credentials.get("apikey", wx_credentials.get("api_key", api_key)) + token = wx_credentials.get( + "token", + wx_credentials.get( + "watsonx_token", None + ), # follow format of {provider}_token, same as azure - e.g. 'azure_ad_token=..' + ) + + if token is not None: + optional_params["token"] = token + + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="watsonx_text", + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) + + +def complete_cloudflare(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + custom_prompt_dict = ctx.custom_prompt_dict + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_key = api_key or litellm.cloudflare_api_key or litellm.api_key or get_secret("CLOUDFLARE_API_KEY") + api_base = api_base or litellm.api_base or get_secret("CLOUDFLARE_API_BASE") + + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="cloudflare", + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + ) + + +def complete_snowflake(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key: Final = ctx.api_key + injected_client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + client: Final = ( + injected_client if injected_client is not None else (HTTPHandler(timeout=timeout) if stream is False else None) + ) # Keep this here, otherwise, the httpx.client closes and streaming is impossible + + try: + response: Final = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + ) + + except Exception as e: + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + + return response + + +def complete_gradient_ai(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key: Final = ctx.api_key + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_base = litellm.api_base or api_base + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="gradient_ai", + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, + ) + + +def complete_ovhcloud(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_key = api_key or litellm.ovhcloud_key or get_secret_str("OVHCLOUD_API_KEY") or litellm.api_key + + api_base = ( + api_base + or litellm.api_base + or get_secret_str("OVHCLOUD_API_BASE") + or "https://oai.endpoints.kepler.ai.cloud.ovh.net/v1" + ) + + response: Final = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + provider_config=ovhcloud_transformation, + ) + + return response diff --git a/litellm/native/chat_providers/self_hosted.py b/litellm/native/chat_providers/self_hosted.py new file mode 100644 index 00000000000..1bb4e83c2e7 --- /dev/null +++ b/litellm/native/chat_providers/self_hosted.py @@ -0,0 +1,325 @@ +import os +from typing import Final + +import litellm +from litellm.llms.oobabooga.chat import oobabooga +from litellm.llms.petals.completion import handler as petals_handler +from litellm.llms.vllm.completion import handler as vllm_handler +from litellm.native.chat_providers.handlers import base_llm_http_handler, lemonade_transformation +from litellm.native.chat_providers.support import dispatch_client_http, get_encoding, print_verbose +from litellm.secret_managers.main import get_secret_str +from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult +from litellm.utils import CustomStreamWrapper, get_secret + + +def complete_hosted_vllm(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key: Final = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + provider_config: Final = ctx.provider_config + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_base = api_base or litellm.api_base or get_secret_str("HOSTED_VLLM_API_BASE") + + response: Final = base_llm_http_handler.completion( + model=model, + messages=messages, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + model_response=model_response, + encoding=get_encoding(), + logging_obj=logging, + optional_params=optional_params, + timeout=timeout, + litellm_params=litellm_params, + shared_session=shared_session, + acompletion=acompletion, + stream=stream, + api_key=api_key, + headers=headers, + client=client, + provider_config=provider_config, + ) + logging.post_call(input=messages, api_key=api_key, original_response=response) + + return response + + +def complete_oobabooga(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + api_base: Final = ctx.api_base + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response = ctx.model_response + optional_params: Final = ctx.optional_params + + model_response = oobabooga.completion( + model=model, + messages=messages, + model_response=model_response, + api_base=api_base, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + api_key=None, + logger_fn=logger_fn, + encoding=get_encoding(), + logging_obj=logging, + ) + if "stream" in optional_params and optional_params["stream"] is True: + # don't try to access stream object, + return CustomStreamWrapper( + model_response, + model, + custom_llm_provider="oobabooga", + logging_obj=logging, + ) + return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + + +def complete_vllm(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + custom_prompt_dict = ctx.custom_prompt_dict + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response = ctx.model_response + optional_params: Final = ctx.optional_params + + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + model_response = vllm_handler.completion( + model=model, + messages=messages, + custom_prompt_dict=custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=get_encoding(), + logging_obj=logging, + ) + + if "stream" in optional_params and optional_params["stream"] is True: ## [BETA] + # don't try to access stream object, + return CustomStreamWrapper( + model_response, + model, + custom_llm_provider="vllm", + logging_obj=logging, + ) + + ## RESPONSE OBJECT + return model_response + + +def complete_ollama(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key: Final = ctx.api_key + client: Final = dispatch_client_http(ctx) + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_base = litellm.api_base or api_base or get_secret("OLLAMA_API_BASE") or "http://localhost:11434" + if api_key is not None and "Authorization" not in headers: + headers["Authorization"] = f"Bearer {api_key}" + + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="ollama", + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) + + +def complete_ollama_chat(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_base = litellm.api_base or api_base or get_secret("OLLAMA_API_BASE") or "http://localhost:11434" + + api_key = api_key or litellm.ollama_key or os.environ.get("OLLAMA_API_KEY") or litellm.api_key + if api_key is not None and "Authorization" not in headers: + headers["Authorization"] = f"Bearer {api_key}" + + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="ollama_chat", + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) + + +def complete_triton(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key: Final = ctx.api_key + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_base = litellm.api_base or api_base + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, + ) + + +def complete_petals(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + api_base = ctx.api_base + client: Final = dispatch_client_http(ctx) + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response = ctx.model_response + optional_params: Final = ctx.optional_params + stream = ctx.stream + + api_base = api_base or litellm.api_base + + stream = optional_params.pop("stream", False) + model_response = petals_handler.completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=get_encoding(), + logging_obj=logging, + client=client, + ) + if stream is True: ## [BETA] + # Fake streaming for petals + resp_string: Final = model_response["choices"][0]["message"]["content"] + return CustomStreamWrapper( + resp_string, + model, + custom_llm_provider="petals", + logging_obj=logging, + ) + return model_response + + +def complete_lemonade(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_key = api_key or litellm.lemonade_key or get_secret_str("LEMONADE_API_KEY") or litellm.api_key + + response: Final = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + provider_config=lemonade_transformation, + ) + + return response diff --git a/litellm/native/chat_providers/support.py b/litellm/native/chat_providers/support.py new file mode 100644 index 00000000000..a36b81b0c32 --- /dev/null +++ b/litellm/native/chat_providers/support.py @@ -0,0 +1,47 @@ +from collections.abc import Iterable, Mapping +from functools import cache + +import openai + +import litellm +from litellm._lazy_imports import get_default_encoding +from litellm._logging import verbose_logger +from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.types.completion import CompletionDispatchContext + + +def dispatch_metadata(ctx: CompletionDispatchContext) -> Mapping[str, object] | None: + return ctx.metadata + + +def dispatch_client_http(ctx: CompletionDispatchContext) -> HTTPHandler | AsyncHTTPHandler | None: + return ctx.client + + +def dispatch_client_azure( + ctx: CompletionDispatchContext, +) -> openai.AzureOpenAI | openai.AsyncAzureOpenAI | HTTPHandler | AsyncHTTPHandler | None: + return ctx.client + + +def dispatch_client_openai(ctx: CompletionDispatchContext) -> openai.OpenAI | openai.AsyncOpenAI | None: + return ctx.client + + +def provider_config_items(config: Mapping[str, object]) -> Iterable[tuple[str, object]]: + return config.items() + + +def print_verbose(print_statement: object) -> None: + try: + verbose_logger.debug(print_statement) + if litellm.set_verbose: + print(print_statement) # noqa: T201 + except Exception: + pass + + +@cache +def get_encoding() -> Tokenizer: + return get_default_encoding() diff --git a/litellm/proxy/common_utils/load_config_utils.py b/litellm/proxy/common_utils/load_config_utils.py index 4a082eb307b..7dced776e4b 100644 --- a/litellm/proxy/common_utils/load_config_utils.py +++ b/litellm/proxy/common_utils/load_config_utils.py @@ -47,7 +47,7 @@ def s3_object_reader(bucket_name: str) -> SyncBucketObjectReader: import boto3 from botocore.credentials import Credentials - from litellm.main import bedrock_converse_chat_completion + from litellm.native.chat_providers.handlers import bedrock_converse_chat_completion credentials: Final[Credentials] = bedrock_converse_chat_completion.get_credentials() s3_client: Final = boto3.client( diff --git a/litellm/responses/bridge_check.py b/litellm/responses/bridge_check.py index 60318d6b184..051b77f3c27 100644 --- a/litellm/responses/bridge_check.py +++ b/litellm/responses/bridge_check.py @@ -68,9 +68,9 @@ def _is_openai_backed_api_base(api_base: str) -> bool: return hostname is not None and (hostname == _OPENAI_API_HOST or hostname.endswith(f".{_OPENAI_API_HOST}")) -def _resolve_openai_api_base(api_base: str | None) -> str: +def resolve_openai_api_base(api_base: str | None) -> str: """Effective OpenAI base a chat request will hit: arg > global > env > default. The bridge gate - and the ``_complete_custom_openai`` chat handler MUST resolve this identically, or a custom base + and the ``complete_custom_openai`` chat handler MUST resolve this identically, or a custom base set via ``litellm.api_base`` or ``OPENAI_BASE_URL``/``OPENAI_API_BASE`` is invisible to the gate, which then misreads it as the default OpenAI endpoint and bridges a request the backend can't serve.""" return ( @@ -239,7 +239,7 @@ def responses_api_bridge_check( # exactly as the chat handler does, so a custom base set via litellm.api_base or # OPENAI_BASE_URL/OPENAI_API_BASE isn't misread as the default and bridged to a /responses route it # lacks. A whitespace-only base collapses to the default too. - resolved_api_base: Final = _resolve_openai_api_base(api_base).strip() + resolved_api_base: Final = resolve_openai_api_base(api_base).strip() on_foundry_openai_endpoint: Final = custom_llm_provider == "azure_ai" and azure_ai_supports_native_responses( model, api_base ) diff --git a/litellm/types/completion.py b/litellm/types/completion.py index a6693030e89..866266261a2 100644 --- a/litellm/types/completion.py +++ b/litellm/types/completion.py @@ -245,7 +245,7 @@ class CompletionDispatchContext: _CompletionDispatchContext = CompletionDispatchContext -_CompletionDispatchResult = Union[ +CompletionDispatchResult = Union[ Coroutine[Any, Any, Union["ModelResponse", "CustomStreamWrapper"]], "ModelResponse", "CustomStreamWrapper", diff --git a/tests/unit/llms/azure/chat/test_azure_chat_o_series_transformation.py b/tests/unit/llms/azure/chat/test_azure_chat_o_series_transformation.py index 1cdc288ff87..79d87ee24ad 100644 --- a/tests/unit/llms/azure/chat/test_azure_chat_o_series_transformation.py +++ b/tests/unit/llms/azure/chat/test_azure_chat_o_series_transformation.py @@ -172,7 +172,7 @@ def test_azure_o_series_routing(): assert "stream" not in mock_create.call_args.kwargs["body"] -@patch("litellm.main.azure_o1_chat_completions._get_openai_client") +@patch("litellm.native.chat_providers.handlers.azure_o1_chat_completions._get_openai_client") def test_openai_o_series_max_retries_0(mock_get_openai_client): import litellm diff --git a/tests/unit/llms/azure/test_azure_common_utils.py b/tests/unit/llms/azure/test_azure_common_utils.py index 41ad27d6f40..b270b41b029 100644 --- a/tests/unit/llms/azure/test_azure_common_utils.py +++ b/tests/unit/llms/azure/test_azure_common_utils.py @@ -728,7 +728,7 @@ async def test_ensure_initialize_azure_sdk_client_always_used_azure_text(call_ty # Get appropriate input for this call type input_kwarg = test_inputs.get(call_type.value, {}) - patch_target = "litellm.main.azure_text_completions.initialize_azure_sdk_client" + patch_target = "litellm.native.chat_providers.handlers.azure_text_completions.initialize_azure_sdk_client" # Mock the initialize_azure_sdk_client function with patch(patch_target) as mock_init_azure: diff --git a/tests/unit/llms/azure_ai/claude/test_main_azure_anthropic_timeout.py b/tests/unit/llms/azure_ai/claude/test_main_azure_anthropic_timeout.py index e08098e9aab..9a48aafdea6 100644 --- a/tests/unit/llms/azure_ai/claude/test_main_azure_anthropic_timeout.py +++ b/tests/unit/llms/azure_ai/claude/test_main_azure_anthropic_timeout.py @@ -21,7 +21,7 @@ def test_main_azure_ai_claude_completion_passes_timeout_to_azure_anthropic_handl captured.update(kwargs) return ModelResponse() - with patch("litellm.main.azure_anthropic_chat_completions") as mock_azure_anthropic: + with patch("litellm.native.chat_providers.azure.azure_anthropic_chat_completions") as mock_azure_anthropic: mock_azure_anthropic.completion = MagicMock( side_effect=fake_azure_anthropic_completion ) diff --git a/tests/unit/llms/gdc/chat/test_gdc_chat_transformation.py b/tests/unit/llms/gdc/chat/test_gdc_chat_transformation.py index 3d15a2e8870..440a5455209 100644 --- a/tests/unit/llms/gdc/chat/test_gdc_chat_transformation.py +++ b/tests/unit/llms/gdc/chat/test_gdc_chat_transformation.py @@ -686,7 +686,7 @@ class TestGDCGeminiConfig: class TestCompleteGDC: @patch("litellm.main.base_llm_http_handler.completion") def test_complete_gdc_resolves_key_and_base(self, mock_completion, monkeypatch): - from litellm.main import gdc_transformation + from litellm.native.chat_providers.handlers import gdc_transformation mock_completion.return_value = MagicMock() monkeypatch.setattr(litellm, "gdc_key", "resolved-key", raising=False) diff --git a/tests/unit/llms/gemini/test_gemini_client_setup.py b/tests/unit/llms/gemini/test_gemini_client_setup.py index 48b010aca48..2aac6d69163 100644 --- a/tests/unit/llms/gemini/test_gemini_client_setup.py +++ b/tests/unit/llms/gemini/test_gemini_client_setup.py @@ -28,7 +28,7 @@ def test_gemini_completion_no_api_key(): del os.environ[key] # Test without mock_response to ensure actual API key validation - with pytest.raises(Exception, match='in _complete_vertex_ai_beta') as exc_info: + with pytest.raises(Exception, match='in complete_vertex_ai_beta') as exc_info: completion( model="gemini/gemini-1.5-flash", messages=[{"role": "user", "content": "Test message"}], @@ -60,7 +60,7 @@ def test_gemini_completion_no_api_key_with_mock(): with patch("litellm.get_secret") as mock_get_secret: mock_get_secret.return_value = None - with pytest.raises(Exception, match='in _complete_vertex_ai_beta') as exc_info: + with pytest.raises(Exception, match='in complete_vertex_ai_beta') as exc_info: completion( model="gemini/gemini-1.5-flash", messages=[{"role": "user", "content": "Test message"}], diff --git a/tests/unit/llms/github_copilot/test_per_user_auth.py b/tests/unit/llms/github_copilot/test_per_user_auth.py index 9b16d3c0aa7..b7284bf6a2f 100644 --- a/tests/unit/llms/github_copilot/test_per_user_auth.py +++ b/tests/unit/llms/github_copilot/test_per_user_auth.py @@ -550,7 +550,7 @@ def test_e2e_completion_per_user_credential(per_user_credential): def test_e2e_completion_per_user_session_token_wins_over_caller_authorization(per_user_credential): """The session token reaches the wire even when the caller passes their own - Authorization in extra_headers (chat dispatches through _complete_custom_openai).""" + Authorization in extra_headers (chat dispatches through complete_custom_openai).""" with ( github_http(_exchange_responder()), _no_authenticator(), diff --git a/tests/unit/llms/vertex_ai/test_vertex.py b/tests/unit/llms/vertex_ai/test_vertex.py index 04cf391c700..501ec5d6058 100644 --- a/tests/unit/llms/vertex_ai/test_vertex.py +++ b/tests/unit/llms/vertex_ai/test_vertex.py @@ -1545,9 +1545,9 @@ def test_system_prompt_only_adds_blank_user_message(): @pytest.fixture def _cached_vertex_credentials(monkeypatch: pytest.MonkeyPatch) -> str: - from litellm.main import ( + from litellm.main import vertex_embedding + from litellm.native.chat_providers.handlers import ( vertex_chat_completion, - vertex_embedding, vertex_model_garden_chat_completion, vertex_partner_models_chat_completion, ) diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_anthropic_image_url_handling.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_anthropic_image_url_handling.py index 98071594ebf..7dde7e6b38d 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_anthropic_image_url_handling.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_anthropic_image_url_handling.py @@ -15,6 +15,7 @@ import pytest import litellm from litellm._uuid import uuid +from litellm.native.chat_providers.handlers import vertex_partner_models_chat_completion from litellm.litellm_core_utils.prompt_templates.factory import ( anthropic_messages_pt, convert_to_anthropic_tool_result, @@ -406,7 +407,7 @@ async def test_vertex_ai_anthropic_async_completion_inlines_https_images_off_the with ( patch.dict(sys.modules, {"vertexai": vertexai}), patch.object( # test-quality-ok: litellm.acompletion has no seam for Vertex token minting - litellm.main.vertex_partner_models_chat_completion, + vertex_partner_models_chat_completion, "_ensure_access_token", return_value=("token", "test-project"), ), diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py index 21bda6f0345..fbf332df29c 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py @@ -538,7 +538,7 @@ async def test_vertex_ai_anthropic_prompt_caching_does_not_add_beta_header( monkeypatch: pytest.MonkeyPatch, sync_mode: bool, ) -> None: - from litellm.main import vertex_partner_models_chat_completion + from litellm.native.chat_providers.handlers import vertex_partner_models_chat_completion monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) monkeypatch.setitem( diff --git a/tests/unit/llms/vertex_ai/vertex_gemma_models/test_vertex_gemma_transformation.py b/tests/unit/llms/vertex_ai/vertex_gemma_models/test_vertex_gemma_transformation.py index fb1cf4ced2d..8681882d3df 100644 --- a/tests/unit/llms/vertex_ai/vertex_gemma_models/test_vertex_gemma_transformation.py +++ b/tests/unit/llms/vertex_ai/vertex_gemma_models/test_vertex_gemma_transformation.py @@ -1320,7 +1320,7 @@ def _gemma_cached_access_token(): """Serve a fake token from the handler's credential cache so no auth round-trip runs.""" from types import SimpleNamespace - from litellm.main import vertex_gemma_chat_completion + from litellm.native.chat_providers.handlers import vertex_gemma_chat_completion cache = vertex_gemma_chat_completion._credentials_project_mapping key = (_FAKE_GEMMA_CREDENTIALS, "test") diff --git a/tests/unit/proxy/common_utils/test_load_config_utils.py b/tests/unit/proxy/common_utils/test_load_config_utils.py index 927e6ee511d..107ea31c607 100644 --- a/tests/unit/proxy/common_utils/test_load_config_utils.py +++ b/tests/unit/proxy/common_utils/test_load_config_utils.py @@ -20,7 +20,7 @@ class TestGetFileContentsFromS3: """Test suite for S3 config loading functionality.""" @patch("boto3.client") - @patch("litellm.main.bedrock_converse_chat_completion") + @patch("litellm.native.chat_providers.handlers.bedrock_converse_chat_completion") @patch("yaml.safe_load") def test_get_file_contents_from_s3_no_temp_file_creation( self, mock_yaml_load, mock_bedrock, mock_boto3_client diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 2da266b9912..e3aa3686103 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -5243,7 +5243,7 @@ def test_bridged_responses_with_openai_http_handler_keeps_forwarded_headers_out_ def test_aiohttp_openai_warns_only_when_http2_enabled( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, http2_on: bool ): - from litellm.main import base_llm_aiohttp_handler + from litellm.native.chat_providers.handlers import base_llm_aiohttp_handler monkeypatch.setattr(litellm, "http2", http2_on) monkeypatch.delenv("LITELLM_HTTP2", raising=False)