diff --git a/.github/workflows/auto_update_price_and_context_window.yml b/.github/workflows/auto_update_price_and_context_window.yml new file mode 100644 index 00000000000..e7d65242c19 --- /dev/null +++ b/.github/workflows/auto_update_price_and_context_window.yml @@ -0,0 +1,28 @@ +name: Updates model_prices_and_context_window.json and Create Pull Request + +on: + schedule: + - cron: "0 0 * * 0" # Run every Sundays at midnight + #- cron: "0 0 * * *" # Run daily at midnight + +jobs: + auto_update_price_and_context_window: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Install Dependencies + run: | + pip install aiohttp + - name: Update JSON Data + run: | + python ".github/workflows/auto_update_price_and_context_window_file.py" + - name: Create Pull Request + run: | + git add model_prices_and_context_window.json + git commit -m "Update model_prices_and_context_window.json file: $(date +'%Y-%m-%d')" + gh pr create --title "Update model_prices_and_context_window.json file" \ + --body "Automated update for model_prices_and_context_window.json" \ + --head auto-update-price-and-context-window-$(date +'%Y-%m-%d') \ + --base main + env: + GH_TOKEN: ${{ secrets.GH_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/auto_update_price_and_context_window_file.py b/.github/workflows/auto_update_price_and_context_window_file.py new file mode 100644 index 00000000000..3e0731b94bd --- /dev/null +++ b/.github/workflows/auto_update_price_and_context_window_file.py @@ -0,0 +1,121 @@ +import asyncio +import aiohttp +import json + +# Asynchronously fetch data from a given URL +async def fetch_data(url): + try: + # Create an asynchronous session + async with aiohttp.ClientSession() as session: + # Send a GET request to the URL + async with session.get(url) as resp: + # Raise an error if the response status is not OK + resp.raise_for_status() + # Parse the response JSON + resp_json = await resp.json() + print("Fetch the data from URL.") + # Return the 'data' field from the JSON response + return resp_json['data'] + except Exception as e: + # Print an error message if fetching data fails + print("Error fetching data from URL:", e) + return None + +# Synchronize local data with remote data +def sync_local_data_with_remote(local_data, remote_data): + # Update existing keys in local_data with values from remote_data + for key in (set(local_data) & set(remote_data)): + local_data[key].update(remote_data[key]) + + # Add new keys from remote_data to local_data + for key in (set(remote_data) - set(local_data)): + local_data[key] = remote_data[key] + +# Write data to the json file +def write_to_file(file_path, data): + try: + # Open the file in write mode + with open(file_path, "w") as file: + # Dump the data as JSON into the file + json.dump(data, file, indent=4) + print("Values updated successfully.") + except Exception as e: + # Print an error message if writing to file fails + print("Error updating JSON file:", e) + +# Update the existing models and add the missing models +def transform_remote_data(data): + transformed = {} + for row in data: + # Add the fields 'max_tokens' and 'input_cost_per_token' + obj = { + "max_tokens": row["context_length"], + "input_cost_per_token": float(row["pricing"]["prompt"]), + } + + # Add 'max_output_tokens' as a field if it is not None + if "top_provider" in row and "max_completion_tokens" in row["top_provider"] and row["top_provider"]["max_completion_tokens"] is not None: + obj['max_output_tokens'] = int(row["top_provider"]["max_completion_tokens"]) + + # Add the field 'output_cost_per_token' + obj.update({ + "output_cost_per_token": float(row["pricing"]["completion"]), + }) + + # Add field 'input_cost_per_image' if it exists and is non-zero + if "pricing" in row and "image" in row["pricing"] and float(row["pricing"]["image"]) != 0.0: + obj['input_cost_per_image'] = float(row["pricing"]["image"]) + + # Add the fields 'litellm_provider' and 'mode' + obj.update({ + "litellm_provider": "openrouter", + "mode": "chat" + }) + + # Add the 'supports_vision' field if the modality is 'multimodal' + if row.get('architecture', {}).get('modality') == 'multimodal': + obj['supports_vision'] = True + + # Use a composite key to store the transformed object + transformed[f'openrouter/{row["id"]}'] = obj + + return transformed + + +# Load local data from a specified file +def load_local_data(file_path): + try: + # Open the file in read mode + with open(file_path, "r") as file: + # Load and return the JSON data + return json.load(file) + except FileNotFoundError: + # Print an error message if the file is not found + print("File not found:", file_path) + return None + except json.JSONDecodeError as e: + # Print an error message if JSON decoding fails + print("Error decoding JSON:", e) + return None + +def main(): + local_file_path = "model_prices_and_context_window.json" # Path to the local data file + url = "https://openrouter.ai/api/v1/models" # URL to fetch remote data + + # Load local data from file + local_data = load_local_data(local_file_path) + # Fetch remote data asynchronously + remote_data = asyncio.run(fetch_data(url)) + # Transform the fetched remote data + remote_data = transform_remote_data(remote_data) + + # If both local and remote data are available, synchronize and save + if local_data and remote_data: + sync_local_data_with_remote(local_data, remote_data) + write_to_file(local_file_path, local_data) + else: + print("Failed to fetch model data from either local file or URL.") + +# Entry point of the script +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/.github/workflows/load_test.yml b/.github/workflows/load_test.yml index ddf613fa660..cdaffa328c9 100644 --- a/.github/workflows/load_test.yml +++ b/.github/workflows/load_test.yml @@ -22,14 +22,23 @@ jobs: run: | python -m pip install --upgrade pip pip install PyGithub + - name: re-deploy proxy + run: | + echo "Current working directory: $PWD" + ls + python ".github/workflows/redeploy_proxy.py" + env: + LOAD_TEST_REDEPLOY_URL1: ${{ secrets.LOAD_TEST_REDEPLOY_URL1 }} + LOAD_TEST_REDEPLOY_URL2: ${{ secrets.LOAD_TEST_REDEPLOY_URL2 }} + working-directory: ${{ github.workspace }} - name: Run Load Test id: locust_run uses: BerriAI/locust-github-action@master with: LOCUSTFILE: ".github/workflows/locustfile.py" - URL: "https://litellm-database-docker-build-production.up.railway.app/" - USERS: "100" - RATE: "10" + URL: "https://post-release-load-test-proxy.onrender.com/" + USERS: "20" + RATE: "20" RUNTIME: "300s" - name: Process Load Test Stats run: | diff --git a/.github/workflows/locustfile.py b/.github/workflows/locustfile.py index 5dce0bb02f2..34ac7bee027 100644 --- a/.github/workflows/locustfile.py +++ b/.github/workflows/locustfile.py @@ -10,7 +10,7 @@ class MyUser(HttpUser): def chat_completion(self): headers = { "Content-Type": "application/json", - "Authorization": f"Bearer sk-S2-EZTUUDY0EmM6-Fy0Fyw", + "Authorization": f"Bearer sk-ZoHqrLIs2-5PzJrqBaviAA", # Include any additional headers you may need for authentication, etc. } @@ -28,15 +28,3 @@ class MyUser(HttpUser): response = self.client.post("chat/completions", json=payload, headers=headers) # Print or log the response if needed - - @task(10) - def health_readiness(self): - start_time = time.time() - response = self.client.get("health/readiness") - response_time = time.time() - start_time - - @task(10) - def health_liveliness(self): - start_time = time.time() - response = self.client.get("health/liveliness") - response_time = time.time() - start_time diff --git a/.github/workflows/redeploy_proxy.py b/.github/workflows/redeploy_proxy.py new file mode 100644 index 00000000000..ed46bef73a2 --- /dev/null +++ b/.github/workflows/redeploy_proxy.py @@ -0,0 +1,20 @@ +""" + +redeploy_proxy.py +""" + +import os +import requests +import time + +# send a get request to this endpoint +deploy_hook1 = os.getenv("LOAD_TEST_REDEPLOY_URL1") +response = requests.get(deploy_hook1, timeout=20) + + +deploy_hook2 = os.getenv("LOAD_TEST_REDEPLOY_URL2") +response = requests.get(deploy_hook2, timeout=20) + +print("SENT GET REQUESTS to re-deploy proxy") +print("sleeeping.... for 60s") +time.sleep(60) diff --git a/README.md b/README.md index 5e94b0fd94a..415ea8480e0 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,12 @@ 🚅 LiteLLM
+
+
+
+
+
+
Call all LLM APIs using the OpenAI format [Bedrock, Huggingface, VertexAI, TogetherAI, Azure, OpenAI, etc.]
Hi {recipient_email},
+
+ I'm happy to provide you with an OpenAI Proxy API Key, loaded with ${key_budget} per month.
+
+
+ Key: {key_token}
+
+
+
+
+ import openai
+ client = openai.OpenAI(
+ api_key="{key_token}",
+ base_url={os.getenv("PROXY_BASE_URL", "http://0.0.0.0:4000")}
+ )
+
+ response = client.chat.completions.create(
+ model="gpt-3.5-turbo", # model to send to the proxy
+ messages = [
+ {{
+ "role": "user",
+ "content": "this is a test request, write a short poem"
+ }}
+ ]
+ )
+
+
+
+
+ If you have any questions, please send an email to {EMAIL_SUPPORT_CONTACT} Hi {user_name},
+
+ Your LLM API usage this month has reached your account's monthly budget of ${max_budget}
+
+ API requests will be rejected until either (a) you increase your monthly budget or (b) your monthly usage resets at the beginning of the next calendar month.
+
+ If you have any questions, please send an email to {EMAIL_SUPPORT_CONTACT}
+
+ Best,
+ The LiteLLM team
+ """
+
+ payload = webhook_event.model_dump_json()
+ email_event = {
+ "to": recipient_email,
+ "subject": f"LiteLLM: {event_name}",
+ "html": email_html_content,
+ }
+
+ response = await send_email(
+ receiver_email=email_event["to"],
+ subject=email_event["subject"],
+ html=email_event["html"],
+ )
+
+ return False
+
async def send_alert(
self,
message: str,
level: Literal["Low", "Medium", "High"],
- alert_type: Literal[
- "llm_exceptions",
- "llm_too_slow",
- "llm_requests_hanging",
- "budget_alerts",
- "db_exceptions",
- "daily_reports",
- "spend_reports",
- "new_model_added",
- "cooldown_deployment",
- ],
+ alert_type: Literal[AlertType],
user_info: Optional[WebhookEvent] = None,
**kwargs,
):
@@ -818,6 +1327,14 @@ Model Info:
):
await self.send_webhook_alert(webhook_event=user_info)
+ if (
+ "email" in self.alerting
+ and alert_type == "budget_alerts"
+ and user_info is not None
+ ):
+ # only send budget alerts over Email
+ await self.send_email_alert_using_smtp(webhook_event=user_info)
+
if "slack" not in self.alerting:
return
@@ -905,18 +1422,36 @@ Model Info:
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
"""Log failure + deployment latency"""
- if "daily_reports" in self.alert_types:
- model_id = (
- kwargs.get("litellm_params", {}).get("model_info", {}).get("id", "")
- )
- await self.async_update_daily_reports(
- DeploymentMetrics(
- id=model_id,
- failed_request=True,
- latency_per_output_token=None,
- updated_at=litellm.utils.get_utc_datetime(),
- )
- )
+ _litellm_params = kwargs.get("litellm_params", {})
+ _model_info = _litellm_params.get("model_info", {}) or {}
+ model_id = _model_info.get("id", "")
+ try:
+ if "daily_reports" in self.alert_types:
+ try:
+ await self.async_update_daily_reports(
+ DeploymentMetrics(
+ id=model_id,
+ failed_request=True,
+ latency_per_output_token=None,
+ updated_at=litellm.utils.get_utc_datetime(),
+ )
+ )
+ except Exception as e:
+ verbose_logger.debug(f"Exception raises -{str(e)}")
+
+ if isinstance(kwargs.get("exception", ""), APIError):
+ if "outage_alerts" in self.alert_types:
+ await self.outage_alerts(
+ exception=kwargs["exception"],
+ deployment_id=model_id,
+ )
+
+ if "region_outage_alerts" in self.alert_types:
+ await self.region_outage_alerts(
+ exception=kwargs["exception"], deployment_id=model_id
+ )
+ except Exception as e:
+ pass
async def _run_scheduler_helper(self, llm_router) -> bool:
"""
@@ -928,40 +1463,26 @@ Model Info:
report_sent = await self.internal_usage_cache.async_get_cache(
key=SlackAlertingCacheKeys.report_sent_key.value
- ) # None | datetime
+ ) # None | float
- current_time = litellm.utils.get_utc_datetime()
+ current_time = time.time()
if report_sent is None:
- _current_time = current_time.isoformat()
await self.internal_usage_cache.async_set_cache(
key=SlackAlertingCacheKeys.report_sent_key.value,
- value=_current_time,
+ value=current_time,
)
- else:
+ elif isinstance(report_sent, float):
# Check if current time - interval >= time last sent
- delta_naive = timedelta(seconds=self.alerting_args.daily_report_frequency)
- if isinstance(report_sent, str):
- report_sent = dt.fromisoformat(report_sent)
+ interval_seconds = self.alerting_args.daily_report_frequency
- # Ensure report_sent is an aware datetime object
- if report_sent.tzinfo is None:
- report_sent = report_sent.replace(tzinfo=timezone.utc)
-
- # Calculate delta as an aware datetime object with the same timezone as report_sent
- delta = report_sent - delta_naive
-
- current_time_utc = current_time.astimezone(timezone.utc)
- delta_utc = delta.astimezone(timezone.utc)
-
- if current_time_utc >= delta_utc:
+ if current_time - report_sent >= interval_seconds:
# Sneak in the reporting logic here
await self.send_daily_reports(router=llm_router)
# Also, don't forget to update the report_sent time after sending the report!
- _current_time = current_time.isoformat()
await self.internal_usage_cache.async_set_cache(
key=SlackAlertingCacheKeys.report_sent_key.value,
- value=_current_time,
+ value=current_time,
)
report_sent_bool = True
diff --git a/litellm/llms/bedrock_httpx.py b/litellm/llms/bedrock_httpx.py
index 5fe0e0cc17f..337055dc2ee 100644
--- a/litellm/llms/bedrock_httpx.py
+++ b/litellm/llms/bedrock_httpx.py
@@ -44,6 +44,7 @@ from .base import BaseLLM
import httpx # type: ignore
from .bedrock import BedrockError, convert_messages_to_prompt, ModelResponseIterator
from litellm.types.llms.bedrock import *
+import urllib.parse
class AmazonCohereChatConfig:
@@ -524,6 +525,16 @@ class BedrockLLM(BaseLLM):
return model_response
+ def encode_model_id(self, model_id: str) -> str:
+ """
+ Double encode the model ID to ensure it matches the expected double-encoded format.
+ Args:
+ model_id (str): The model ID to encode.
+ Returns:
+ str: The double-encoded model ID.
+ """
+ return urllib.parse.quote(model_id, safe="")
+
def completion(
self,
model: str,
@@ -552,6 +563,12 @@ class BedrockLLM(BaseLLM):
## SETUP ##
stream = optional_params.pop("stream", None)
+ modelId = optional_params.pop("model_id", None)
+ if modelId is not None:
+ modelId = self.encode_model_id(model_id=modelId)
+ else:
+ modelId = model
+
provider = model.split(".")[0]
## CREDENTIALS ##
@@ -609,9 +626,9 @@ class BedrockLLM(BaseLLM):
endpoint_url = f"https://bedrock-runtime.{aws_region_name}.amazonaws.com"
if (stream is not None and stream == True) and provider != "ai21":
- endpoint_url = f"{endpoint_url}/model/{model}/invoke-with-response-stream"
+ endpoint_url = f"{endpoint_url}/model/{modelId}/invoke-with-response-stream"
else:
- endpoint_url = f"{endpoint_url}/model/{model}/invoke"
+ endpoint_url = f"{endpoint_url}/model/{modelId}/invoke"
sigv4 = SigV4Auth(credentials, "bedrock", aws_region_name)
diff --git a/litellm/llms/clarifai.py b/litellm/llms/clarifai.py
index e07a8d9e8aa..4610911e14b 100644
--- a/litellm/llms/clarifai.py
+++ b/litellm/llms/clarifai.py
@@ -14,28 +14,25 @@ class ClarifaiError(Exception):
def __init__(self, status_code, message, url):
self.status_code = status_code
self.message = message
- self.request = httpx.Request(
- method="POST", url=url
- )
+ self.request = httpx.Request(method="POST", url=url)
self.response = httpx.Response(status_code=status_code, request=self.request)
- super().__init__(
- self.message
- )
+ super().__init__(self.message)
+
class ClarifaiConfig:
"""
Reference: https://clarifai.com/meta/Llama-2/models/llama2-70b-chat
- TODO fill in the details
"""
+
max_tokens: Optional[int] = None
temperature: Optional[int] = None
top_k: Optional[int] = None
def __init__(
- self,
- max_tokens: Optional[int] = None,
- temperature: Optional[int] = None,
- top_k: Optional[int] = None,
+ self,
+ max_tokens: Optional[int] = None,
+ temperature: Optional[int] = None,
+ top_k: Optional[int] = None,
) -> None:
locals_ = locals()
for key, value in locals_.items():
@@ -60,6 +57,7 @@ class ClarifaiConfig:
and v is not None
}
+
def validate_environment(api_key):
headers = {
"accept": "application/json",
@@ -69,42 +67,37 @@ def validate_environment(api_key):
headers["Authorization"] = f"Bearer {api_key}"
return headers
-def completions_to_model(payload):
- # if payload["n"] != 1:
- # raise HTTPException(
- # status_code=422,
- # detail="Only one generation is supported. Please set candidate_count to 1.",
- # )
- params = {}
- if temperature := payload.get("temperature"):
- params["temperature"] = temperature
- if max_tokens := payload.get("max_tokens"):
- params["max_tokens"] = max_tokens
- return {
- "inputs": [{"data": {"text": {"raw": payload["prompt"]}}}],
- "model": {"output_info": {"params": params}},
-}
-
+def completions_to_model(payload):
+ # if payload["n"] != 1:
+ # raise HTTPException(
+ # status_code=422,
+ # detail="Only one generation is supported. Please set candidate_count to 1.",
+ # )
+
+ params = {}
+ if temperature := payload.get("temperature"):
+ params["temperature"] = temperature
+ if max_tokens := payload.get("max_tokens"):
+ params["max_tokens"] = max_tokens
+ return {
+ "inputs": [{"data": {"text": {"raw": payload["prompt"]}}}],
+ "model": {"output_info": {"params": params}},
+ }
+
+
def process_response(
- model,
- prompt,
- response,
- model_response,
- api_key,
- data,
- encoding,
- logging_obj
- ):
+ model, prompt, response, model_response, api_key, data, encoding, logging_obj
+):
logging_obj.post_call(
- input=prompt,
- api_key=api_key,
- original_response=response.text,
- additional_args={"complete_input_dict": data},
- )
- ## RESPONSE OBJECT
+ input=prompt,
+ api_key=api_key,
+ original_response=response.text,
+ additional_args={"complete_input_dict": data},
+ )
+ ## RESPONSE OBJECT
try:
- completion_response = response.json()
+ completion_response = response.json()
except Exception:
raise ClarifaiError(
message=response.text, status_code=response.status_code, url=model
@@ -119,7 +112,7 @@ def process_response(
message_obj = Message(content=None)
choice_obj = Choices(
finish_reason="stop",
- index=idx + 1, #check
+ index=idx + 1, # check
message=message_obj,
)
choices_list.append(choice_obj)
@@ -143,53 +136,56 @@ def process_response(
)
return model_response
+
def convert_model_to_url(model: str, api_base: str):
user_id, app_id, model_id = model.split(".")
return f"{api_base}/users/{user_id}/apps/{app_id}/models/{model_id}/outputs"
+
def get_prompt_model_name(url: str):
clarifai_model_name = url.split("/")[-2]
if "claude" in clarifai_model_name:
return "anthropic", clarifai_model_name.replace("_", ".")
- if ("llama" in clarifai_model_name)or ("mistral" in clarifai_model_name):
+ if ("llama" in clarifai_model_name) or ("mistral" in clarifai_model_name):
return "", "meta-llama/llama-2-chat"
else:
return "", clarifai_model_name
+
async def async_completion(
- model: str,
- prompt: str,
- api_base: str,
- custom_prompt_dict: dict,
- model_response: ModelResponse,
- print_verbose: Callable,
- encoding,
- api_key,
- logging_obj,
- data=None,
- optional_params=None,
- litellm_params=None,
- logger_fn=None,
- headers={}):
-
- async_handler = AsyncHTTPHandler(
- timeout=httpx.Timeout(timeout=600.0, connect=5.0)
- )
+ model: str,
+ prompt: str,
+ api_base: str,
+ custom_prompt_dict: dict,
+ model_response: ModelResponse,
+ print_verbose: Callable,
+ encoding,
+ api_key,
+ logging_obj,
+ data=None,
+ optional_params=None,
+ litellm_params=None,
+ logger_fn=None,
+ headers={},
+):
+
+ async_handler = AsyncHTTPHandler(timeout=httpx.Timeout(timeout=600.0, connect=5.0))
response = await async_handler.post(
- api_base, headers=headers, data=json.dumps(data)
- )
-
- return process_response(
- model=model,
- prompt=prompt,
- response=response,
- model_response=model_response,
- api_key=api_key,
- data=data,
- encoding=encoding,
- logging_obj=logging_obj,
+ api_base, headers=headers, data=json.dumps(data)
)
+ return process_response(
+ model=model,
+ prompt=prompt,
+ response=response,
+ model_response=model_response,
+ api_key=api_key,
+ data=data,
+ encoding=encoding,
+ logging_obj=logging_obj,
+ )
+
+
def completion(
model: str,
messages: list,
@@ -207,14 +203,12 @@ def completion(
):
headers = validate_environment(api_key)
model = convert_model_to_url(model, api_base)
- prompt = " ".join(message["content"] for message in messages) # TODO
+ prompt = " ".join(message["content"] for message in messages) # TODO
## Load Config
config = litellm.ClarifaiConfig.get_config()
for k, v in config.items():
- if (
- k not in optional_params
- ):
+ if k not in optional_params:
optional_params[k] = v
custom_llm_provider, orig_model_name = get_prompt_model_name(model)
@@ -223,14 +217,14 @@ def completion(
model=orig_model_name,
messages=messages,
api_key=api_key,
- custom_llm_provider="clarifai"
+ custom_llm_provider="clarifai",
)
else:
prompt = prompt_factory(
model=orig_model_name,
messages=messages,
api_key=api_key,
- custom_llm_provider=custom_llm_provider
+ custom_llm_provider=custom_llm_provider,
)
# print(prompt); exit(0)
@@ -240,7 +234,6 @@ def completion(
}
data = completions_to_model(data)
-
## LOGGING
logging_obj.pre_call(
input=prompt,
@@ -251,7 +244,7 @@ def completion(
"api_base": api_base,
},
)
- if acompletion==True:
+ if acompletion == True:
return async_completion(
model=model,
prompt=prompt,
@@ -271,15 +264,17 @@ def completion(
else:
## COMPLETION CALL
response = requests.post(
- model,
- headers=headers,
- data=json.dumps(data),
- )
+ model,
+ headers=headers,
+ data=json.dumps(data),
+ )
# print(response.content); exit()
if response.status_code != 200:
- raise ClarifaiError(status_code=response.status_code, message=response.text, url=model)
-
+ raise ClarifaiError(
+ status_code=response.status_code, message=response.text, url=model
+ )
+
if "stream" in optional_params and optional_params["stream"] == True:
completion_stream = response.iter_lines()
stream_response = CustomStreamWrapper(
@@ -287,11 +282,11 @@ def completion(
model=model,
custom_llm_provider="clarifai",
logging_obj=logging_obj,
- )
+ )
return stream_response
-
+
else:
- return process_response(
+ return process_response(
model=model,
prompt=prompt,
response=response,
@@ -299,8 +294,9 @@ def completion(
api_key=api_key,
data=data,
encoding=encoding,
- logging_obj=logging_obj)
-
+ logging_obj=logging_obj,
+ )
+
class ModelResponseIterator:
def __init__(self, model_response):
@@ -325,4 +321,4 @@ class ModelResponseIterator:
if self.is_done:
raise StopAsyncIteration
self.is_done = True
- return self.model_response
\ No newline at end of file
+ return self.model_response
diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py
index 0adbd95bf90..4df25944b87 100644
--- a/litellm/llms/custom_httpx/http_handler.py
+++ b/litellm/llms/custom_httpx/http_handler.py
@@ -7,8 +7,12 @@ _DEFAULT_TIMEOUT = httpx.Timeout(timeout=5.0, connect=5.0)
class AsyncHTTPHandler:
def __init__(
- self, timeout: httpx.Timeout = _DEFAULT_TIMEOUT, concurrent_limit=1000
+ self,
+ timeout: Optional[Union[float, httpx.Timeout]] = None,
+ concurrent_limit=1000,
):
+ if timeout is None:
+ timeout = _DEFAULT_TIMEOUT
# Create a client with a connection pool
self.client = httpx.AsyncClient(
timeout=timeout,
@@ -59,7 +63,7 @@ class AsyncHTTPHandler:
class HTTPHandler:
def __init__(
self,
- timeout: Optional[httpx.Timeout] = None,
+ timeout: Optional[Union[float, httpx.Timeout]] = None,
concurrent_limit=1000,
client: Optional[httpx.Client] = None,
):
diff --git a/litellm/llms/databricks.py b/litellm/llms/databricks.py
new file mode 100644
index 00000000000..7b2013710ea
--- /dev/null
+++ b/litellm/llms/databricks.py
@@ -0,0 +1,696 @@
+# What is this?
+## Handler file for databricks API https://docs.databricks.com/en/machine-learning/foundation-models/api-reference.html#chat-request
+import os, types
+import json
+from enum import Enum
+import requests, copy # type: ignore
+import time
+from typing import Callable, Optional, List, Union, Tuple, Literal
+from litellm.utils import (
+ ModelResponse,
+ Usage,
+ map_finish_reason,
+ CustomStreamWrapper,
+ EmbeddingResponse,
+)
+import litellm
+from .prompt_templates.factory import prompt_factory, custom_prompt
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
+from .base import BaseLLM
+import httpx # type: ignore
+from litellm.types.llms.databricks import GenericStreamingChunk
+from litellm.types.utils import ProviderField
+
+
+class DatabricksError(Exception):
+ def __init__(self, status_code, message):
+ self.status_code = status_code
+ self.message = message
+ self.request = httpx.Request(method="POST", url="https://docs.databricks.com/")
+ self.response = httpx.Response(status_code=status_code, request=self.request)
+ super().__init__(
+ self.message
+ ) # Call the base class constructor with the parameters it needs
+
+
+class DatabricksConfig:
+ """
+ Reference: https://docs.databricks.com/en/machine-learning/foundation-models/api-reference.html#chat-request
+ """
+
+ max_tokens: Optional[int] = None
+ temperature: Optional[int] = None
+ top_p: Optional[int] = None
+ top_k: Optional[int] = None
+ stop: Optional[Union[List[str], str]] = None
+ n: Optional[int] = None
+
+ def __init__(
+ self,
+ max_tokens: Optional[int] = None,
+ temperature: Optional[int] = None,
+ top_p: Optional[int] = None,
+ top_k: Optional[int] = None,
+ stop: Optional[Union[List[str], str]] = None,
+ n: Optional[int] = None,
+ ) -> None:
+ locals_ = locals()
+ for key, value in locals_.items():
+ if key != "self" and value is not None:
+ setattr(self.__class__, key, value)
+
+ @classmethod
+ def get_config(cls):
+ return {
+ k: v
+ for k, v in cls.__dict__.items()
+ if not k.startswith("__")
+ and not isinstance(
+ v,
+ (
+ types.FunctionType,
+ types.BuiltinFunctionType,
+ classmethod,
+ staticmethod,
+ ),
+ )
+ and v is not None
+ }
+
+ def get_required_params(self) -> List[ProviderField]:
+ """For a given provider, return it's required fields with a description"""
+ return [
+ ProviderField(
+ field_name="api_key",
+ field_type="string",
+ field_description="Your Databricks API Key.",
+ field_value="dapi...",
+ ),
+ ProviderField(
+ field_name="api_base",
+ field_type="string",
+ field_description="Your Databricks API Base.",
+ field_value="https://adb-..",
+ ),
+ ]
+
+ def get_supported_openai_params(self):
+ return ["stream", "stop", "temperature", "top_p", "max_tokens", "n"]
+
+ def map_openai_params(self, non_default_params: dict, optional_params: dict):
+ for param, value in non_default_params.items():
+ if param == "max_tokens":
+ optional_params["max_tokens"] = value
+ if param == "n":
+ optional_params["n"] = value
+ if param == "stream" and value == True:
+ optional_params["stream"] = value
+ if param == "temperature":
+ optional_params["temperature"] = value
+ if param == "top_p":
+ optional_params["top_p"] = value
+ if param == "stop":
+ optional_params["stop"] = value
+ return optional_params
+
+ def _chunk_parser(self, chunk_data: str) -> GenericStreamingChunk:
+ try:
+ text = ""
+ is_finished = False
+ finish_reason = None
+ logprobs = None
+ usage = None
+ original_chunk = None # this is used for function/tool calling
+ chunk_data = chunk_data.replace("data:", "")
+ chunk_data = chunk_data.strip()
+ if len(chunk_data) == 0:
+ return {
+ "text": "",
+ "is_finished": is_finished,
+ "finish_reason": finish_reason,
+ }
+ chunk_data_dict = json.loads(chunk_data)
+ str_line = litellm.ModelResponse(**chunk_data_dict, stream=True)
+
+ if len(str_line.choices) > 0:
+ if (
+ str_line.choices[0].delta is not None # type: ignore
+ and str_line.choices[0].delta.content is not None # type: ignore
+ ):
+ text = str_line.choices[0].delta.content # type: ignore
+ else: # function/tool calling chunk - when content is None. in this case we just return the original chunk from openai
+ original_chunk = str_line
+ if str_line.choices[0].finish_reason:
+ is_finished = True
+ finish_reason = str_line.choices[0].finish_reason
+ if finish_reason == "content_filter":
+ if hasattr(str_line.choices[0], "content_filter_result"):
+ error_message = json.dumps(
+ str_line.choices[0].content_filter_result # type: ignore
+ )
+ else:
+ error_message = "Azure Response={}".format(
+ str(dict(str_line))
+ )
+ raise litellm.AzureOpenAIError(
+ status_code=400, message=error_message
+ )
+
+ # checking for logprobs
+ if (
+ hasattr(str_line.choices[0], "logprobs")
+ and str_line.choices[0].logprobs is not None
+ ):
+ logprobs = str_line.choices[0].logprobs
+ else:
+ logprobs = None
+
+ usage = getattr(str_line, "usage", None)
+
+ return GenericStreamingChunk(
+ text=text,
+ is_finished=is_finished,
+ finish_reason=finish_reason,
+ logprobs=logprobs,
+ original_chunk=original_chunk,
+ usage=usage,
+ )
+ except Exception as e:
+ raise e
+
+
+class DatabricksEmbeddingConfig:
+ """
+ Reference: https://learn.microsoft.com/en-us/azure/databricks/machine-learning/foundation-models/api-reference#--embedding-task
+ """
+
+ instruction: Optional[str] = (
+ None # An optional instruction to pass to the embedding model. BGE Authors recommend 'Represent this sentence for searching relevant passages:' for retrieval queries
+ )
+
+ def __init__(self, instruction: Optional[str] = None) -> None:
+ locals_ = locals()
+ for key, value in locals_.items():
+ if key != "self" and value is not None:
+ setattr(self.__class__, key, value)
+
+ @classmethod
+ def get_config(cls):
+ return {
+ k: v
+ for k, v in cls.__dict__.items()
+ if not k.startswith("__")
+ and not isinstance(
+ v,
+ (
+ types.FunctionType,
+ types.BuiltinFunctionType,
+ classmethod,
+ staticmethod,
+ ),
+ )
+ and v is not None
+ }
+
+ def get_supported_openai_params(
+ self,
+ ): # no optional openai embedding params supported
+ return []
+
+ def map_openai_params(self, non_default_params: dict, optional_params: dict):
+ return optional_params
+
+
+class DatabricksChatCompletion(BaseLLM):
+ def __init__(self) -> None:
+ super().__init__()
+
+ # makes headers for API call
+
+ def _validate_environment(
+ self,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ endpoint_type: Literal["chat_completions", "embeddings"],
+ ) -> Tuple[str, dict]:
+ if api_key is None:
+ raise DatabricksError(
+ status_code=400,
+ message="Missing Databricks API Key - A call is being made to Databricks but no key is set either in the environment variables (DATABRICKS_API_KEY) or via params",
+ )
+
+ if api_base is None:
+ raise DatabricksError(
+ status_code=400,
+ message="Missing Databricks API Base - A call is being made to Databricks but no api base is set either in the environment variables (DATABRICKS_API_BASE) or via params",
+ )
+
+ headers = {
+ "Authorization": "Bearer {}".format(api_key),
+ "Content-Type": "application/json",
+ }
+
+ if endpoint_type == "chat_completions":
+ api_base = "{}/chat/completions".format(api_base)
+ elif endpoint_type == "embeddings":
+ api_base = "{}/embeddings".format(api_base)
+ return api_base, headers
+
+ def process_response(
+ self,
+ model: str,
+ response: Union[requests.Response, httpx.Response],
+ model_response: ModelResponse,
+ stream: bool,
+ logging_obj: litellm.utils.Logging,
+ optional_params: dict,
+ api_key: str,
+ data: Union[dict, str],
+ messages: List,
+ print_verbose,
+ encoding,
+ ) -> ModelResponse:
+ ## LOGGING
+ logging_obj.post_call(
+ input=messages,
+ api_key=api_key,
+ original_response=response.text,
+ additional_args={"complete_input_dict": data},
+ )
+ print_verbose(f"raw model_response: {response.text}")
+ ## RESPONSE OBJECT
+ try:
+ completion_response = response.json()
+ except:
+ raise DatabricksError(
+ message=response.text, status_code=response.status_code
+ )
+ if "error" in completion_response:
+ raise DatabricksError(
+ message=str(completion_response["error"]),
+ status_code=response.status_code,
+ )
+ else:
+ text_content = ""
+ tool_calls = []
+ for content in completion_response["content"]:
+ if content["type"] == "text":
+ text_content += content["text"]
+ ## TOOL CALLING
+ elif content["type"] == "tool_use":
+ tool_calls.append(
+ {
+ "id": content["id"],
+ "type": "function",
+ "function": {
+ "name": content["name"],
+ "arguments": json.dumps(content["input"]),
+ },
+ }
+ )
+
+ _message = litellm.Message(
+ tool_calls=tool_calls,
+ content=text_content or None,
+ )
+ model_response.choices[0].message = _message # type: ignore
+ model_response._hidden_params["original_response"] = completion_response[
+ "content"
+ ] # allow user to access raw anthropic tool calling response
+
+ model_response.choices[0].finish_reason = map_finish_reason(
+ completion_response["stop_reason"]
+ )
+
+ ## CALCULATING USAGE
+ prompt_tokens = completion_response["usage"]["input_tokens"]
+ completion_tokens = completion_response["usage"]["output_tokens"]
+ total_tokens = prompt_tokens + completion_tokens
+
+ model_response["created"] = int(time.time())
+ model_response["model"] = model
+ usage = Usage(
+ prompt_tokens=prompt_tokens,
+ completion_tokens=completion_tokens,
+ total_tokens=total_tokens,
+ )
+ setattr(model_response, "usage", usage) # type: ignore
+ return model_response
+
+ async def acompletion_stream_function(
+ self,
+ model: str,
+ messages: list,
+ api_base: str,
+ custom_prompt_dict: dict,
+ model_response: ModelResponse,
+ print_verbose: Callable,
+ encoding,
+ api_key,
+ logging_obj,
+ stream,
+ data: dict,
+ optional_params=None,
+ litellm_params=None,
+ logger_fn=None,
+ headers={},
+ ):
+ self.async_handler = AsyncHTTPHandler(
+ timeout=httpx.Timeout(timeout=600.0, connect=5.0)
+ )
+ data["stream"] = True
+ try:
+ response = await self.async_handler.post(
+ api_base, headers=headers, data=json.dumps(data), stream=True
+ )
+ response.raise_for_status()
+
+ completion_stream = response.aiter_lines()
+ except httpx.HTTPStatusError as e:
+ raise DatabricksError(
+ status_code=e.response.status_code, message=response.text
+ )
+ except httpx.TimeoutException as e:
+ raise DatabricksError(status_code=408, message="Timeout error occurred.")
+ except Exception as e:
+ raise DatabricksError(status_code=500, message=str(e))
+
+ streamwrapper = CustomStreamWrapper(
+ completion_stream=completion_stream,
+ model=model,
+ custom_llm_provider="databricks",
+ logging_obj=logging_obj,
+ )
+ return streamwrapper
+
+ async def acompletion_function(
+ self,
+ model: str,
+ messages: list,
+ api_base: str,
+ custom_prompt_dict: dict,
+ model_response: ModelResponse,
+ print_verbose: Callable,
+ encoding,
+ api_key,
+ logging_obj,
+ stream,
+ data: dict,
+ optional_params: dict,
+ litellm_params=None,
+ logger_fn=None,
+ headers={},
+ timeout: Optional[Union[float, httpx.Timeout]] = None,
+ ) -> ModelResponse:
+ if timeout is None:
+ timeout = httpx.Timeout(timeout=600.0, connect=5.0)
+
+ self.async_handler = AsyncHTTPHandler(timeout=timeout)
+
+ try:
+ response = await self.async_handler.post(
+ api_base, headers=headers, data=json.dumps(data)
+ )
+ response.raise_for_status()
+
+ response_json = response.json()
+ except httpx.HTTPStatusError as e:
+ raise DatabricksError(
+ status_code=e.response.status_code,
+ message=response.text if response else str(e),
+ )
+ except httpx.TimeoutException as e:
+ raise DatabricksError(status_code=408, message="Timeout error occurred.")
+ except Exception as e:
+ raise DatabricksError(status_code=500, message=str(e))
+
+ return ModelResponse(**response_json)
+
+ def completion(
+ self,
+ model: str,
+ messages: list,
+ api_base: str,
+ custom_prompt_dict: dict,
+ model_response: ModelResponse,
+ print_verbose: Callable,
+ encoding,
+ api_key,
+ logging_obj,
+ optional_params: dict,
+ acompletion=None,
+ litellm_params=None,
+ logger_fn=None,
+ headers={},
+ timeout: Optional[Union[float, httpx.Timeout]] = None,
+ client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
+ ):
+ api_base, headers = self._validate_environment(
+ api_base=api_base, api_key=api_key, endpoint_type="chat_completions"
+ )
+ ## Load Config
+ config = litellm.DatabricksConfig().get_config()
+ for k, v in config.items():
+ if (
+ k not in optional_params
+ ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in
+ optional_params[k] = v
+
+ stream = optional_params.pop("stream", None)
+
+ data = {
+ "model": model,
+ "messages": messages,
+ **optional_params,
+ }
+
+ ## LOGGING
+ logging_obj.pre_call(
+ input=messages,
+ api_key=api_key,
+ additional_args={
+ "complete_input_dict": data,
+ "api_base": api_base,
+ "headers": headers,
+ },
+ )
+ if acompletion == True:
+ if (
+ stream is not None and stream == True
+ ): # if function call - fake the streaming (need complete blocks for output parsing in openai format)
+ print_verbose("makes async anthropic streaming POST request")
+ data["stream"] = stream
+ return self.acompletion_stream_function(
+ model=model,
+ messages=messages,
+ data=data,
+ api_base=api_base,
+ custom_prompt_dict=custom_prompt_dict,
+ model_response=model_response,
+ print_verbose=print_verbose,
+ encoding=encoding,
+ api_key=api_key,
+ logging_obj=logging_obj,
+ optional_params=optional_params,
+ stream=stream,
+ litellm_params=litellm_params,
+ logger_fn=logger_fn,
+ headers=headers,
+ )
+ else:
+ return self.acompletion_function(
+ model=model,
+ messages=messages,
+ data=data,
+ api_base=api_base,
+ custom_prompt_dict=custom_prompt_dict,
+ model_response=model_response,
+ print_verbose=print_verbose,
+ encoding=encoding,
+ api_key=api_key,
+ logging_obj=logging_obj,
+ optional_params=optional_params,
+ stream=stream,
+ litellm_params=litellm_params,
+ logger_fn=logger_fn,
+ headers=headers,
+ timeout=timeout,
+ )
+ else:
+ if client is None or isinstance(client, AsyncHTTPHandler):
+ self.client = HTTPHandler(timeout=timeout) # type: ignore
+ else:
+ self.client = client
+ ## COMPLETION CALL
+ if (
+ stream is not None and stream == True
+ ): # if function call - fake the streaming (need complete blocks for output parsing in openai format)
+ print_verbose("makes dbrx streaming POST request")
+ data["stream"] = stream
+ try:
+ response = self.client.post(
+ api_base, headers=headers, data=json.dumps(data), stream=stream
+ )
+ response.raise_for_status()
+ completion_stream = response.iter_lines()
+ except httpx.HTTPStatusError as e:
+ raise DatabricksError(
+ status_code=e.response.status_code, message=response.text
+ )
+ except httpx.TimeoutException as e:
+ raise DatabricksError(
+ status_code=408, message="Timeout error occurred."
+ )
+ except Exception as e:
+ raise DatabricksError(status_code=408, message=str(e))
+
+ streaming_response = CustomStreamWrapper(
+ completion_stream=completion_stream,
+ model=model,
+ custom_llm_provider="databricks",
+ logging_obj=logging_obj,
+ )
+ return streaming_response
+
+ else:
+ try:
+ response = self.client.post(
+ api_base, headers=headers, data=json.dumps(data)
+ )
+ response.raise_for_status()
+
+ response_json = response.json()
+ except httpx.HTTPStatusError as e:
+ raise DatabricksError(
+ status_code=e.response.status_code, message=response.text
+ )
+ except httpx.TimeoutException as e:
+ raise DatabricksError(
+ status_code=408, message="Timeout error occurred."
+ )
+ except Exception as e:
+ raise DatabricksError(status_code=500, message=str(e))
+
+ return ModelResponse(**response_json)
+
+ async def aembedding(
+ self,
+ input: list,
+ data: dict,
+ model_response: ModelResponse,
+ timeout: float,
+ api_key: str,
+ api_base: str,
+ logging_obj,
+ headers: dict,
+ client=None,
+ ) -> EmbeddingResponse:
+ response = None
+ try:
+ if client is None or isinstance(client, AsyncHTTPHandler):
+ self.async_client = AsyncHTTPHandler(timeout=timeout) # type: ignore
+ else:
+ self.async_client = client
+
+ try:
+ response = await self.async_client.post(
+ api_base,
+ headers=headers,
+ data=json.dumps(data),
+ ) # type: ignore
+
+ response.raise_for_status()
+
+ response_json = response.json()
+ except httpx.HTTPStatusError as e:
+ raise DatabricksError(
+ status_code=e.response.status_code,
+ message=response.text if response else str(e),
+ )
+ except httpx.TimeoutException as e:
+ raise DatabricksError(
+ status_code=408, message="Timeout error occurred."
+ )
+ except Exception as e:
+ raise DatabricksError(status_code=500, message=str(e))
+
+ ## LOGGING
+ logging_obj.post_call(
+ input=input,
+ api_key=api_key,
+ additional_args={"complete_input_dict": data},
+ original_response=response_json,
+ )
+ return EmbeddingResponse(**response_json)
+ except Exception as e:
+ ## LOGGING
+ logging_obj.post_call(
+ input=input,
+ api_key=api_key,
+ original_response=str(e),
+ )
+ raise e
+
+ def embedding(
+ self,
+ model: str,
+ input: list,
+ timeout: float,
+ logging_obj,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ optional_params: dict,
+ model_response: Optional[litellm.utils.EmbeddingResponse] = None,
+ client=None,
+ aembedding=None,
+ ) -> EmbeddingResponse:
+ api_base, headers = self._validate_environment(
+ api_base=api_base, api_key=api_key, endpoint_type="embeddings"
+ )
+ model = model
+ data = {"model": model, "input": input, **optional_params}
+
+ ## LOGGING
+ logging_obj.pre_call(
+ input=input,
+ api_key=api_key,
+ additional_args={"complete_input_dict": data, "api_base": api_base},
+ )
+
+ if aembedding == True:
+ return self.aembedding(data=data, input=input, logging_obj=logging_obj, model_response=model_response, api_base=api_base, api_key=api_key, timeout=timeout, client=client, headers=headers) # type: ignore
+ if client is None or isinstance(client, AsyncHTTPHandler):
+ self.client = HTTPHandler(timeout=timeout) # type: ignore
+ else:
+ self.client = client
+
+ ## EMBEDDING CALL
+ try:
+ response = self.client.post(
+ api_base,
+ headers=headers,
+ data=json.dumps(data),
+ ) # type: ignore
+
+ response.raise_for_status() # type: ignore
+
+ response_json = response.json() # type: ignore
+ except httpx.HTTPStatusError as e:
+ raise DatabricksError(
+ status_code=e.response.status_code,
+ message=response.text if response else str(e),
+ )
+ except httpx.TimeoutException as e:
+ raise DatabricksError(status_code=408, message="Timeout error occurred.")
+ except Exception as e:
+ raise DatabricksError(status_code=500, message=str(e))
+
+ ## LOGGING
+ logging_obj.post_call(
+ input=input,
+ api_key=api_key,
+ additional_args={"complete_input_dict": data},
+ original_response=response_json,
+ )
+
+ return litellm.EmbeddingResponse(**response_json)
diff --git a/litellm/llms/openai.py b/litellm/llms/openai.py
index 9d143f5d9a5..2e0196faa3a 100644
--- a/litellm/llms/openai.py
+++ b/litellm/llms/openai.py
@@ -404,6 +404,7 @@ class OpenAIChatCompletion(BaseLLM):
self,
model_response: ModelResponse,
timeout: Union[float, httpx.Timeout],
+ optional_params: dict,
model: Optional[str] = None,
messages: Optional[list] = None,
print_verbose: Optional[Callable] = None,
@@ -411,7 +412,6 @@ class OpenAIChatCompletion(BaseLLM):
api_base: Optional[str] = None,
acompletion: bool = False,
logging_obj=None,
- optional_params=None,
litellm_params=None,
logger_fn=None,
headers: Optional[dict] = None,
@@ -795,10 +795,10 @@ class OpenAIChatCompletion(BaseLLM):
model: str,
input: list,
timeout: float,
+ logging_obj,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
model_response: Optional[litellm.utils.EmbeddingResponse] = None,
- logging_obj=None,
optional_params=None,
client=None,
aembedding=None,
diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py
index e6e8ef50eca..41ecb486ce7 100644
--- a/litellm/llms/prompt_templates/factory.py
+++ b/litellm/llms/prompt_templates/factory.py
@@ -115,6 +115,26 @@ def llama_2_chat_pt(messages):
return prompt
+def convert_to_ollama_image(openai_image_url: str):
+ try:
+ if openai_image_url.startswith("http"):
+ openai_image_url = convert_url_to_base64(url=openai_image_url)
+
+ if openai_image_url.startswith("data:image/"):
+ # Extract the base64 image data
+ base64_data = openai_image_url.split("data:image/")[1].split(";base64,")[1]
+ else:
+ base64_data = openai_image_url
+
+ return base64_data
+ except Exception as e:
+ if "Error: Unable to fetch image from URL" in str(e):
+ raise e
+ raise Exception(
+ """Image url not in expected format. Example Expected input - "image_url": "data:image/jpeg;base64,{base64_image}". """
+ )
+
+
def ollama_pt(
model, messages
): # https://github.com/ollama/ollama/blob/af4cf55884ac54b9e637cd71dadfe9b7a5685877/docs/modelfile.md#template
@@ -147,8 +167,10 @@ def ollama_pt(
if element["type"] == "text":
prompt += element["text"]
elif element["type"] == "image_url":
- image_url = element["image_url"]["url"]
- images.append(image_url)
+ base64_image = convert_to_ollama_image(
+ element["image_url"]["url"]
+ )
+ images.append(base64_image)
return {"prompt": prompt, "images": images}
else:
prompt = "".join(
@@ -1509,11 +1531,12 @@ def _gemini_vision_convert_messages(messages: list):
raise Exception(
"gemini image conversion failed please run `pip install Pillow`"
)
-
+
if "base64" in img:
# Case 2: Base64 image data
import base64
import io
+
# Extract the base64 image data
base64_data = img.split("base64,")[1]
diff --git a/litellm/llms/vertex_ai.py b/litellm/llms/vertex_ai.py
index b52e8689f10..dc185aef9de 100644
--- a/litellm/llms/vertex_ai.py
+++ b/litellm/llms/vertex_ai.py
@@ -376,17 +376,31 @@ def _gemini_convert_messages_with_history(messages: list) -> List[ContentType]:
assistant_content = []
## MERGE CONSECUTIVE ASSISTANT CONTENT ##
while msg_i < len(messages) and messages[msg_i]["role"] == "assistant":
- assistant_text = (
- messages[msg_i].get("content") or ""
- ) # either string or none
- if assistant_text:
- assistant_content.append(PartType(text=assistant_text))
- if messages[msg_i].get(
+ if isinstance(messages[msg_i]["content"], list):
+ _parts = []
+ for element in messages[msg_i]["content"]:
+ if isinstance(element, dict):
+ if element["type"] == "text":
+ _part = PartType(text=element["text"])
+ _parts.append(_part)
+ elif element["type"] == "image_url":
+ image_url = element["image_url"]["url"]
+ _part = _process_gemini_image(image_url=image_url)
+ _parts.append(_part) # type: ignore
+ assistant_content.extend(_parts)
+ elif messages[msg_i].get(
"tool_calls", []
): # support assistant tool invoke convertion
assistant_content.extend(
convert_to_gemini_tool_call_invoke(messages[msg_i]["tool_calls"])
)
+ else:
+ assistant_text = (
+ messages[msg_i].get("content") or ""
+ ) # either string or none
+ if assistant_text:
+ assistant_content.append(PartType(text=assistant_text))
+
msg_i += 1
if assistant_content:
diff --git a/litellm/llms/vertex_ai_anthropic.py b/litellm/llms/vertex_ai_anthropic.py
index 3bdcf4fd612..0652942801c 100644
--- a/litellm/llms/vertex_ai_anthropic.py
+++ b/litellm/llms/vertex_ai_anthropic.py
@@ -35,7 +35,7 @@ class VertexAIError(Exception):
class VertexAIAnthropicConfig:
"""
- Reference: https://docs.anthropic.com/claude/reference/messages_post
+ Reference:https://docs.anthropic.com/claude/reference/messages_post
Note that the API for Claude on Vertex differs from the Anthropic API documentation in the following ways:
diff --git a/litellm/main.py b/litellm/main.py
index 42c4eb8ff3d..37fc1db8f67 100644
--- a/litellm/main.py
+++ b/litellm/main.py
@@ -73,6 +73,7 @@ from .llms import (
)
from .llms.openai import OpenAIChatCompletion, OpenAITextCompletion
from .llms.azure import AzureChatCompletion
+from .llms.databricks import DatabricksChatCompletion
from .llms.azure_text import AzureTextCompletion
from .llms.anthropic import AnthropicChatCompletion
from .llms.anthropic_text import AnthropicTextCompletion
@@ -111,6 +112,7 @@ from litellm.utils import (
####### ENVIRONMENT VARIABLES ###################
openai_chat_completions = OpenAIChatCompletion()
openai_text_completions = OpenAITextCompletion()
+databricks_chat_completions = DatabricksChatCompletion()
anthropic_chat_completions = AnthropicChatCompletion()
anthropic_text_completions = AnthropicTextCompletion()
azure_chat_completions = AzureChatCompletion()
@@ -329,6 +331,7 @@ async def acompletion(
or custom_llm_provider == "anthropic"
or custom_llm_provider == "predibase"
or custom_llm_provider == "bedrock"
+ or custom_llm_provider == "databricks"
or custom_llm_provider in litellm.openai_compatible_providers
): # currently implemented aiohttp calls for just azure, openai, hf, ollama, vertex ai soon all.
init_response = await loop.run_in_executor(None, func_with_context)
@@ -417,6 +420,8 @@ def mock_completion(
api_key="mock-key",
)
if isinstance(mock_response, Exception):
+ if isinstance(mock_response, openai.APIError):
+ raise mock_response
raise litellm.APIError(
status_code=500, # type: ignore
message=str(mock_response),
@@ -460,7 +465,9 @@ def mock_completion(
return model_response
- except:
+ except Exception as e:
+ if isinstance(e, openai.APIError):
+ raise e
traceback.print_exc()
raise Exception("Mock completion response failed")
@@ -861,6 +868,7 @@ def completion(
user=user,
optional_params=optional_params,
litellm_params=litellm_params,
+ custom_llm_provider=custom_llm_provider,
)
if mock_response:
return mock_completion(
@@ -1615,6 +1623,61 @@ def completion(
)
return response
response = model_response
+ elif custom_llm_provider == "databricks":
+ api_base = (
+ api_base # for databricks we check in get_llm_provider and pass in the api base from there
+ or litellm.api_base
+ or os.getenv("DATABRICKS_API_BASE")
+ )
+
+ # set API KEY
+ api_key = (
+ api_key
+ or litellm.api_key # for databricks we check in get_llm_provider and pass in the api key from there
+ or litellm.databricks_key
+ or get_secret("DATABRICKS_API_KEY")
+ )
+
+ headers = headers or litellm.headers
+
+ ## COMPLETION CALL
+ try:
+ response = databricks_chat_completions.completion(
+ model=model,
+ messages=messages,
+ headers=headers,
+ model_response=model_response,
+ print_verbose=print_verbose,
+ api_key=api_key,
+ api_base=api_base,
+ acompletion=acompletion,
+ logging_obj=logging,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ logger_fn=logger_fn,
+ timeout=timeout, # type: ignore
+ custom_prompt_dict=custom_prompt_dict,
+ client=client, # pass AsyncOpenAI, OpenAI client
+ encoding=encoding,
+ )
+ except Exception as e:
+ ## LOGGING - log the original exception returned
+ logging.post_call(
+ input=messages,
+ api_key=api_key,
+ original_response=str(e),
+ additional_args={"headers": headers},
+ )
+ raise e
+
+ if optional_params.get("stream", False):
+ ## LOGGING
+ logging.post_call(
+ input=messages,
+ api_key=api_key,
+ original_response=response,
+ additional_args={"headers": headers},
+ )
elif custom_llm_provider == "openrouter":
api_base = api_base or litellm.api_base or "https://openrouter.ai/api/v1"
@@ -2036,6 +2099,7 @@ def completion(
extra_headers=extra_headers,
timeout=timeout,
acompletion=acompletion,
+ client=client,
)
if optional_params.get("stream", False):
## LOGGING
@@ -2477,6 +2541,7 @@ def batch_completion(
list: A list of completion results.
"""
args = locals()
+
batch_messages = messages
completions = []
model = model
@@ -2530,7 +2595,15 @@ def batch_completion(
completions.append(future)
# Retrieve the results from the futures
- results = [future.result() for future in completions]
+ # results = [future.result() for future in completions]
+ # return exceptions if any
+ results = []
+ for future in completions:
+ try:
+ results.append(future.result())
+ except Exception as exc:
+ results.append(exc)
+
return results
@@ -2669,7 +2742,7 @@ def batch_completion_models_all_responses(*args, **kwargs):
### EMBEDDING ENDPOINTS ####################
@client
-async def aembedding(*args, **kwargs):
+async def aembedding(*args, **kwargs) -> EmbeddingResponse:
"""
Asynchronously calls the `embedding` function with the given arguments and keyword arguments.
@@ -2714,12 +2787,13 @@ async def aembedding(*args, **kwargs):
or custom_llm_provider == "fireworks_ai"
or custom_llm_provider == "ollama"
or custom_llm_provider == "vertex_ai"
+ or custom_llm_provider == "databricks"
): # currently implemented aiohttp calls for just azure and openai, soon all.
# Await normally
init_response = await loop.run_in_executor(None, func_with_context)
- if isinstance(init_response, dict) or isinstance(
- init_response, ModelResponse
- ): ## CACHING SCENARIO
+ if isinstance(init_response, dict):
+ response = EmbeddingResponse(**init_response)
+ elif isinstance(init_response, EmbeddingResponse): ## CACHING SCENARIO
response = init_response
elif asyncio.iscoroutine(init_response):
response = await init_response
@@ -2759,7 +2833,7 @@ def embedding(
litellm_logging_obj=None,
logger_fn=None,
**kwargs,
-):
+) -> EmbeddingResponse:
"""
Embedding function that calls an API to generate embeddings for the given input.
@@ -2907,7 +2981,7 @@ def embedding(
)
try:
response = None
- logging = litellm_logging_obj
+ logging: Logging = litellm_logging_obj # type: ignore
logging.update_environment_variables(
model=model,
user=user,
@@ -2997,6 +3071,32 @@ def embedding(
client=client,
aembedding=aembedding,
)
+ elif custom_llm_provider == "databricks":
+ api_base = (
+ api_base or litellm.api_base or get_secret("DATABRICKS_API_BASE")
+ ) # type: ignore
+
+ # set API KEY
+ api_key = (
+ api_key
+ or litellm.api_key
+ or litellm.databricks_key
+ or get_secret("DATABRICKS_API_KEY")
+ ) # type: ignore
+
+ ## EMBEDDING CALL
+ response = databricks_chat_completions.embedding(
+ model=model,
+ input=input,
+ api_base=api_base,
+ api_key=api_key,
+ logging_obj=logging,
+ timeout=timeout,
+ model_response=EmbeddingResponse(),
+ optional_params=optional_params,
+ client=client,
+ aembedding=aembedding,
+ )
elif custom_llm_provider == "cohere":
cohere_key = (
api_key
diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index bede36764e9..aab9c9af172 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -1272,6 +1272,12 @@
"supports_function_calling": true,
"supports_vision": true
},
+ "vertex_ai/imagegeneration@006": {
+ "cost_per_image": 0.020,
+ "litellm_provider": "vertex_ai-image-models",
+ "mode": "image_generation",
+ "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing"
+ },
"textembedding-gecko": {
"max_tokens": 3072,
"max_input_tokens": 3072,
@@ -1599,36 +1605,36 @@
"mode": "chat"
},
"replicate/meta/llama-3-70b": {
- "max_tokens": 4096,
- "max_input_tokens": 4096,
- "max_output_tokens": 4096,
+ "max_tokens": 8192,
+ "max_input_tokens": 8192,
+ "max_output_tokens": 8192,
"input_cost_per_token": 0.00000065,
"output_cost_per_token": 0.00000275,
"litellm_provider": "replicate",
"mode": "chat"
},
"replicate/meta/llama-3-70b-instruct": {
- "max_tokens": 4096,
- "max_input_tokens": 4096,
- "max_output_tokens": 4096,
+ "max_tokens": 8192,
+ "max_input_tokens": 8192,
+ "max_output_tokens": 8192,
"input_cost_per_token": 0.00000065,
"output_cost_per_token": 0.00000275,
"litellm_provider": "replicate",
"mode": "chat"
},
"replicate/meta/llama-3-8b": {
- "max_tokens": 4096,
- "max_input_tokens": 4096,
- "max_output_tokens": 4096,
+ "max_tokens": 8086,
+ "max_input_tokens": 8086,
+ "max_output_tokens": 8086,
"input_cost_per_token": 0.00000005,
"output_cost_per_token": 0.00000025,
"litellm_provider": "replicate",
"mode": "chat"
},
"replicate/meta/llama-3-8b-instruct": {
- "max_tokens": 4096,
- "max_input_tokens": 4096,
- "max_output_tokens": 4096,
+ "max_tokens": 8086,
+ "max_input_tokens": 8086,
+ "max_output_tokens": 8086,
"input_cost_per_token": 0.00000005,
"output_cost_per_token": 0.00000025,
"litellm_provider": "replicate",
@@ -1892,7 +1898,7 @@
"mode": "chat"
},
"openrouter/meta-llama/codellama-34b-instruct": {
- "max_tokens": 8096,
+ "max_tokens": 8192,
"input_cost_per_token": 0.0000005,
"output_cost_per_token": 0.0000005,
"litellm_provider": "openrouter",
@@ -3384,9 +3390,10 @@
"output_cost_per_token": 0.00000015,
"litellm_provider": "anyscale",
"mode": "chat",
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/mistralai-Mistral-7B-Instruct-v0.1"
},
- "anyscale/Mixtral-8x7B-Instruct-v0.1": {
+ "anyscale/mistralai/Mixtral-8x7B-Instruct-v0.1": {
"max_tokens": 16384,
"max_input_tokens": 16384,
"max_output_tokens": 16384,
@@ -3394,7 +3401,19 @@
"output_cost_per_token": 0.00000015,
"litellm_provider": "anyscale",
"mode": "chat",
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/mistralai-Mixtral-8x7B-Instruct-v0.1"
+ },
+ "anyscale/mistralai/Mixtral-8x22B-Instruct-v0.1": {
+ "max_tokens": 65536,
+ "max_input_tokens": 65536,
+ "max_output_tokens": 65536,
+ "input_cost_per_token": 0.00000090,
+ "output_cost_per_token": 0.00000090,
+ "litellm_provider": "anyscale",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/mistralai-Mixtral-8x22B-Instruct-v0.1"
},
"anyscale/HuggingFaceH4/zephyr-7b-beta": {
"max_tokens": 16384,
@@ -3405,6 +3424,16 @@
"litellm_provider": "anyscale",
"mode": "chat"
},
+ "anyscale/google/gemma-7b-it": {
+ "max_tokens": 8192,
+ "max_input_tokens": 8192,
+ "max_output_tokens": 8192,
+ "input_cost_per_token": 0.00000015,
+ "output_cost_per_token": 0.00000015,
+ "litellm_provider": "anyscale",
+ "mode": "chat",
+ "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/google-gemma-7b-it"
+ },
"anyscale/meta-llama/Llama-2-7b-chat-hf": {
"max_tokens": 4096,
"max_input_tokens": 4096,
@@ -3441,6 +3470,36 @@
"litellm_provider": "anyscale",
"mode": "chat"
},
+ "anyscale/codellama/CodeLlama-70b-Instruct-hf": {
+ "max_tokens": 4096,
+ "max_input_tokens": 4096,
+ "max_output_tokens": 4096,
+ "input_cost_per_token": 0.000001,
+ "output_cost_per_token": 0.000001,
+ "litellm_provider": "anyscale",
+ "mode": "chat",
+ "source" : "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/codellama-CodeLlama-70b-Instruct-hf"
+ },
+ "anyscale/meta-llama/Meta-Llama-3-8B-Instruct": {
+ "max_tokens": 8192,
+ "max_input_tokens": 8192,
+ "max_output_tokens": 8192,
+ "input_cost_per_token": 0.00000015,
+ "output_cost_per_token": 0.00000015,
+ "litellm_provider": "anyscale",
+ "mode": "chat",
+ "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/meta-llama-Meta-Llama-3-8B-Instruct"
+ },
+ "anyscale/meta-llama/Meta-Llama-3-70B-Instruct": {
+ "max_tokens": 8192,
+ "max_input_tokens": 8192,
+ "max_output_tokens": 8192,
+ "input_cost_per_token": 0.00000100,
+ "output_cost_per_token": 0.00000100,
+ "litellm_provider": "anyscale",
+ "mode": "chat",
+ "source" : "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/meta-llama-Meta-Llama-3-70B-Instruct"
+ },
"cloudflare/@cf/meta/llama-2-7b-chat-fp16": {
"max_tokens": 3072,
"max_input_tokens": 3072,
@@ -3532,6 +3591,76 @@
"output_cost_per_token": 0.000000,
"litellm_provider": "voyage",
"mode": "embedding"
- }
+ },
+ "databricks/databricks-dbrx-instruct": {
+ "max_tokens": 32768,
+ "max_input_tokens": 32768,
+ "max_output_tokens": 32768,
+ "input_cost_per_token": 0.00000075,
+ "output_cost_per_token": 0.00000225,
+ "litellm_provider": "databricks",
+ "mode": "chat",
+ "source": "https://www.databricks.com/product/pricing/foundation-model-serving"
+ },
+ "databricks/databricks-meta-llama-3-70b-instruct": {
+ "max_tokens": 8192,
+ "max_input_tokens": 8192,
+ "max_output_tokens": 8192,
+ "input_cost_per_token": 0.000001,
+ "output_cost_per_token": 0.000003,
+ "litellm_provider": "databricks",
+ "mode": "chat",
+ "source": "https://www.databricks.com/product/pricing/foundation-model-serving"
+ },
+ "databricks/databricks-llama-2-70b-chat": {
+ "max_tokens": 4096,
+ "max_input_tokens": 4096,
+ "max_output_tokens": 4096,
+ "input_cost_per_token": 0.0000005,
+ "output_cost_per_token": 0.0000015,
+ "litellm_provider": "databricks",
+ "mode": "chat",
+ "source": "https://www.databricks.com/product/pricing/foundation-model-serving"
+ },
+ "databricks/databricks-mixtral-8x7b-instruct": {
+ "max_tokens": 4096,
+ "max_input_tokens": 4096,
+ "max_output_tokens": 4096,
+ "input_cost_per_token": 0.0000005,
+ "output_cost_per_token": 0.000001,
+ "litellm_provider": "databricks",
+ "mode": "chat",
+ "source": "https://www.databricks.com/product/pricing/foundation-model-serving"
+ },
+ "databricks/databricks-mpt-30b-instruct": {
+ "max_tokens": 8192,
+ "max_input_tokens": 8192,
+ "max_output_tokens": 8192,
+ "input_cost_per_token": 0.000001,
+ "output_cost_per_token": 0.000001,
+ "litellm_provider": "databricks",
+ "mode": "chat",
+ "source": "https://www.databricks.com/product/pricing/foundation-model-serving"
+ },
+ "databricks/databricks-mpt-7b-instruct": {
+ "max_tokens": 8192,
+ "max_input_tokens": 8192,
+ "max_output_tokens": 8192,
+ "input_cost_per_token": 0.0000005,
+ "output_cost_per_token": 0.0000005,
+ "litellm_provider": "databricks",
+ "mode": "chat",
+ "source": "https://www.databricks.com/product/pricing/foundation-model-serving"
+ },
+ "databricks/databricks-bge-large-en": {
+ "max_tokens": 512,
+ "max_input_tokens": 512,
+ "output_vector_size": 1024,
+ "input_cost_per_token": 0.0000001,
+ "output_cost_per_token": 0.0,
+ "litellm_provider": "databricks",
+ "mode": "embedding",
+ "source": "https://www.databricks.com/product/pricing/foundation-model-serving"
+ }
}
diff --git a/litellm/proxy/_experimental/out/404.html b/litellm/proxy/_experimental/out/404.html
index fa19572edc3..3716021f516 100644
--- a/litellm/proxy/_experimental/out/404.html
+++ b/litellm/proxy/_experimental/out/404.html
@@ -1 +1 @@
-
0?a=a.charAt(0)+"."+a.slice(1)+x(r):i>1&&(a=a.charAt(0)+"."+a.slice(1)),a=a+(o<0?"e":"e+")+o):o<0?(a="0."+x(-o-1)+a,n&&(r=n-i)>0&&(a+=x(r))):o>=i?(a+=x(o+1-i),n&&(r=n-o-1)>0&&(a=a+"."+x(r))):((r=o+1)0&&(o+1===i&&(a+="."),a+=x(r))),e.s<0?"-"+a:a}function N(e,t){if(e.length>t)return e.length=t,!0}function I(e){if(!e||"object"!=typeof e)throw Error(s+"Object expected");var t,n,r,o=["precision",1,1e9,"rounding",0,8,"toExpNeg",-1/0,0,"toExpPos",0,1/0];for(t=0;t239?4:c>223?3:c>191?2:1;if(o+d<=n)switch(d){case 1:c<128&&(u=c);break;case 2:(192&(a=e[o+1]))==128&&(s=(31&c)<<6|63&a)>127&&(u=s);break;case 3:a=e[o+1],i=e[o+2],(192&a)==128&&(192&i)==128&&(s=(15&c)<<12|(63&a)<<6|63&i)>2047&&(s<55296||s>57343)&&(u=s);break;case 4:a=e[o+1],i=e[o+2],l=e[o+3],(192&a)==128&&(192&i)==128&&(192&l)==128&&(s=(15&c)<<18|(63&a)<<12|(63&i)<<6|63&l)>65535&&s<1114112&&(u=s)}null===u?(u=65533,d=1):u>65535&&(u-=65536,r.push(u>>>10&1023|55296),u=56320|1023&u),r.push(u),o+=d}return function(e){var t=e.length;if(t<=4096)return String.fromCharCode.apply(String,e);for(var n="",r=0;r>>=0,isFinite(n)?(n>>>=0,void 0===r&&(r="utf8")):(r=n,n=void 0);else throw Error("Buffer.write(string, encoding, offset[, length]) is no longer supported");var o,a,i,l,s,c,u,d,p,f,m,g,h=this.length-t;if((void 0===n||n>h)&&(n=h),e.length>0&&(n<0||t<0)||t>this.length)throw RangeError("Attempt to write outside buffer bounds");r||(r="utf8");for(var b=!1;;)switch(r){case"hex":return function(e,t,n,r){n=Number(n)||0;var o=e.length-n;r?(r=Number(r))>o&&(r=o):r=o;var a=t.length;r>a/2&&(r=a/2);for(var i=0;i