From 9c3a1124d7bdcd448871e0fb6cdc8bcbc4086394 Mon Sep 17 00:00:00 2001 From: Vivek Aditya Date: Wed, 17 Apr 2024 00:06:16 +0530 Subject: [PATCH 001/192] Athina callback now handles streaming mode --- litellm/integrations/athina.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/litellm/integrations/athina.py b/litellm/integrations/athina.py index 897cf6c8d53..97d84c57433 100644 --- a/litellm/integrations/athina.py +++ b/litellm/integrations/athina.py @@ -1,6 +1,5 @@ import datetime - class AthinaLogger: def __init__(self): import os @@ -17,7 +16,18 @@ class AthinaLogger: import json import traceback try: - response_json = response_obj.model_dump() if response_obj else {} + is_stream = kwargs.get("stream", False) + if is_stream: + if "complete_streaming_response" in kwargs: + # Log the completion response in streaming mode + completion_response = kwargs["complete_streaming_response"] + response_json = completion_response.model_dump() if completion_response else {} + else: + # Skip logging if the completion response is not available + return + else: + # Log the completion response in non streaming mode + response_json = response_obj.model_dump() if response_obj else {} data = { "language_model_id": kwargs.get("model"), "request": kwargs, From 2713272bba2f109e75dda21427f875c198a5c370 Mon Sep 17 00:00:00 2001 From: ffreemt Date: Sun, 5 May 2024 13:11:21 +0800 Subject: [PATCH 002/192] Add return_exceptions to batch_completion (retry) --- litellm/main.py | 19 ++++++++++-- ...test_batch_completion_return_exceptions.py | 30 +++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 litellm/tests/test_batch_completion_return_exceptions.py diff --git a/litellm/main.py b/litellm/main.py index d19463f532f..826eb5c12e3 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -2166,7 +2166,7 @@ def completion( """ assume input to custom LLM api bases follow this format: resp = requests.post( - api_base, + api_base, json={ 'model': 'meta-llama/Llama-2-13b-hf', # model name 'params': { @@ -2331,6 +2331,12 @@ def batch_completion( list: A list of completion results. """ args = locals() + + # extra kw for dealing with exceptions + return_exceptions = args.get("kwargs").get("return_exceptions", False) + if "return_exceptions" in args.get("kwargs"): + args.get("kwargs").pop("return_exceptions") + batch_messages = messages completions = [] model = model @@ -2384,7 +2390,16 @@ def batch_completion( completions.append(future) # Retrieve the results from the futures - results = [future.result() for future in completions] + # results = [future.result() for future in completions] + if return_exceptions: + results = [] + for future in completions: + try: + results.append(future.result()) + except Exception as exc: + results.append(exc) + else: # original + results = [future.result() for future in completions] return results diff --git a/litellm/tests/test_batch_completion_return_exceptions.py b/litellm/tests/test_batch_completion_return_exceptions.py new file mode 100644 index 00000000000..df330f65d69 --- /dev/null +++ b/litellm/tests/test_batch_completion_return_exceptions.py @@ -0,0 +1,30 @@ +"""https://github.com/BerriAI/litellm/pull/3397/commits/a7ec1772b1457594d3af48cdcb0a382279b841c7#diff-44852387ceb00aade916d6b314dfd5d180499e54f35209ae9c07179febe08b4b.""" +"""Test batch_completion's return_exceptions.""" +import pytest +import litellm + +msg1 = [{"role": "user", "content": "hi 1"}] +msg2 = [{"role": "user", "content": "hi 2"}] + + +def test_batch_completion_return_exceptions_default(): + """Test batch_completion's return_exceptions.""" + with pytest.raises(Exception): + _ = litellm.batch_completion( + model="gpt-3.5-turbo", + messages=[msg1, msg2], + api_key="sk_xxx", # deliberately set invalid key + # return_exceptions=False, + ) + + +def test_batch_completion_return_exceptions_true(): + """Test batch_completion's return_exceptions.""" + res = litellm.batch_completion( + model="gpt-3.5-turbo", + messages=[msg1, msg2], + api_key="sk_xxx", # deliberately set invalid key + return_exceptions=True, + ) + + assert isinstance(res[0], litellm.exceptions.AuthenticationError) \ No newline at end of file From 5149a1de86b80da12e8fe105151cc9267b395d71 Mon Sep 17 00:00:00 2001 From: vietpham1911 Date: Tue, 14 May 2024 19:50:11 +0930 Subject: [PATCH 003/192] Added script for auto updating the pricing with the pricings of openrouter.ai --- auto_update_pricing.py | 93 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 auto_update_pricing.py diff --git a/auto_update_pricing.py b/auto_update_pricing.py new file mode 100644 index 00000000000..87ce32c0b96 --- /dev/null +++ b/auto_update_pricing.py @@ -0,0 +1,93 @@ +import requests +import json + +def fetch_data(url): + """ + Fetches data from the specified URL. + + Args: + url (str): The URL to fetch data from. + + Returns: + dict or None: The JSON response if successful, None otherwise. + """ + try: + response = requests.get(url) + response.raise_for_status() + return response.json() + except requests.exceptions.RequestException as e: + print("Error fetching data from URL:", e) + return None + +def update_local_data(local_data, remote_data): + """ + Updates local data with information fetched remotely. + + Args: + local_data (dict): Local data to be updated. + remote_data (dict): Remote data fetched from an API. + + """ + for model_name, model_info in local_data.items(): + if model_name.startswith("openrouter/"): + model_suffix = model_name[len("openrouter/"):] + for model in remote_data["data"]: + if model["id"] == model_suffix: + model_info["max_tokens"] = model["context_length"] + model_info["input_cost_per_token"] = model["pricing"]["prompt"] + model_info["output_cost_per_token"] = model["pricing"]["completion"] + break + +def write_to_file(file_path, data): + """ + Writes data to a JSON file. + + Args: + file_path (str): The path to the JSON file. + data (dict): The data to write to the file. + """ + try: + with open(file_path, "w") as file: + json.dump(data, file, indent=4) + print("Values updated successfully.") + except Exception as e: + print("Error updating JSON file:", e) + +def main(): + """ + Main function to orchestrate the process. + """ + local_file_path = "model_prices_and_context_window.json" + url = "https://openrouter.ai/api/v1/models" + + local_data = load_local_data(local_file_path) + remote_data = fetch_data(url) + + if local_data and remote_data: + update_local_data(local_data, remote_data) + write_to_file(local_file_path, local_data) + else: + print("Failed to fetch model data from either local file or URL.") + +def load_local_data(file_path): + """ + Loads data from a local JSON file. + + Args: + file_path (str): The path to the JSON file. + + Returns: + dict or None: The loaded data if successful, None otherwise. + """ + try: + with open(file_path, "r") as file: + return json.load(file) + except FileNotFoundError: + print("File not found:", file_path) + return None + except json.JSONDecodeError as e: + print("Error decoding JSON:", e) + return None + +if __name__ == "__main__": + main() From 771f06bd17fb0ff8223eccaf33a70e271ebf2206 Mon Sep 17 00:00:00 2001 From: vietpham1911 Date: Thu, 16 May 2024 13:40:58 +0930 Subject: [PATCH 004/192] Add the models of openrouter that are not in the json file with the pricings of the models yet --- auto_update_pricing.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/auto_update_pricing.py b/auto_update_pricing.py index 87ce32c0b96..bda59d398ef 100644 --- a/auto_update_pricing.py +++ b/auto_update_pricing.py @@ -37,6 +37,17 @@ def update_local_data(local_data, remote_data): model_info["input_cost_per_token"] = model["pricing"]["prompt"] model_info["output_cost_per_token"] = model["pricing"]["completion"] break + # Add models not in local data yet + for model in remote_data["data"]: + model_id = model["id"] + if model_id not in local_data: + local_data[f"openrouter/{model_id}"] = { + "max_tokens": model["context_length"], + "input_cost_per_token": model["pricing"]["prompt"], + "output_cost_per_token": model["pricing"]["completion"], + "litellm_provider": "openrouter", + "mode": "chat" + } def write_to_file(file_path, data): """ From 514477b1e717d4cba84a19e428bf170b9e3b0c34 Mon Sep 17 00:00:00 2001 From: vietpham1911 Date: Thu, 16 May 2024 14:11:29 +0930 Subject: [PATCH 005/192] Bug fix: The prior code removed values like "supports_function_calling" or "supports_vision". This is fixed now. Nothing is being removed. Only the pricings and the maximum tokens are being edited and new models in openrouter are being added. --- auto_update_pricing.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/auto_update_pricing.py b/auto_update_pricing.py index bda59d398ef..f59d4b4729b 100644 --- a/auto_update_pricing.py +++ b/auto_update_pricing.py @@ -33,14 +33,17 @@ def update_local_data(local_data, remote_data): model_suffix = model_name[len("openrouter/"):] for model in remote_data["data"]: if model["id"] == model_suffix: - model_info["max_tokens"] = model["context_length"] - model_info["input_cost_per_token"] = model["pricing"]["prompt"] - model_info["output_cost_per_token"] = model["pricing"]["completion"] + # Update only the values that need to be updated + model_info.update({ + "max_tokens": model["context_length"], + "input_cost_per_token": model["pricing"]["prompt"], + "output_cost_per_token": model["pricing"]["completion"] + }) break # Add models not in local data yet for model in remote_data["data"]: model_id = model["id"] - if model_id not in local_data: + if f"openrouter/{model_id}" not in local_data: local_data[f"openrouter/{model_id}"] = { "max_tokens": model["context_length"], "input_cost_per_token": model["pricing"]["prompt"], From ea4b766a9434c2a03a1dc7572d73d3a9a25c414a Mon Sep 17 00:00:00 2001 From: Rajan Paneru Date: Sat, 18 May 2024 12:18:22 +0930 Subject: [PATCH 006/192] updated the config to run the script --- .circleci/config.yml | 3 +++ .../auto_update_price_and_context_window_file.py | 0 2 files changed, 3 insertions(+) rename auto_update_pricing.py => ci_cd/auto_update_price_and_context_window_file.py (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 516f2b20d33..3768452ca64 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -183,6 +183,9 @@ jobs: pip install "asyncio==3.4.3" pip install "PyGithub==1.59.1" # Run pytest and generate JUnit XML report + - run: + name: Update File for Price & Content Window + command: python ci_cd/auto_update_price_and_context_window_file.py - run: name: Build Docker image command: docker build -t my-app:latest -f Dockerfile.database . diff --git a/auto_update_pricing.py b/ci_cd/auto_update_price_and_context_window_file.py similarity index 100% rename from auto_update_pricing.py rename to ci_cd/auto_update_price_and_context_window_file.py From 4a7fd2ea3e1062dde7bfb0d83f64ecbda9e4bd42 Mon Sep 17 00:00:00 2001 From: vietpham1911 Date: Sat, 18 May 2024 13:55:06 +0930 Subject: [PATCH 007/192] Wrote the code more clean and more efficient. Furthermore, the code adds the input_cost_per_image and supports_vision to the models if the two values exist. --- ...to_update_price_and_context_window_file.py | 156 +++++++++--------- 1 file changed, 76 insertions(+), 80 deletions(-) diff --git a/ci_cd/auto_update_price_and_context_window_file.py b/ci_cd/auto_update_price_and_context_window_file.py index f59d4b4729b..ae0814f9bb2 100644 --- a/ci_cd/auto_update_price_and_context_window_file.py +++ b/ci_cd/auto_update_price_and_context_window_file.py @@ -1,107 +1,103 @@ -import requests +import asyncio +import aiohttp import json -def fetch_data(url): - """ - Fetches data from the specified URL. - - Args: - url (str): The URL to fetch data from. - - Returns: - dict or None: The JSON response if successful, None otherwise. - """ +# Asynchronously fetch data from a given URL +async def fetch_data(url): try: - response = requests.get(url) - response.raise_for_status() - return response.json() - except requests.exceptions.RequestException as e: + # Create an asynchronous session + async with aiohttp.ClientSession() as session: + # Send a GET request to the URL + async with session.get(url) as resp: + # Raise an error if the response status is not OK + resp.raise_for_status() + # Parse the response JSON + resp_json = await resp.json() + print("Fetch the data from URL:") + # Return the 'data' field from the JSON response + return resp_json['data'] + except Exception as e: + # Print an error message if fetching data fails print("Error fetching data from URL:", e) return None -def update_local_data(local_data, remote_data): - """ - Updates local data with information fetched remotely. +# Synchronize local data with remote data +def sync_local_data_with_remote(local_data, remote_data): + # Update existing keys in local_data with values from remote_data + for key in (set(local_data) & set(remote_data)): + local_data[key].update(remote_data[key]) - Args: - local_data (dict): Local data to be updated. - remote_data (dict): Remote data fetched from an API. - - """ - for model_name, model_info in local_data.items(): - if model_name.startswith("openrouter/"): - model_suffix = model_name[len("openrouter/"):] - for model in remote_data["data"]: - if model["id"] == model_suffix: - # Update only the values that need to be updated - model_info.update({ - "max_tokens": model["context_length"], - "input_cost_per_token": model["pricing"]["prompt"], - "output_cost_per_token": model["pricing"]["completion"] - }) - break - # Add models not in local data yet - for model in remote_data["data"]: - model_id = model["id"] - if f"openrouter/{model_id}" not in local_data: - local_data[f"openrouter/{model_id}"] = { - "max_tokens": model["context_length"], - "input_cost_per_token": model["pricing"]["prompt"], - "output_cost_per_token": model["pricing"]["completion"], - "litellm_provider": "openrouter", - "mode": "chat" - } + # Add new keys from remote_data to local_data + for key in (set(remote_data) - set(local_data)): + local_data[key] = remote_data[key] +# Write data to the json file def write_to_file(file_path, data): - """ - Writes data to a JSON file. - - Args: - file_path (str): The path to the JSON file. - data (dict): The data to write to the file. - """ try: + # Open the file in write mode with open(file_path, "w") as file: + # Dump the data as JSON into the file json.dump(data, file, indent=4) print("Values updated successfully.") except Exception as e: + # Print an error message if writing to file fails print("Error updating JSON file:", e) +# Update the existing models and add the missing models +def transform_remote_data(data): + transformed = {} + for row in data: + # Create a new dictionary with transformed data + obj = { + "max_tokens": row["context_length"], + "input_cost_per_token": float(row["pricing"]["prompt"]), + "output_cost_per_token": float(row["pricing"]["completion"]), + "litellm_provider": "openrouter", + "mode": "chat" + } + # Add an additional field if the modality is 'multimodal' + if row.get('architecture', {}).get('modality') == 'multimodal': + obj['supports_vision'] = True + + # Use a composite key to store the transformed object + transformed[f'openrouter/{row["id"]}'] = obj + + return transformed + +# Load local data from a specified file +def load_local_data(file_path): + try: + # Open the file in read mode + with open(file_path, "r") as file: + # Load and return the JSON data + return json.load(file) + except FileNotFoundError: + # Print an error message if the file is not found + print("File not found:", file_path) + return None + except json.JSONDecodeError as e: + # Print an error message if JSON decoding fails + print("Error decoding JSON:", e) + return None + def main(): - """ - Main function to orchestrate the process. - """ - local_file_path = "model_prices_and_context_window.json" - url = "https://openrouter.ai/api/v1/models" + local_file_path = "model_prices_and_context_window.json" # Path to the local data file + url = "https://openrouter.ai/api/v1/models" # URL to fetch remote data + # Load local data from file local_data = load_local_data(local_file_path) - remote_data = fetch_data(url) + # Fetch remote data asynchronously + remote_data = asyncio.run(fetch_data(url)) + # Transform the fetched remote data + remote_data = transform_remote_data(remote_data) + # If both local and remote data are available, synchronize and save if local_data and remote_data: - update_local_data(local_data, remote_data) + sync_local_data_with_remote(local_data, remote_data) write_to_file(local_file_path, local_data) else: print("Failed to fetch model data from either local file or URL.") -def load_local_data(file_path): - """ - Loads data from a local JSON file. - - Args: - file_path (str): The path to the JSON file. - - Returns: - dict or None: The loaded data if successful, None otherwise. - """ - try: - with open(file_path, "r") as file: - return json.load(file) - except FileNotFoundError: - print("File not found:", file_path) - return None - except json.JSONDecodeError as e: - print("Error decoding JSON:", e) - return None - +# Entry point of the script if __name__ == "__main__": main() From 1b1665262903a863cf707d8fae66c92bb542a20b Mon Sep 17 00:00:00 2001 From: vietpham1911 Date: Sat, 18 May 2024 16:59:17 +0930 Subject: [PATCH 008/192] Implemented feature to add input_cost_per_image --- ...to_update_price_and_context_window_file.py | 7 +- model_prices_and_context_window.json | 1064 ++++++++++++++++- 2 files changed, 1044 insertions(+), 27 deletions(-) diff --git a/ci_cd/auto_update_price_and_context_window_file.py b/ci_cd/auto_update_price_and_context_window_file.py index ae0814f9bb2..06b08bd3081 100644 --- a/ci_cd/auto_update_price_and_context_window_file.py +++ b/ci_cd/auto_update_price_and_context_window_file.py @@ -13,7 +13,7 @@ async def fetch_data(url): resp.raise_for_status() # Parse the response JSON resp_json = await resp.json() - print("Fetch the data from URL:") + print("Fetch the data from URL.") # Return the 'data' field from the JSON response return resp_json['data'] except Exception as e: @@ -55,6 +55,11 @@ def transform_remote_data(data): "litellm_provider": "openrouter", "mode": "chat" } + + # Add an field 'input_cost_per_image' + if "pricing" in row and "image" in row["pricing"]: + obj['input_cost_per_image'] = float(row["pricing"]["image"]) + # Add an additional field if the modality is 'multimodal' if row.get('architecture', {}).get('modality') == 'multimodal': obj['supports_vision'] = True diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index bede36764e9..0080ef41719 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1685,21 +1685,24 @@ "input_cost_per_token": 0.00000065, "output_cost_per_token": 0.00000065, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/cohere/command-r-plus": { "max_tokens": 128000, "input_cost_per_token": 0.000003, "output_cost_per_token": 0.000015, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/databricks/dbrx-instruct": { "max_tokens": 32768, "input_cost_per_token": 0.0000006, "output_cost_per_token": 0.0000006, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/anthropic/claude-3-haiku": { "max_tokens": 200000, @@ -1726,14 +1729,16 @@ "input_cost_per_token": 0.000008, "output_cost_per_token": 0.000024, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/cognitivecomputations/dolphin-mixtral-8x7b": { "max_tokens": 32769, "input_cost_per_token": 0.0000005, "output_cost_per_token": 0.0000005, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/google/gemini-pro-vision": { "max_tokens": 45875, @@ -1750,35 +1755,45 @@ "input_cost_per_token": 0.0000002, "output_cost_per_token": 0.0000002, "litellm_provider": "openrouter", +<<<<<<< HEAD "mode": "chat" +======= + "mode": "chat", + "supports_vision": true, + "input_cost_per_image": 0.0001152 +>>>>>>> 65d82367d (Implemented feature to add input_cost_per_image) }, "openrouter/meta-llama/llama-3-8b-instruct:free": { "max_tokens": 8192, "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/meta-llama/llama-3-8b-instruct:extended": { "max_tokens": 16384, "input_cost_per_token": 0.000000225, "output_cost_per_token": 0.00000225, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/meta-llama/llama-3-70b-instruct:nitro": { "max_tokens": 8192, "input_cost_per_token": 0.0000009, "output_cost_per_token": 0.0000009, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/meta-llama/llama-3-70b-instruct": { "max_tokens": 8192, "input_cost_per_token": 0.00000059, "output_cost_per_token": 0.00000079, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/openai/gpt-4o": { "max_tokens": 4096, @@ -1819,21 +1834,24 @@ "input_cost_per_token": 0.0000015, "output_cost_per_token": 0.000002, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/openai/gpt-3.5-turbo-16k": { "max_tokens": 16383, "input_cost_per_token": 0.000003, "output_cost_per_token": 0.000004, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/openai/gpt-4": { "max_tokens": 8192, "input_cost_per_token": 0.00003, "output_cost_per_token": 0.00006, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/anthropic/claude-instant-v1": { "max_tokens": 100000, @@ -1849,7 +1867,8 @@ "input_cost_per_token": 0.00001102, "output_cost_per_token": 0.00003268, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/anthropic/claude-3-opus": { "max_tokens": 4096, @@ -1861,63 +1880,72 @@ "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 395 + "tool_use_system_prompt_tokens": 395, + "input_cost_per_image": 0.024 }, "openrouter/google/palm-2-chat-bison": { "max_tokens": 25804, "input_cost_per_token": 0.0000005, "output_cost_per_token": 0.0000005, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/google/palm-2-codechat-bison": { "max_tokens": 20070, "input_cost_per_token": 0.0000005, "output_cost_per_token": 0.0000005, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/meta-llama/llama-2-13b-chat": { "max_tokens": 4096, "input_cost_per_token": 0.0000002, "output_cost_per_token": 0.0000002, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/meta-llama/llama-2-70b-chat": { "max_tokens": 4096, "input_cost_per_token": 0.0000015, "output_cost_per_token": 0.0000015, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/meta-llama/codellama-34b-instruct": { "max_tokens": 8096, "input_cost_per_token": 0.0000005, "output_cost_per_token": 0.0000005, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/nousresearch/nous-hermes-llama2-13b": { "max_tokens": 4096, "input_cost_per_token": 0.0000002, "output_cost_per_token": 0.0000002, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/mancer/weaver": { "max_tokens": 8000, "input_cost_per_token": 0.000005625, "output_cost_per_token": 0.000005625, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/gryphe/mythomax-l2-13b": { "max_tokens": 8192, "input_cost_per_token": 0.000001875, "output_cost_per_token": 0.000001875, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/jondurbin/airoboros-l2-70b-2.1": { "max_tokens": 4096, @@ -1931,28 +1959,32 @@ "input_cost_per_token": 0.000001875, "output_cost_per_token": 0.000001875, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/pygmalionai/mythalion-13b": { "max_tokens": 4096, "input_cost_per_token": 0.000001875, "output_cost_per_token": 0.000001875, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/mistralai/mistral-7b-instruct": { "max_tokens": 8192, "input_cost_per_token": 0.00000013, "output_cost_per_token": 0.00000013, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/mistralai/mistral-7b-instruct:free": { "max_tokens": 8192, "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "j2-ultra": { "max_tokens": 8192, @@ -3532,6 +3564,986 @@ "output_cost_per_token": 0.000000, "litellm_provider": "voyage", "mode": "embedding" +<<<<<<< HEAD +======= + }, + "openrouter/openchat/openchat-7b": { + "max_tokens": 8192, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 7e-08, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/anthropic/claude-2.0": { + "max_tokens": 100000, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/mistralai/mixtral-8x7b-instruct:nitro": { + "max_tokens": 32768, + "input_cost_per_token": 5.4e-07, + "output_cost_per_token": 5.4e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/meta-llama/llama-3-8b-instruct": { + "max_tokens": 8192, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 7e-08, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/perplexity/llama-3-sonar-small-32k-chat": { + "max_tokens": 32768, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/openrouter/auto": { + "max_tokens": 200000, + "input_cost_per_token": -1.0, + "output_cost_per_token": -1.0, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": -1.0 + }, + "openrouter/sophosympatheia/midnight-rose-70b": { + "max_tokens": 4096, + "input_cost_per_token": 9e-06, + "output_cost_per_token": 9e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/togethercomputer/stripedhyena-hessian-7b": { + "max_tokens": 32768, + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 1.8e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/recursal/eagle-7b": { + "max_tokens": 10000, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/google/gemma-7b-it:nitro": { + "max_tokens": 8192, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/lynn/soliloquy-l3": { + "max_tokens": 24576, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 5e-08, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/rwkv/rwkv-5-world-3b": { + "max_tokens": 10000, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/google/palm-2-chat-bison-32k": { + "max_tokens": 91750, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 5e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/codellama/codellama-70b-instruct": { + "max_tokens": 2048, + "input_cost_per_token": 8.1e-07, + "output_cost_per_token": 8.1e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/openchat/openchat-7b:free": { + "max_tokens": 8192, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/anthropic/claude-1": { + "max_tokens": 100000, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/anthropic/claude-3-opus:beta": { + "max_tokens": 200000, + "input_cost_per_token": 1.5e-05, + "output_cost_per_token": 7.5e-05, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_vision": true, + "input_cost_per_image": 0.024 + }, + "openrouter/qwen/qwen-7b-chat": { + "max_tokens": 32768, + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 1.8e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/teknium/openhermes-2.5-mistral-7b": { + "max_tokens": 4096, + "input_cost_per_token": 1.7e-07, + "output_cost_per_token": 1.7e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/nousresearch/nous-capybara-7b": { + "max_tokens": 4096, + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 1.8e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/openai/gpt-4-turbo": { + "max_tokens": 128000, + "input_cost_per_token": 1e-05, + "output_cost_per_token": 3e-05, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_vision": true, + "input_cost_per_image": 0.01445 + }, + "openrouter/liuhaotian/llava-yi-34b": { + "max_tokens": 4096, + "input_cost_per_token": 9e-07, + "output_cost_per_token": 9e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_vision": true, + "input_cost_per_image": 0.0005184 + }, + "openrouter/allenai/olmo-7b-instruct": { + "max_tokens": 2048, + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 1.8e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/nousresearch/nous-hermes-2-mistral-7b-dpo": { + "max_tokens": 8192, + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 1.8e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/sao10k/fimbulvetr-11b-v2": { + "max_tokens": 8192, + "input_cost_per_token": 5.499375e-07, + "output_cost_per_token": 2.825625e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/openai/gpt-3.5-turbo-1106": { + "max_tokens": 16385, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/anthropic/claude-2.0:beta": { + "max_tokens": 100000, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/neversleep/llama-3-lumimaid-8b:extended": { + "max_tokens": 24576, + "input_cost_per_token": 2.25e-07, + "output_cost_per_token": 2.25e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/open-orca/mistral-7b-openorca": { + "max_tokens": 8192, + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 1.8e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/openai/gpt-4o": { + "max_tokens": 128000, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 1.5e-05, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_vision": true, + "input_cost_per_image": 0.002312 + }, + "openrouter/microsoft/wizardlm-2-7b": { + "max_tokens": 32000, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 7e-08, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/neversleep/llama-3-lumimaid-8b": { + "max_tokens": 24576, + "input_cost_per_token": 2.25e-07, + "output_cost_per_token": 2.25e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/meta-llama/llama-2-70b-chat:nitro": { + "max_tokens": 4096, + "input_cost_per_token": 9e-07, + "output_cost_per_token": 9e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/anthropic/claude-3-haiku:beta": { + "max_tokens": 200000, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 1.25e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_vision": true, + "input_cost_per_image": 0.0004 + }, + "openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo": { + "max_tokens": 32768, + "input_cost_per_token": 2.7e-07, + "output_cost_per_token": 2.7e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/mistralai/mixtral-8x7b-instruct": { + "max_tokens": 32768, + "input_cost_per_token": 2.4e-07, + "output_cost_per_token": 2.4e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/meta-llama/llama-guard-2-8b": { + "max_tokens": 8192, + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 1.8e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/undi95/toppy-m-7b:free": { + "max_tokens": 4096, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/huggingfaceh4/zephyr-7b-beta:free": { + "max_tokens": 4096, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-sft": { + "max_tokens": 32768, + "input_cost_per_token": 5.4e-07, + "output_cost_per_token": 5.4e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/anthropic/claude-3-sonnet:beta": { + "max_tokens": 200000, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_vision": true, + "input_cost_per_image": 0.0048 + }, + "openrouter/mistralai/mistral-small": { + "max_tokens": 32000, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/mistralai/mistral-tiny": { + "max_tokens": 32000, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 2.5e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/meta-llama/llama-3-70b": { + "max_tokens": 8192, + "input_cost_per_token": 8.1e-07, + "output_cost_per_token": 8.1e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/gryphe/mythomist-7b": { + "max_tokens": 32768, + "input_cost_per_token": 3.75e-07, + "output_cost_per_token": 3.75e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/openai/gpt-3.5-turbo-0613": { + "max_tokens": 4095, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/qwen/qwen-14b-chat": { + "max_tokens": 32768, + "input_cost_per_token": 2.7e-07, + "output_cost_per_token": 2.7e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/perplexity/llama-3-sonar-small-32k-online": { + "max_tokens": 28000, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/nousresearch/nous-capybara-34b": { + "max_tokens": 32768, + "input_cost_per_token": 9e-07, + "output_cost_per_token": 9e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/liuhaotian/llava-13b": { + "max_tokens": 2048, + "input_cost_per_token": 1e-05, + "output_cost_per_token": 1e-05, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_vision": true, + "input_cost_per_image": 0.0 + }, + "openrouter/gryphe/mythomist-7b:free": { + "max_tokens": 32768, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/undi95/remm-slerp-l2-13b:extended": { + "max_tokens": 6144, + "input_cost_per_token": 1.125e-06, + "output_cost_per_token": 1.125e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/openrouter/cinematika-7b:free": { + "max_tokens": 8000, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/gryphe/mythomax-l2-13b:nitro": { + "max_tokens": 4096, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/nousresearch/nous-hermes-yi-34b": { + "max_tokens": 4096, + "input_cost_per_token": 7.2e-07, + "output_cost_per_token": 7.2e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/perplexity/llama-3-sonar-large-32k-online": { + "max_tokens": 28000, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 1e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/recursal/rwkv-5-3b-ai-town": { + "max_tokens": 10000, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/anthropic/claude-1.2": { + "max_tokens": 100000, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/qwen/qwen-110b-chat": { + "max_tokens": 32768, + "input_cost_per_token": 1.62e-06, + "output_cost_per_token": 1.62e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/openai/gpt-4-1106-preview": { + "max_tokens": 128000, + "input_cost_per_token": 1e-05, + "output_cost_per_token": 3e-05, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/openai/gpt-3.5-turbo-instruct": { + "max_tokens": 4095, + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 2e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/jondurbin/airoboros-l2-70b": { + "max_tokens": 4096, + "input_cost_per_token": 7e-07, + "output_cost_per_token": 9e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/koboldai/psyfighter-13b-2": { + "max_tokens": 4096, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 1e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/phind/phind-codellama-34b": { + "max_tokens": 4096, + "input_cost_per_token": 7.2e-07, + "output_cost_per_token": 7.2e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/jebcarter/psyfighter-13b": { + "max_tokens": 4096, + "input_cost_per_token": -1.0, + "output_cost_per_token": -1.0, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": -1.0 + }, + "openrouter/alpindale/goliath-120b": { + "max_tokens": 6144, + "input_cost_per_token": 9.375e-06, + "output_cost_per_token": 9.375e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/01-ai/yi-34b-chat": { + "max_tokens": 4096, + "input_cost_per_token": 7.2e-07, + "output_cost_per_token": 7.2e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/neversleep/llama-3-lumimaid-70b": { + "max_tokens": 8192, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 8e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/huggingfaceh4/zephyr-7b-beta": { + "max_tokens": 4096, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/deepseek/deepseek-chat": { + "max_tokens": 128000, + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 2.8e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/qwen/qwen-4b-chat": { + "max_tokens": 32768, + "input_cost_per_token": 9e-08, + "output_cost_per_token": 9e-08, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/google/gemma-7b-it": { + "max_tokens": 8192, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 7e-08, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/teknium/openhermes-2-mistral-7b": { + "max_tokens": 4096, + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 1.8e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/anthropic/claude-instant-1.0": { + "max_tokens": 100000, + "input_cost_per_token": 8e-07, + "output_cost_per_token": 2.4e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/microsoft/wizardlm-2-8x22b": { + "max_tokens": 65536, + "input_cost_per_token": 6.5e-07, + "output_cost_per_token": 6.5e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/qwen/qwen-72b-chat": { + "max_tokens": 32768, + "input_cost_per_token": 9e-07, + "output_cost_per_token": 9e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/openai/gpt-4-32k-0314": { + "max_tokens": 32767, + "input_cost_per_token": 6e-05, + "output_cost_per_token": 0.00012, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/openai/gpt-4-32k": { + "max_tokens": 32767, + "input_cost_per_token": 6e-05, + "output_cost_per_token": 0.00012, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/undi95/toppy-m-7b:nitro": { + "max_tokens": 4096, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 7e-08, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/google/palm-2-codechat-bison-32k": { + "max_tokens": 91750, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 5e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/snowflake/snowflake-arctic-instruct": { + "max_tokens": 4096, + "input_cost_per_token": 2.16e-06, + "output_cost_per_token": 2.16e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/gryphe/mythomax-l2-13b:extended": { + "max_tokens": 8192, + "input_cost_per_token": 1.125e-06, + "output_cost_per_token": 1.125e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/01-ai/yi-34b": { + "max_tokens": 4096, + "input_cost_per_token": 7.2e-07, + "output_cost_per_token": 7.2e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/togethercomputer/stripedhyena-nous-7b": { + "max_tokens": 32768, + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 1.8e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/deepseek/deepseek-coder": { + "max_tokens": 16000, + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 2.8e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/mistralai/mixtral-8x7b": { + "max_tokens": 32768, + "input_cost_per_token": 5.4e-07, + "output_cost_per_token": 5.4e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/openai/gpt-4-turbo-preview": { + "max_tokens": 128000, + "input_cost_per_token": 1e-05, + "output_cost_per_token": 3e-05, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/xwin-lm/xwin-lm-70b": { + "max_tokens": 8192, + "input_cost_per_token": 3.75e-06, + "output_cost_per_token": 3.75e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/nousresearch/nous-capybara-7b:free": { + "max_tokens": 4096, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/anthropic/claude-instant-1.1": { + "max_tokens": 100000, + "input_cost_per_token": 8e-07, + "output_cost_per_token": 2.4e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/intel/neural-chat-7b": { + "max_tokens": 4096, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 5e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/anthropic/claude-2:beta": { + "max_tokens": 200000, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/jondurbin/bagel-34b": { + "max_tokens": 8000, + "input_cost_per_token": -1.0, + "output_cost_per_token": -1.0, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": -1.0 + }, + "openrouter/lizpreciatior/lzlv-70b-fp16-hf": { + "max_tokens": 4096, + "input_cost_per_token": 5.9e-07, + "output_cost_per_token": 7.9e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/openai/gpt-4-0314": { + "max_tokens": 8191, + "input_cost_per_token": 3e-05, + "output_cost_per_token": 6e-05, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/austism/chronos-hermes-13b": { + "max_tokens": 4096, + "input_cost_per_token": 1.3e-07, + "output_cost_per_token": 1.3e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/anthropic/claude-2.1": { + "max_tokens": 200000, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/anthropic/claude-2.1:beta": { + "max_tokens": 200000, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/google/gemma-7b-it:free": { + "max_tokens": 8192, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/openai/gpt-4o-2024-05-13": { + "max_tokens": 128000, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 1.5e-05, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.002312 + }, + "openrouter/google/gemini-flash-1.5": { + "max_tokens": 2800000, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 7.5e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_vision": true, + "input_cost_per_image": 0.000265 + }, + "openrouter/meta-llama/llama-3-8b": { + "max_tokens": 8192, + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 1.8e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/openai/gpt-3.5-turbo-0301": { + "max_tokens": 4095, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/openrouter/cinematika-7b": { + "max_tokens": 8000, + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 1.8e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/qwen/qwen-32b-chat": { + "max_tokens": 32768, + "input_cost_per_token": 7.2e-07, + "output_cost_per_token": 7.2e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/anthropic/claude-instant-1:beta": { + "max_tokens": 100000, + "input_cost_per_token": 8e-07, + "output_cost_per_token": 2.4e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/mistralai/mistral-7b-instruct:nitro": { + "max_tokens": 32768, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/cohere/command-r": { + "max_tokens": 128000, + "input_cost_per_token": 5e-07, + "output_cost_per_token": 1.5e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/cohere/command": { + "max_tokens": 4096, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/neversleep/noromaid-mixtral-8x7b-instruct": { + "max_tokens": 8000, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 8e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/neversleep/noromaid-20b": { + "max_tokens": 8192, + "input_cost_per_token": 2.25e-06, + "output_cost_per_token": 2.25e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/perplexity/llama-3-sonar-large-32k-chat": { + "max_tokens": 32768, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 1e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/anthropic/claude-instant-1": { + "max_tokens": 100000, + "input_cost_per_token": 8e-07, + "output_cost_per_token": 2.4e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/mistralai/mixtral-8x22b": { + "max_tokens": 65536, + "input_cost_per_token": 9e-07, + "output_cost_per_token": 9e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/meta-llama/llama-3-8b-instruct:nitro": { + "max_tokens": 8192, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/mistralai/mistral-medium": { + "max_tokens": 32000, + "input_cost_per_token": 2.7e-06, + "output_cost_per_token": 8.1e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/undi95/toppy-m-7b": { + "max_tokens": 4096, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 7e-08, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/google/gemini-pro": { + "max_tokens": 91728, + "input_cost_per_token": 1.25e-07, + "output_cost_per_token": 3.75e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0025 + }, + "openrouter/nousresearch/nous-hermes-2-vision-7b": { + "max_tokens": 4096, + "input_cost_per_token": 1e-05, + "output_cost_per_token": 1e-05, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_vision": true, + "input_cost_per_image": 0.0 + }, + "openrouter/01-ai/yi-6b": { + "max_tokens": 4096, + "input_cost_per_token": 1.26e-07, + "output_cost_per_token": 1.26e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/openai/gpt-3.5-turbo-0125": { + "max_tokens": 16385, + "input_cost_per_token": 5e-07, + "output_cost_per_token": 1.5e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 +>>>>>>> 65d82367d (Implemented feature to add input_cost_per_image) } } From 51fa42e13a1e24b778b7f12f9ea03c5db4498ac0 Mon Sep 17 00:00:00 2001 From: Rajan Paneru Date: Sat, 18 May 2024 19:22:24 +0930 Subject: [PATCH 009/192] resolve conflict --- model_prices_and_context_window.json | 1064 +------------------------- 1 file changed, 26 insertions(+), 1038 deletions(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 0080ef41719..bede36764e9 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1685,24 +1685,21 @@ "input_cost_per_token": 0.00000065, "output_cost_per_token": 0.00000065, "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 + "mode": "chat" }, "openrouter/cohere/command-r-plus": { "max_tokens": 128000, "input_cost_per_token": 0.000003, "output_cost_per_token": 0.000015, "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 + "mode": "chat" }, "openrouter/databricks/dbrx-instruct": { "max_tokens": 32768, "input_cost_per_token": 0.0000006, "output_cost_per_token": 0.0000006, "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 + "mode": "chat" }, "openrouter/anthropic/claude-3-haiku": { "max_tokens": 200000, @@ -1729,16 +1726,14 @@ "input_cost_per_token": 0.000008, "output_cost_per_token": 0.000024, "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 + "mode": "chat" }, "openrouter/cognitivecomputations/dolphin-mixtral-8x7b": { "max_tokens": 32769, "input_cost_per_token": 0.0000005, "output_cost_per_token": 0.0000005, "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 + "mode": "chat" }, "openrouter/google/gemini-pro-vision": { "max_tokens": 45875, @@ -1755,45 +1750,35 @@ "input_cost_per_token": 0.0000002, "output_cost_per_token": 0.0000002, "litellm_provider": "openrouter", -<<<<<<< HEAD "mode": "chat" -======= - "mode": "chat", - "supports_vision": true, - "input_cost_per_image": 0.0001152 ->>>>>>> 65d82367d (Implemented feature to add input_cost_per_image) }, "openrouter/meta-llama/llama-3-8b-instruct:free": { "max_tokens": 8192, "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 + "mode": "chat" }, "openrouter/meta-llama/llama-3-8b-instruct:extended": { "max_tokens": 16384, "input_cost_per_token": 0.000000225, "output_cost_per_token": 0.00000225, "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 + "mode": "chat" }, "openrouter/meta-llama/llama-3-70b-instruct:nitro": { "max_tokens": 8192, "input_cost_per_token": 0.0000009, "output_cost_per_token": 0.0000009, "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 + "mode": "chat" }, "openrouter/meta-llama/llama-3-70b-instruct": { "max_tokens": 8192, "input_cost_per_token": 0.00000059, "output_cost_per_token": 0.00000079, "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 + "mode": "chat" }, "openrouter/openai/gpt-4o": { "max_tokens": 4096, @@ -1834,24 +1819,21 @@ "input_cost_per_token": 0.0000015, "output_cost_per_token": 0.000002, "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 + "mode": "chat" }, "openrouter/openai/gpt-3.5-turbo-16k": { "max_tokens": 16383, "input_cost_per_token": 0.000003, "output_cost_per_token": 0.000004, "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 + "mode": "chat" }, "openrouter/openai/gpt-4": { "max_tokens": 8192, "input_cost_per_token": 0.00003, "output_cost_per_token": 0.00006, "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 + "mode": "chat" }, "openrouter/anthropic/claude-instant-v1": { "max_tokens": 100000, @@ -1867,8 +1849,7 @@ "input_cost_per_token": 0.00001102, "output_cost_per_token": 0.00003268, "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 + "mode": "chat" }, "openrouter/anthropic/claude-3-opus": { "max_tokens": 4096, @@ -1880,72 +1861,63 @@ "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 395, - "input_cost_per_image": 0.024 + "tool_use_system_prompt_tokens": 395 }, "openrouter/google/palm-2-chat-bison": { "max_tokens": 25804, "input_cost_per_token": 0.0000005, "output_cost_per_token": 0.0000005, "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 + "mode": "chat" }, "openrouter/google/palm-2-codechat-bison": { "max_tokens": 20070, "input_cost_per_token": 0.0000005, "output_cost_per_token": 0.0000005, "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 + "mode": "chat" }, "openrouter/meta-llama/llama-2-13b-chat": { "max_tokens": 4096, "input_cost_per_token": 0.0000002, "output_cost_per_token": 0.0000002, "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 + "mode": "chat" }, "openrouter/meta-llama/llama-2-70b-chat": { "max_tokens": 4096, "input_cost_per_token": 0.0000015, "output_cost_per_token": 0.0000015, "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 + "mode": "chat" }, "openrouter/meta-llama/codellama-34b-instruct": { "max_tokens": 8096, "input_cost_per_token": 0.0000005, "output_cost_per_token": 0.0000005, "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 + "mode": "chat" }, "openrouter/nousresearch/nous-hermes-llama2-13b": { "max_tokens": 4096, "input_cost_per_token": 0.0000002, "output_cost_per_token": 0.0000002, "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 + "mode": "chat" }, "openrouter/mancer/weaver": { "max_tokens": 8000, "input_cost_per_token": 0.000005625, "output_cost_per_token": 0.000005625, "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 + "mode": "chat" }, "openrouter/gryphe/mythomax-l2-13b": { "max_tokens": 8192, "input_cost_per_token": 0.000001875, "output_cost_per_token": 0.000001875, "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 + "mode": "chat" }, "openrouter/jondurbin/airoboros-l2-70b-2.1": { "max_tokens": 4096, @@ -1959,32 +1931,28 @@ "input_cost_per_token": 0.000001875, "output_cost_per_token": 0.000001875, "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 + "mode": "chat" }, "openrouter/pygmalionai/mythalion-13b": { "max_tokens": 4096, "input_cost_per_token": 0.000001875, "output_cost_per_token": 0.000001875, "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 + "mode": "chat" }, "openrouter/mistralai/mistral-7b-instruct": { "max_tokens": 8192, "input_cost_per_token": 0.00000013, "output_cost_per_token": 0.00000013, "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 + "mode": "chat" }, "openrouter/mistralai/mistral-7b-instruct:free": { "max_tokens": 8192, "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 + "mode": "chat" }, "j2-ultra": { "max_tokens": 8192, @@ -3564,986 +3532,6 @@ "output_cost_per_token": 0.000000, "litellm_provider": "voyage", "mode": "embedding" -<<<<<<< HEAD -======= - }, - "openrouter/openchat/openchat-7b": { - "max_tokens": 8192, - "input_cost_per_token": 7e-08, - "output_cost_per_token": 7e-08, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/anthropic/claude-2.0": { - "max_tokens": 100000, - "input_cost_per_token": 8e-06, - "output_cost_per_token": 2.4e-05, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/mistralai/mixtral-8x7b-instruct:nitro": { - "max_tokens": 32768, - "input_cost_per_token": 5.4e-07, - "output_cost_per_token": 5.4e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/meta-llama/llama-3-8b-instruct": { - "max_tokens": 8192, - "input_cost_per_token": 7e-08, - "output_cost_per_token": 7e-08, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/perplexity/llama-3-sonar-small-32k-chat": { - "max_tokens": 32768, - "input_cost_per_token": 2e-07, - "output_cost_per_token": 2e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/openrouter/auto": { - "max_tokens": 200000, - "input_cost_per_token": -1.0, - "output_cost_per_token": -1.0, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": -1.0 - }, - "openrouter/sophosympatheia/midnight-rose-70b": { - "max_tokens": 4096, - "input_cost_per_token": 9e-06, - "output_cost_per_token": 9e-06, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/togethercomputer/stripedhyena-hessian-7b": { - "max_tokens": 32768, - "input_cost_per_token": 1.8e-07, - "output_cost_per_token": 1.8e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/recursal/eagle-7b": { - "max_tokens": 10000, - "input_cost_per_token": 0.0, - "output_cost_per_token": 0.0, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/google/gemma-7b-it:nitro": { - "max_tokens": 8192, - "input_cost_per_token": 2e-07, - "output_cost_per_token": 2e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/lynn/soliloquy-l3": { - "max_tokens": 24576, - "input_cost_per_token": 5e-08, - "output_cost_per_token": 5e-08, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/rwkv/rwkv-5-world-3b": { - "max_tokens": 10000, - "input_cost_per_token": 0.0, - "output_cost_per_token": 0.0, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/google/palm-2-chat-bison-32k": { - "max_tokens": 91750, - "input_cost_per_token": 2.5e-07, - "output_cost_per_token": 5e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/codellama/codellama-70b-instruct": { - "max_tokens": 2048, - "input_cost_per_token": 8.1e-07, - "output_cost_per_token": 8.1e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/openchat/openchat-7b:free": { - "max_tokens": 8192, - "input_cost_per_token": 0.0, - "output_cost_per_token": 0.0, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/anthropic/claude-1": { - "max_tokens": 100000, - "input_cost_per_token": 8e-06, - "output_cost_per_token": 2.4e-05, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/anthropic/claude-3-opus:beta": { - "max_tokens": 200000, - "input_cost_per_token": 1.5e-05, - "output_cost_per_token": 7.5e-05, - "litellm_provider": "openrouter", - "mode": "chat", - "supports_vision": true, - "input_cost_per_image": 0.024 - }, - "openrouter/qwen/qwen-7b-chat": { - "max_tokens": 32768, - "input_cost_per_token": 1.8e-07, - "output_cost_per_token": 1.8e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/teknium/openhermes-2.5-mistral-7b": { - "max_tokens": 4096, - "input_cost_per_token": 1.7e-07, - "output_cost_per_token": 1.7e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/nousresearch/nous-capybara-7b": { - "max_tokens": 4096, - "input_cost_per_token": 1.8e-07, - "output_cost_per_token": 1.8e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/openai/gpt-4-turbo": { - "max_tokens": 128000, - "input_cost_per_token": 1e-05, - "output_cost_per_token": 3e-05, - "litellm_provider": "openrouter", - "mode": "chat", - "supports_vision": true, - "input_cost_per_image": 0.01445 - }, - "openrouter/liuhaotian/llava-yi-34b": { - "max_tokens": 4096, - "input_cost_per_token": 9e-07, - "output_cost_per_token": 9e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "supports_vision": true, - "input_cost_per_image": 0.0005184 - }, - "openrouter/allenai/olmo-7b-instruct": { - "max_tokens": 2048, - "input_cost_per_token": 1.8e-07, - "output_cost_per_token": 1.8e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/nousresearch/nous-hermes-2-mistral-7b-dpo": { - "max_tokens": 8192, - "input_cost_per_token": 1.8e-07, - "output_cost_per_token": 1.8e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/sao10k/fimbulvetr-11b-v2": { - "max_tokens": 8192, - "input_cost_per_token": 5.499375e-07, - "output_cost_per_token": 2.825625e-06, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/openai/gpt-3.5-turbo-1106": { - "max_tokens": 16385, - "input_cost_per_token": 1e-06, - "output_cost_per_token": 2e-06, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/anthropic/claude-2.0:beta": { - "max_tokens": 100000, - "input_cost_per_token": 8e-06, - "output_cost_per_token": 2.4e-05, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/neversleep/llama-3-lumimaid-8b:extended": { - "max_tokens": 24576, - "input_cost_per_token": 2.25e-07, - "output_cost_per_token": 2.25e-06, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/open-orca/mistral-7b-openorca": { - "max_tokens": 8192, - "input_cost_per_token": 1.8e-07, - "output_cost_per_token": 1.8e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/openai/gpt-4o": { - "max_tokens": 128000, - "input_cost_per_token": 5e-06, - "output_cost_per_token": 1.5e-05, - "litellm_provider": "openrouter", - "mode": "chat", - "supports_vision": true, - "input_cost_per_image": 0.002312 - }, - "openrouter/microsoft/wizardlm-2-7b": { - "max_tokens": 32000, - "input_cost_per_token": 7e-08, - "output_cost_per_token": 7e-08, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/neversleep/llama-3-lumimaid-8b": { - "max_tokens": 24576, - "input_cost_per_token": 2.25e-07, - "output_cost_per_token": 2.25e-06, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/meta-llama/llama-2-70b-chat:nitro": { - "max_tokens": 4096, - "input_cost_per_token": 9e-07, - "output_cost_per_token": 9e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/anthropic/claude-3-haiku:beta": { - "max_tokens": 200000, - "input_cost_per_token": 2.5e-07, - "output_cost_per_token": 1.25e-06, - "litellm_provider": "openrouter", - "mode": "chat", - "supports_vision": true, - "input_cost_per_image": 0.0004 - }, - "openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo": { - "max_tokens": 32768, - "input_cost_per_token": 2.7e-07, - "output_cost_per_token": 2.7e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/mistralai/mixtral-8x7b-instruct": { - "max_tokens": 32768, - "input_cost_per_token": 2.4e-07, - "output_cost_per_token": 2.4e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/meta-llama/llama-guard-2-8b": { - "max_tokens": 8192, - "input_cost_per_token": 1.8e-07, - "output_cost_per_token": 1.8e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/undi95/toppy-m-7b:free": { - "max_tokens": 4096, - "input_cost_per_token": 0.0, - "output_cost_per_token": 0.0, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/huggingfaceh4/zephyr-7b-beta:free": { - "max_tokens": 4096, - "input_cost_per_token": 0.0, - "output_cost_per_token": 0.0, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-sft": { - "max_tokens": 32768, - "input_cost_per_token": 5.4e-07, - "output_cost_per_token": 5.4e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/anthropic/claude-3-sonnet:beta": { - "max_tokens": 200000, - "input_cost_per_token": 3e-06, - "output_cost_per_token": 1.5e-05, - "litellm_provider": "openrouter", - "mode": "chat", - "supports_vision": true, - "input_cost_per_image": 0.0048 - }, - "openrouter/mistralai/mistral-small": { - "max_tokens": 32000, - "input_cost_per_token": 2e-06, - "output_cost_per_token": 6e-06, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/mistralai/mistral-tiny": { - "max_tokens": 32000, - "input_cost_per_token": 2.5e-07, - "output_cost_per_token": 2.5e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/meta-llama/llama-3-70b": { - "max_tokens": 8192, - "input_cost_per_token": 8.1e-07, - "output_cost_per_token": 8.1e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/gryphe/mythomist-7b": { - "max_tokens": 32768, - "input_cost_per_token": 3.75e-07, - "output_cost_per_token": 3.75e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/openai/gpt-3.5-turbo-0613": { - "max_tokens": 4095, - "input_cost_per_token": 1e-06, - "output_cost_per_token": 2e-06, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/qwen/qwen-14b-chat": { - "max_tokens": 32768, - "input_cost_per_token": 2.7e-07, - "output_cost_per_token": 2.7e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/perplexity/llama-3-sonar-small-32k-online": { - "max_tokens": 28000, - "input_cost_per_token": 2e-07, - "output_cost_per_token": 2e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/nousresearch/nous-capybara-34b": { - "max_tokens": 32768, - "input_cost_per_token": 9e-07, - "output_cost_per_token": 9e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/liuhaotian/llava-13b": { - "max_tokens": 2048, - "input_cost_per_token": 1e-05, - "output_cost_per_token": 1e-05, - "litellm_provider": "openrouter", - "mode": "chat", - "supports_vision": true, - "input_cost_per_image": 0.0 - }, - "openrouter/gryphe/mythomist-7b:free": { - "max_tokens": 32768, - "input_cost_per_token": 0.0, - "output_cost_per_token": 0.0, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/undi95/remm-slerp-l2-13b:extended": { - "max_tokens": 6144, - "input_cost_per_token": 1.125e-06, - "output_cost_per_token": 1.125e-06, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/openrouter/cinematika-7b:free": { - "max_tokens": 8000, - "input_cost_per_token": 0.0, - "output_cost_per_token": 0.0, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/gryphe/mythomax-l2-13b:nitro": { - "max_tokens": 4096, - "input_cost_per_token": 2e-07, - "output_cost_per_token": 2e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/nousresearch/nous-hermes-yi-34b": { - "max_tokens": 4096, - "input_cost_per_token": 7.2e-07, - "output_cost_per_token": 7.2e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/perplexity/llama-3-sonar-large-32k-online": { - "max_tokens": 28000, - "input_cost_per_token": 1e-06, - "output_cost_per_token": 1e-06, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/recursal/rwkv-5-3b-ai-town": { - "max_tokens": 10000, - "input_cost_per_token": 0.0, - "output_cost_per_token": 0.0, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/anthropic/claude-1.2": { - "max_tokens": 100000, - "input_cost_per_token": 8e-06, - "output_cost_per_token": 2.4e-05, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/qwen/qwen-110b-chat": { - "max_tokens": 32768, - "input_cost_per_token": 1.62e-06, - "output_cost_per_token": 1.62e-06, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/openai/gpt-4-1106-preview": { - "max_tokens": 128000, - "input_cost_per_token": 1e-05, - "output_cost_per_token": 3e-05, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/openai/gpt-3.5-turbo-instruct": { - "max_tokens": 4095, - "input_cost_per_token": 1.5e-06, - "output_cost_per_token": 2e-06, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/jondurbin/airoboros-l2-70b": { - "max_tokens": 4096, - "input_cost_per_token": 7e-07, - "output_cost_per_token": 9e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/koboldai/psyfighter-13b-2": { - "max_tokens": 4096, - "input_cost_per_token": 1e-06, - "output_cost_per_token": 1e-06, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/phind/phind-codellama-34b": { - "max_tokens": 4096, - "input_cost_per_token": 7.2e-07, - "output_cost_per_token": 7.2e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/jebcarter/psyfighter-13b": { - "max_tokens": 4096, - "input_cost_per_token": -1.0, - "output_cost_per_token": -1.0, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": -1.0 - }, - "openrouter/alpindale/goliath-120b": { - "max_tokens": 6144, - "input_cost_per_token": 9.375e-06, - "output_cost_per_token": 9.375e-06, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/01-ai/yi-34b-chat": { - "max_tokens": 4096, - "input_cost_per_token": 7.2e-07, - "output_cost_per_token": 7.2e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/neversleep/llama-3-lumimaid-70b": { - "max_tokens": 8192, - "input_cost_per_token": 8e-06, - "output_cost_per_token": 8e-06, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/huggingfaceh4/zephyr-7b-beta": { - "max_tokens": 4096, - "input_cost_per_token": 2e-07, - "output_cost_per_token": 2e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/deepseek/deepseek-chat": { - "max_tokens": 128000, - "input_cost_per_token": 1.4e-07, - "output_cost_per_token": 2.8e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/qwen/qwen-4b-chat": { - "max_tokens": 32768, - "input_cost_per_token": 9e-08, - "output_cost_per_token": 9e-08, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/google/gemma-7b-it": { - "max_tokens": 8192, - "input_cost_per_token": 7e-08, - "output_cost_per_token": 7e-08, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/teknium/openhermes-2-mistral-7b": { - "max_tokens": 4096, - "input_cost_per_token": 1.8e-07, - "output_cost_per_token": 1.8e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/anthropic/claude-instant-1.0": { - "max_tokens": 100000, - "input_cost_per_token": 8e-07, - "output_cost_per_token": 2.4e-06, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/microsoft/wizardlm-2-8x22b": { - "max_tokens": 65536, - "input_cost_per_token": 6.5e-07, - "output_cost_per_token": 6.5e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/qwen/qwen-72b-chat": { - "max_tokens": 32768, - "input_cost_per_token": 9e-07, - "output_cost_per_token": 9e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/openai/gpt-4-32k-0314": { - "max_tokens": 32767, - "input_cost_per_token": 6e-05, - "output_cost_per_token": 0.00012, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/openai/gpt-4-32k": { - "max_tokens": 32767, - "input_cost_per_token": 6e-05, - "output_cost_per_token": 0.00012, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/undi95/toppy-m-7b:nitro": { - "max_tokens": 4096, - "input_cost_per_token": 7e-08, - "output_cost_per_token": 7e-08, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/google/palm-2-codechat-bison-32k": { - "max_tokens": 91750, - "input_cost_per_token": 2.5e-07, - "output_cost_per_token": 5e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/snowflake/snowflake-arctic-instruct": { - "max_tokens": 4096, - "input_cost_per_token": 2.16e-06, - "output_cost_per_token": 2.16e-06, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/gryphe/mythomax-l2-13b:extended": { - "max_tokens": 8192, - "input_cost_per_token": 1.125e-06, - "output_cost_per_token": 1.125e-06, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/01-ai/yi-34b": { - "max_tokens": 4096, - "input_cost_per_token": 7.2e-07, - "output_cost_per_token": 7.2e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/togethercomputer/stripedhyena-nous-7b": { - "max_tokens": 32768, - "input_cost_per_token": 1.8e-07, - "output_cost_per_token": 1.8e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/deepseek/deepseek-coder": { - "max_tokens": 16000, - "input_cost_per_token": 1.4e-07, - "output_cost_per_token": 2.8e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/mistralai/mixtral-8x7b": { - "max_tokens": 32768, - "input_cost_per_token": 5.4e-07, - "output_cost_per_token": 5.4e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/openai/gpt-4-turbo-preview": { - "max_tokens": 128000, - "input_cost_per_token": 1e-05, - "output_cost_per_token": 3e-05, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/xwin-lm/xwin-lm-70b": { - "max_tokens": 8192, - "input_cost_per_token": 3.75e-06, - "output_cost_per_token": 3.75e-06, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/nousresearch/nous-capybara-7b:free": { - "max_tokens": 4096, - "input_cost_per_token": 0.0, - "output_cost_per_token": 0.0, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/anthropic/claude-instant-1.1": { - "max_tokens": 100000, - "input_cost_per_token": 8e-07, - "output_cost_per_token": 2.4e-06, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/intel/neural-chat-7b": { - "max_tokens": 4096, - "input_cost_per_token": 5e-06, - "output_cost_per_token": 5e-06, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/anthropic/claude-2:beta": { - "max_tokens": 200000, - "input_cost_per_token": 8e-06, - "output_cost_per_token": 2.4e-05, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/jondurbin/bagel-34b": { - "max_tokens": 8000, - "input_cost_per_token": -1.0, - "output_cost_per_token": -1.0, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": -1.0 - }, - "openrouter/lizpreciatior/lzlv-70b-fp16-hf": { - "max_tokens": 4096, - "input_cost_per_token": 5.9e-07, - "output_cost_per_token": 7.9e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/openai/gpt-4-0314": { - "max_tokens": 8191, - "input_cost_per_token": 3e-05, - "output_cost_per_token": 6e-05, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/austism/chronos-hermes-13b": { - "max_tokens": 4096, - "input_cost_per_token": 1.3e-07, - "output_cost_per_token": 1.3e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/anthropic/claude-2.1": { - "max_tokens": 200000, - "input_cost_per_token": 8e-06, - "output_cost_per_token": 2.4e-05, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/anthropic/claude-2.1:beta": { - "max_tokens": 200000, - "input_cost_per_token": 8e-06, - "output_cost_per_token": 2.4e-05, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/google/gemma-7b-it:free": { - "max_tokens": 8192, - "input_cost_per_token": 0.0, - "output_cost_per_token": 0.0, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/openai/gpt-4o-2024-05-13": { - "max_tokens": 128000, - "input_cost_per_token": 5e-06, - "output_cost_per_token": 1.5e-05, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.002312 - }, - "openrouter/google/gemini-flash-1.5": { - "max_tokens": 2800000, - "input_cost_per_token": 2.5e-07, - "output_cost_per_token": 7.5e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "supports_vision": true, - "input_cost_per_image": 0.000265 - }, - "openrouter/meta-llama/llama-3-8b": { - "max_tokens": 8192, - "input_cost_per_token": 1.8e-07, - "output_cost_per_token": 1.8e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/openai/gpt-3.5-turbo-0301": { - "max_tokens": 4095, - "input_cost_per_token": 1e-06, - "output_cost_per_token": 2e-06, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/openrouter/cinematika-7b": { - "max_tokens": 8000, - "input_cost_per_token": 1.8e-07, - "output_cost_per_token": 1.8e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/qwen/qwen-32b-chat": { - "max_tokens": 32768, - "input_cost_per_token": 7.2e-07, - "output_cost_per_token": 7.2e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/anthropic/claude-instant-1:beta": { - "max_tokens": 100000, - "input_cost_per_token": 8e-07, - "output_cost_per_token": 2.4e-06, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/mistralai/mistral-7b-instruct:nitro": { - "max_tokens": 32768, - "input_cost_per_token": 2e-07, - "output_cost_per_token": 2e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/cohere/command-r": { - "max_tokens": 128000, - "input_cost_per_token": 5e-07, - "output_cost_per_token": 1.5e-06, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/cohere/command": { - "max_tokens": 4096, - "input_cost_per_token": 1e-06, - "output_cost_per_token": 2e-06, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/neversleep/noromaid-mixtral-8x7b-instruct": { - "max_tokens": 8000, - "input_cost_per_token": 8e-06, - "output_cost_per_token": 8e-06, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/neversleep/noromaid-20b": { - "max_tokens": 8192, - "input_cost_per_token": 2.25e-06, - "output_cost_per_token": 2.25e-06, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/perplexity/llama-3-sonar-large-32k-chat": { - "max_tokens": 32768, - "input_cost_per_token": 1e-06, - "output_cost_per_token": 1e-06, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/anthropic/claude-instant-1": { - "max_tokens": 100000, - "input_cost_per_token": 8e-07, - "output_cost_per_token": 2.4e-06, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/mistralai/mixtral-8x22b": { - "max_tokens": 65536, - "input_cost_per_token": 9e-07, - "output_cost_per_token": 9e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/meta-llama/llama-3-8b-instruct:nitro": { - "max_tokens": 8192, - "input_cost_per_token": 2e-07, - "output_cost_per_token": 2e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/mistralai/mistral-medium": { - "max_tokens": 32000, - "input_cost_per_token": 2.7e-06, - "output_cost_per_token": 8.1e-06, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/undi95/toppy-m-7b": { - "max_tokens": 4096, - "input_cost_per_token": 7e-08, - "output_cost_per_token": 7e-08, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/google/gemini-pro": { - "max_tokens": 91728, - "input_cost_per_token": 1.25e-07, - "output_cost_per_token": 3.75e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0025 - }, - "openrouter/nousresearch/nous-hermes-2-vision-7b": { - "max_tokens": 4096, - "input_cost_per_token": 1e-05, - "output_cost_per_token": 1e-05, - "litellm_provider": "openrouter", - "mode": "chat", - "supports_vision": true, - "input_cost_per_image": 0.0 - }, - "openrouter/01-ai/yi-6b": { - "max_tokens": 4096, - "input_cost_per_token": 1.26e-07, - "output_cost_per_token": 1.26e-07, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 - }, - "openrouter/openai/gpt-3.5-turbo-0125": { - "max_tokens": 16385, - "input_cost_per_token": 5e-07, - "output_cost_per_token": 1.5e-06, - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_image": 0.0 ->>>>>>> 65d82367d (Implemented feature to add input_cost_per_image) } } From c0ceac1dba3ad59394153b1841220ca7dfe0694f Mon Sep 17 00:00:00 2001 From: Rajan Paneru Date: Sun, 19 May 2024 15:14:53 +0930 Subject: [PATCH 010/192] update price & context window file daily, commit and push to the main --- .circleci/config.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3768452ca64..516f2b20d33 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -183,9 +183,6 @@ jobs: pip install "asyncio==3.4.3" pip install "PyGithub==1.59.1" # Run pytest and generate JUnit XML report - - run: - name: Update File for Price & Content Window - command: python ci_cd/auto_update_price_and_context_window_file.py - run: name: Build Docker image command: docker build -t my-app:latest -f Dockerfile.database . From 41d76b477b24db5299a190f0e4bd3c1488fdf10f Mon Sep 17 00:00:00 2001 From: Rajan Paneru Date: Sun, 19 May 2024 17:54:47 +0930 Subject: [PATCH 011/192] Daily JSON update: 2024-05-19 --- .circleci/config.yml | 32 ++++++++++++++++++++++++++++ model_prices_and_context_window.json | 5 +++++ 2 files changed, 37 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 516f2b20d33..635ecd5952d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -311,6 +311,32 @@ jobs: "https://api.github.com/repos/BerriAI/litellm/actions/workflows/ghcr_deploy.yml/dispatches" \ -d "{\"ref\":\"main\", \"inputs\":{\"tag\":\"v${VERSION}\", \"commit_hash\":\"$CIRCLE_SHA1\"}}" +<<<<<<< HEAD +======= + auto_update_price_and_context_window: + docker: + - image: circleci/python:3.9 + steps: + - checkout + - run: + name: Install Dependencies + command: | + pip install aiohttp + - run: + name: Update Price and Context Window + command: | + python ci_cd/auto_update_price_and_context_window_file.py + - run: + name: Commit Changes + command: | + git config user.name "CircleCI" + git config user.email "circleci@example.com" + git add model_prices_and_context_window.json + git commit -m "[skip ci] Daily Price and Context window update: $(date +'%Y-%m-%d')" + git push -q origin main + schedule: + - cron: "0 0 * * *" +>>>>>>> dd2e31c59 (Daily JSON update: 2024-05-19) workflows: version: 2 build_and_test: @@ -341,3 +367,9 @@ workflows: branches: only: - main +<<<<<<< HEAD +======= + daily_update_price_and_context_window: + jobs: + - auto_update_price_and_context_window +>>>>>>> dd2e31c59 (Daily JSON update: 2024-05-19) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index bede36764e9..d7a793017ab 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1696,8 +1696,13 @@ }, "openrouter/databricks/dbrx-instruct": { "max_tokens": 32768, +<<<<<<< HEAD "input_cost_per_token": 0.0000006, "output_cost_per_token": 0.0000006, +======= + "input_cost_per_token": 1.08e-06, + "output_cost_per_token": 1.08e-06, +>>>>>>> dd2e31c59 (Daily JSON update: 2024-05-19) "litellm_provider": "openrouter", "mode": "chat" }, From 192feba2aac0dcb5ebb7f62137bed1fd00ba92e0 Mon Sep 17 00:00:00 2001 From: Rajan Paneru Date: Sun, 19 May 2024 18:38:09 +0930 Subject: [PATCH 012/192] reverted the changes made in circleci --- .circleci/config.yml | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 635ecd5952d..516f2b20d33 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -311,32 +311,6 @@ jobs: "https://api.github.com/repos/BerriAI/litellm/actions/workflows/ghcr_deploy.yml/dispatches" \ -d "{\"ref\":\"main\", \"inputs\":{\"tag\":\"v${VERSION}\", \"commit_hash\":\"$CIRCLE_SHA1\"}}" -<<<<<<< HEAD -======= - auto_update_price_and_context_window: - docker: - - image: circleci/python:3.9 - steps: - - checkout - - run: - name: Install Dependencies - command: | - pip install aiohttp - - run: - name: Update Price and Context Window - command: | - python ci_cd/auto_update_price_and_context_window_file.py - - run: - name: Commit Changes - command: | - git config user.name "CircleCI" - git config user.email "circleci@example.com" - git add model_prices_and_context_window.json - git commit -m "[skip ci] Daily Price and Context window update: $(date +'%Y-%m-%d')" - git push -q origin main - schedule: - - cron: "0 0 * * *" ->>>>>>> dd2e31c59 (Daily JSON update: 2024-05-19) workflows: version: 2 build_and_test: @@ -367,9 +341,3 @@ workflows: branches: only: - main -<<<<<<< HEAD -======= - daily_update_price_and_context_window: - jobs: - - auto_update_price_and_context_window ->>>>>>> dd2e31c59 (Daily JSON update: 2024-05-19) From ac044c4bc6145185ca8f7f2143147513dfff6f41 Mon Sep 17 00:00:00 2001 From: Rajan Paneru Date: Sun, 19 May 2024 18:39:08 +0930 Subject: [PATCH 013/192] created github worflow to update the file and create a pull-request --- .../auto_update_price_and_context_window.yml | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/auto_update_price_and_context_window.yml diff --git a/.github/workflows/auto_update_price_and_context_window.yml b/.github/workflows/auto_update_price_and_context_window.yml new file mode 100644 index 00000000000..a2b677e5392 --- /dev/null +++ b/.github/workflows/auto_update_price_and_context_window.yml @@ -0,0 +1,28 @@ +name: Updates model_prices_and_context_window.json and Create Pull Request + +on: + schedule: + - cron: "0 0 * * 0" # Run every Sundays at midnight + #- cron: "0 0 * * *" # Run daily at midnight + +jobs: + auto_update_price_and_context_window: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Install Dependencies + run: | + pip install aiohttp + - name: Update JSON Data + run: | + python ci_cd/auto_update_price_and_context_window_file.py + - name: Create Pull Request + run: | + git add model_prices_and_context_window.json + git commit -m "Update model_prices_and_context_window.json file: $(date +'%Y-%m-%d')" + gh pr create --title "Update model_prices_and_context_window.json file" \ + --body "Automated update for model_prices_and_context_window.json" \ + --head auto-update-price-and-context-window-$(date +'%Y-%m-%d') \ + --base main + env: + GH_TOKEN: ${{ secrets.GH_TOKEN }} \ No newline at end of file From 874847099a619c8c1b517668f25e072bdf7f3b22 Mon Sep 17 00:00:00 2001 From: Rajan Paneru Date: Sun, 19 May 2024 18:46:08 +0930 Subject: [PATCH 014/192] moved script to the workflow folder --- .github/workflows/auto_update_price_and_context_window.yml | 2 +- .../workflows}/auto_update_price_and_context_window_file.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename {ci_cd => .github/workflows}/auto_update_price_and_context_window_file.py (100%) diff --git a/.github/workflows/auto_update_price_and_context_window.yml b/.github/workflows/auto_update_price_and_context_window.yml index a2b677e5392..e7d65242c19 100644 --- a/.github/workflows/auto_update_price_and_context_window.yml +++ b/.github/workflows/auto_update_price_and_context_window.yml @@ -15,7 +15,7 @@ jobs: pip install aiohttp - name: Update JSON Data run: | - python ci_cd/auto_update_price_and_context_window_file.py + python ".github/workflows/auto_update_price_and_context_window_file.py" - name: Create Pull Request run: | git add model_prices_and_context_window.json diff --git a/ci_cd/auto_update_price_and_context_window_file.py b/.github/workflows/auto_update_price_and_context_window_file.py similarity index 100% rename from ci_cd/auto_update_price_and_context_window_file.py rename to .github/workflows/auto_update_price_and_context_window_file.py From 2e7c1cb736648847a4830017d51e77051e26282c Mon Sep 17 00:00:00 2001 From: Rajan Paneru Date: Wed, 22 May 2024 12:06:42 +0930 Subject: [PATCH 015/192] resolve-conclit: pulled the changes from main --- model_prices_and_context_window.json | 5 ----- 1 file changed, 5 deletions(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index d7a793017ab..bede36764e9 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1696,13 +1696,8 @@ }, "openrouter/databricks/dbrx-instruct": { "max_tokens": 32768, -<<<<<<< HEAD "input_cost_per_token": 0.0000006, "output_cost_per_token": 0.0000006, -======= - "input_cost_per_token": 1.08e-06, - "output_cost_per_token": 1.08e-06, ->>>>>>> dd2e31c59 (Daily JSON update: 2024-05-19) "litellm_provider": "openrouter", "mode": "chat" }, From 4dfbe0be0746806209bf7ad0851c390fc022ce31 Mon Sep 17 00:00:00 2001 From: Rajan Paneru Date: Wed, 22 May 2024 12:15:47 +0930 Subject: [PATCH 016/192] ran the script after resolving conflict --- model_prices_and_context_window.json | 2944 +++++++++++++++++--------- 1 file changed, 1962 insertions(+), 982 deletions(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index bede36764e9..498286f8a69 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1,10 +1,10 @@ { "gpt-4": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 8192, - "max_output_tokens": 4096, - "input_cost_per_token": 0.00003, - "output_cost_per_token": 0.00006, + "max_output_tokens": 4096, + "input_cost_per_token": 3e-05, + "output_cost_per_token": 6e-05, "litellm_provider": "openai", "mode": "chat", "supports_function_calling": true @@ -13,8 +13,8 @@ "max_tokens": 4096, "max_input_tokens": 128000, "max_output_tokens": 4096, - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000015, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 1.5e-05, "litellm_provider": "openai", "mode": "chat", "supports_function_calling": true, @@ -25,8 +25,8 @@ "max_tokens": 4096, "max_input_tokens": 128000, "max_output_tokens": 4096, - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000015, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 1.5e-05, "litellm_provider": "openai", "mode": "chat", "supports_function_calling": true, @@ -37,8 +37,8 @@ "max_tokens": 4096, "max_input_tokens": 128000, "max_output_tokens": 4096, - "input_cost_per_token": 0.00001, - "output_cost_per_token": 0.00003, + "input_cost_per_token": 1e-05, + "output_cost_per_token": 3e-05, "litellm_provider": "openai", "mode": "chat", "supports_function_calling": true, @@ -48,8 +48,8 @@ "max_tokens": 4096, "max_input_tokens": 8192, "max_output_tokens": 4096, - "input_cost_per_token": 0.00003, - "output_cost_per_token": 0.00006, + "input_cost_per_token": 3e-05, + "output_cost_per_token": 6e-05, "litellm_provider": "openai", "mode": "chat" }, @@ -57,8 +57,8 @@ "max_tokens": 4096, "max_input_tokens": 8192, "max_output_tokens": 4096, - "input_cost_per_token": 0.00003, - "output_cost_per_token": 0.00006, + "input_cost_per_token": 3e-05, + "output_cost_per_token": 6e-05, "litellm_provider": "openai", "mode": "chat", "supports_function_calling": true @@ -67,7 +67,7 @@ "max_tokens": 4096, "max_input_tokens": 32768, "max_output_tokens": 4096, - "input_cost_per_token": 0.00006, + "input_cost_per_token": 6e-05, "output_cost_per_token": 0.00012, "litellm_provider": "openai", "mode": "chat" @@ -76,7 +76,7 @@ "max_tokens": 4096, "max_input_tokens": 32768, "max_output_tokens": 4096, - "input_cost_per_token": 0.00006, + "input_cost_per_token": 6e-05, "output_cost_per_token": 0.00012, "litellm_provider": "openai", "mode": "chat" @@ -85,7 +85,7 @@ "max_tokens": 4096, "max_input_tokens": 32768, "max_output_tokens": 4096, - "input_cost_per_token": 0.00006, + "input_cost_per_token": 6e-05, "output_cost_per_token": 0.00012, "litellm_provider": "openai", "mode": "chat" @@ -94,8 +94,8 @@ "max_tokens": 4096, "max_input_tokens": 128000, "max_output_tokens": 4096, - "input_cost_per_token": 0.00001, - "output_cost_per_token": 0.00003, + "input_cost_per_token": 1e-05, + "output_cost_per_token": 3e-05, "litellm_provider": "openai", "mode": "chat", "supports_function_calling": true, @@ -106,8 +106,8 @@ "max_tokens": 4096, "max_input_tokens": 128000, "max_output_tokens": 4096, - "input_cost_per_token": 0.00001, - "output_cost_per_token": 0.00003, + "input_cost_per_token": 1e-05, + "output_cost_per_token": 3e-05, "litellm_provider": "openai", "mode": "chat", "supports_function_calling": true, @@ -118,8 +118,8 @@ "max_tokens": 4096, "max_input_tokens": 128000, "max_output_tokens": 4096, - "input_cost_per_token": 0.00001, - "output_cost_per_token": 0.00003, + "input_cost_per_token": 1e-05, + "output_cost_per_token": 3e-05, "litellm_provider": "openai", "mode": "chat", "supports_function_calling": true, @@ -129,8 +129,8 @@ "max_tokens": 4096, "max_input_tokens": 128000, "max_output_tokens": 4096, - "input_cost_per_token": 0.00001, - "output_cost_per_token": 0.00003, + "input_cost_per_token": 1e-05, + "output_cost_per_token": 3e-05, "litellm_provider": "openai", "mode": "chat", "supports_function_calling": true, @@ -140,8 +140,8 @@ "max_tokens": 4096, "max_input_tokens": 128000, "max_output_tokens": 4096, - "input_cost_per_token": 0.00001, - "output_cost_per_token": 0.00003, + "input_cost_per_token": 1e-05, + "output_cost_per_token": 3e-05, "litellm_provider": "openai", "mode": "chat", "supports_vision": true @@ -150,8 +150,8 @@ "max_tokens": 4096, "max_input_tokens": 128000, "max_output_tokens": 4096, - "input_cost_per_token": 0.00001, - "output_cost_per_token": 0.00003, + "input_cost_per_token": 1e-05, + "output_cost_per_token": 3e-05, "litellm_provider": "openai", "mode": "chat", "supports_vision": true @@ -160,8 +160,8 @@ "max_tokens": 4097, "max_input_tokens": 16385, "max_output_tokens": 4096, - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.000002, + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 2e-06, "litellm_provider": "openai", "mode": "chat", "supports_function_calling": true @@ -170,8 +170,8 @@ "max_tokens": 4097, "max_input_tokens": 4097, "max_output_tokens": 4096, - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.000002, + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 2e-06, "litellm_provider": "openai", "mode": "chat" }, @@ -179,8 +179,8 @@ "max_tokens": 4097, "max_input_tokens": 4097, "max_output_tokens": 4096, - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.000002, + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 2e-06, "litellm_provider": "openai", "mode": "chat", "supports_function_calling": true @@ -189,8 +189,8 @@ "max_tokens": 16385, "max_input_tokens": 16385, "max_output_tokens": 4096, - "input_cost_per_token": 0.0000010, - "output_cost_per_token": 0.0000020, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, "litellm_provider": "openai", "mode": "chat", "supports_function_calling": true, @@ -200,8 +200,8 @@ "max_tokens": 16385, "max_input_tokens": 16385, "max_output_tokens": 4096, - "input_cost_per_token": 0.0000005, - "output_cost_per_token": 0.0000015, + "input_cost_per_token": 5e-07, + "output_cost_per_token": 1.5e-06, "litellm_provider": "openai", "mode": "chat", "supports_function_calling": true, @@ -211,8 +211,8 @@ "max_tokens": 16385, "max_input_tokens": 16385, "max_output_tokens": 4096, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000004, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 4e-06, "litellm_provider": "openai", "mode": "chat" }, @@ -220,8 +220,8 @@ "max_tokens": 16385, "max_input_tokens": 16385, "max_output_tokens": 4096, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000004, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 4e-06, "litellm_provider": "openai", "mode": "chat" }, @@ -229,8 +229,8 @@ "max_tokens": 4097, "max_input_tokens": 4097, "max_output_tokens": 4096, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000006, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 6e-06, "litellm_provider": "openai", "mode": "chat" }, @@ -238,8 +238,8 @@ "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 4096, - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000002, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 2e-06, "litellm_provider": "text-completion-openai", "mode": "completion" }, @@ -247,8 +247,8 @@ "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 4096, - "input_cost_per_token": 0.0000004, - "output_cost_per_token": 0.0000004, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 4e-07, "litellm_provider": "text-completion-openai", "mode": "completion" }, @@ -256,34 +256,34 @@ "max_tokens": 8191, "max_input_tokens": 8191, "output_vector_size": 3072, - "input_cost_per_token": 0.00000013, - "output_cost_per_token": 0.000000, + "input_cost_per_token": 1.3e-07, + "output_cost_per_token": 0.0, "litellm_provider": "openai", "mode": "embedding" }, "text-embedding-3-small": { "max_tokens": 8191, "max_input_tokens": 8191, - "output_vector_size": 1536, - "input_cost_per_token": 0.00000002, - "output_cost_per_token": 0.000000, + "output_vector_size": 1536, + "input_cost_per_token": 2e-08, + "output_cost_per_token": 0.0, "litellm_provider": "openai", "mode": "embedding" }, "text-embedding-ada-002": { "max_tokens": 8191, "max_input_tokens": 8191, - "output_vector_size": 1536, - "input_cost_per_token": 0.0000001, - "output_cost_per_token": 0.000000, + "output_vector_size": 1536, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 0.0, "litellm_provider": "openai", "mode": "embedding" }, "text-embedding-ada-002-v2": { "max_tokens": 8191, "max_input_tokens": 8191, - "input_cost_per_token": 0.0000001, - "output_cost_per_token": 0.000000, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 0.0, "litellm_provider": "openai", "mode": "embedding" }, @@ -291,8 +291,8 @@ "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 0, - "input_cost_per_token": 0.000000, - "output_cost_per_token": 0.000000, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, "litellm_provider": "openai", "mode": "moderations" }, @@ -300,8 +300,8 @@ "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 0, - "input_cost_per_token": 0.000000, - "output_cost_per_token": 0.000000, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, "litellm_provider": "openai", "mode": "moderations" }, @@ -309,83 +309,83 @@ "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 0, - "input_cost_per_token": 0.000000, - "output_cost_per_token": 0.000000, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, "litellm_provider": "openai", "mode": "moderations" }, "256-x-256/dall-e-2": { "mode": "image_generation", - "input_cost_per_pixel": 0.00000024414, + "input_cost_per_pixel": 2.4414e-07, "output_cost_per_pixel": 0.0, "litellm_provider": "openai" }, "512-x-512/dall-e-2": { "mode": "image_generation", - "input_cost_per_pixel": 0.0000000686, + "input_cost_per_pixel": 6.86e-08, "output_cost_per_pixel": 0.0, "litellm_provider": "openai" }, "1024-x-1024/dall-e-2": { "mode": "image_generation", - "input_cost_per_pixel": 0.000000019, + "input_cost_per_pixel": 1.9e-08, "output_cost_per_pixel": 0.0, "litellm_provider": "openai" }, "hd/1024-x-1792/dall-e-3": { "mode": "image_generation", - "input_cost_per_pixel": 0.00000006539, + "input_cost_per_pixel": 6.539e-08, "output_cost_per_pixel": 0.0, "litellm_provider": "openai" }, "hd/1792-x-1024/dall-e-3": { "mode": "image_generation", - "input_cost_per_pixel": 0.00000006539, + "input_cost_per_pixel": 6.539e-08, "output_cost_per_pixel": 0.0, "litellm_provider": "openai" }, "hd/1024-x-1024/dall-e-3": { "mode": "image_generation", - "input_cost_per_pixel": 0.00000007629, + "input_cost_per_pixel": 7.629e-08, "output_cost_per_pixel": 0.0, "litellm_provider": "openai" }, "standard/1024-x-1792/dall-e-3": { "mode": "image_generation", - "input_cost_per_pixel": 0.00000004359, + "input_cost_per_pixel": 4.359e-08, "output_cost_per_pixel": 0.0, "litellm_provider": "openai" }, "standard/1792-x-1024/dall-e-3": { "mode": "image_generation", - "input_cost_per_pixel": 0.00000004359, + "input_cost_per_pixel": 4.359e-08, "output_cost_per_pixel": 0.0, "litellm_provider": "openai" }, "standard/1024-x-1024/dall-e-3": { "mode": "image_generation", - "input_cost_per_pixel": 0.0000000381469, + "input_cost_per_pixel": 3.81469e-08, "output_cost_per_pixel": 0.0, "litellm_provider": "openai" }, "whisper-1": { "mode": "audio_transcription", "input_cost_per_second": 0, - "output_cost_per_second": 0.0001, + "output_cost_per_second": 0.0001, "litellm_provider": "openai" - }, + }, "azure/whisper-1": { "mode": "audio_transcription", - "input_cost_per_second": 0, - "output_cost_per_second": 0.0001, + "input_cost_per_second": 0, + "output_cost_per_second": 0.0001, "litellm_provider": "azure" }, "azure/gpt-4-turbo-2024-04-09": { "max_tokens": 4096, "max_input_tokens": 128000, "max_output_tokens": 4096, - "input_cost_per_token": 0.00001, - "output_cost_per_token": 0.00003, + "input_cost_per_token": 1e-05, + "output_cost_per_token": 3e-05, "litellm_provider": "azure", "mode": "chat", "supports_function_calling": true, @@ -396,8 +396,8 @@ "max_tokens": 4096, "max_input_tokens": 128000, "max_output_tokens": 4096, - "input_cost_per_token": 0.00001, - "output_cost_per_token": 0.00003, + "input_cost_per_token": 1e-05, + "output_cost_per_token": 3e-05, "litellm_provider": "azure", "mode": "chat", "supports_function_calling": true, @@ -407,8 +407,8 @@ "max_tokens": 4096, "max_input_tokens": 128000, "max_output_tokens": 4096, - "input_cost_per_token": 0.00001, - "output_cost_per_token": 0.00003, + "input_cost_per_token": 1e-05, + "output_cost_per_token": 3e-05, "litellm_provider": "azure", "mode": "chat", "supports_function_calling": true, @@ -418,8 +418,8 @@ "max_tokens": 4096, "max_input_tokens": 8192, "max_output_tokens": 4096, - "input_cost_per_token": 0.00003, - "output_cost_per_token": 0.00006, + "input_cost_per_token": 3e-05, + "output_cost_per_token": 6e-05, "litellm_provider": "azure", "mode": "chat", "supports_function_calling": true @@ -428,7 +428,7 @@ "max_tokens": 4096, "max_input_tokens": 32768, "max_output_tokens": 4096, - "input_cost_per_token": 0.00006, + "input_cost_per_token": 6e-05, "output_cost_per_token": 0.00012, "litellm_provider": "azure", "mode": "chat" @@ -437,7 +437,7 @@ "max_tokens": 4096, "max_input_tokens": 32768, "max_output_tokens": 4096, - "input_cost_per_token": 0.00006, + "input_cost_per_token": 6e-05, "output_cost_per_token": 0.00012, "litellm_provider": "azure", "mode": "chat" @@ -446,8 +446,8 @@ "max_tokens": 4096, "max_input_tokens": 8192, "max_output_tokens": 4096, - "input_cost_per_token": 0.00003, - "output_cost_per_token": 0.00006, + "input_cost_per_token": 3e-05, + "output_cost_per_token": 6e-05, "litellm_provider": "azure", "mode": "chat", "supports_function_calling": true @@ -456,9 +456,9 @@ "max_tokens": 4096, "max_input_tokens": 128000, "max_output_tokens": 4096, - "input_cost_per_token": 0.00001, - "output_cost_per_token": 0.00003, - "litellm_provider": "azure", + "input_cost_per_token": 1e-05, + "output_cost_per_token": 3e-05, + "litellm_provider": "azure", "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true @@ -467,9 +467,9 @@ "max_tokens": 4096, "max_input_tokens": 128000, "max_output_tokens": 4096, - "input_cost_per_token": 0.00001, - "output_cost_per_token": 0.00003, - "litellm_provider": "azure", + "input_cost_per_token": 1e-05, + "output_cost_per_token": 3e-05, + "litellm_provider": "azure", "mode": "chat", "supports_vision": true }, @@ -477,8 +477,8 @@ "max_tokens": 4096, "max_input_tokens": 16385, "max_output_tokens": 4096, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000004, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 4e-06, "litellm_provider": "azure", "mode": "chat", "supports_function_calling": true @@ -487,8 +487,8 @@ "max_tokens": 4096, "max_input_tokens": 16384, "max_output_tokens": 4096, - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.000002, + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 2e-06, "litellm_provider": "azure", "mode": "chat", "supports_function_calling": true, @@ -498,8 +498,8 @@ "max_tokens": 4096, "max_input_tokens": 16384, "max_output_tokens": 4096, - "input_cost_per_token": 0.0000005, - "output_cost_per_token": 0.0000015, + "input_cost_per_token": 5e-07, + "output_cost_per_token": 1.5e-06, "litellm_provider": "azure", "mode": "chat", "supports_function_calling": true, @@ -509,8 +509,8 @@ "max_tokens": 4096, "max_input_tokens": 16385, "max_output_tokens": 4096, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000004, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 4e-06, "litellm_provider": "azure", "mode": "chat" }, @@ -518,8 +518,8 @@ "max_tokens": 4096, "max_input_tokens": 4097, "max_output_tokens": 4096, - "input_cost_per_token": 0.0000005, - "output_cost_per_token": 0.0000015, + "input_cost_per_token": 5e-07, + "output_cost_per_token": 1.5e-06, "litellm_provider": "azure", "mode": "chat", "supports_function_calling": true @@ -527,24 +527,24 @@ "azure/gpt-3.5-turbo-instruct-0914": { "max_tokens": 4097, "max_input_tokens": 4097, - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.000002, + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 2e-06, "litellm_provider": "text-completion-openai", "mode": "completion" }, "azure/gpt-35-turbo-instruct": { "max_tokens": 4097, "max_input_tokens": 4097, - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.000002, + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 2e-06, "litellm_provider": "text-completion-openai", "mode": "completion" }, "azure/mistral-large-latest": { "max_tokens": 32000, "max_input_tokens": 32000, - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, "litellm_provider": "azure", "mode": "chat", "supports_function_calling": true @@ -552,18 +552,18 @@ "azure/mistral-large-2402": { "max_tokens": 32000, "max_input_tokens": 32000, - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, "litellm_provider": "azure", "mode": "chat", "supports_function_calling": true }, "azure/command-r-plus": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 128000, "max_output_tokens": 4096, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, "litellm_provider": "azure", "mode": "chat", "supports_function_calling": true @@ -571,83 +571,83 @@ "azure/ada": { "max_tokens": 8191, "max_input_tokens": 8191, - "input_cost_per_token": 0.0000001, - "output_cost_per_token": 0.000000, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 0.0, "litellm_provider": "azure", "mode": "embedding" }, "azure/text-embedding-ada-002": { "max_tokens": 8191, "max_input_tokens": 8191, - "input_cost_per_token": 0.0000001, - "output_cost_per_token": 0.000000, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 0.0, "litellm_provider": "azure", "mode": "embedding" }, "azure/text-embedding-3-large": { "max_tokens": 8191, "max_input_tokens": 8191, - "input_cost_per_token": 0.00000013, - "output_cost_per_token": 0.000000, + "input_cost_per_token": 1.3e-07, + "output_cost_per_token": 0.0, "litellm_provider": "azure", "mode": "embedding" }, "azure/text-embedding-3-small": { "max_tokens": 8191, "max_input_tokens": 8191, - "input_cost_per_token": 0.00000002, - "output_cost_per_token": 0.000000, + "input_cost_per_token": 2e-08, + "output_cost_per_token": 0.0, "litellm_provider": "azure", "mode": "embedding" - }, + }, "azure/standard/1024-x-1024/dall-e-3": { - "input_cost_per_pixel": 0.0000000381469, + "input_cost_per_pixel": 3.81469e-08, "output_cost_per_token": 0.0, - "litellm_provider": "azure", + "litellm_provider": "azure", "mode": "image_generation" }, "azure/hd/1024-x-1024/dall-e-3": { - "input_cost_per_pixel": 0.00000007629, + "input_cost_per_pixel": 7.629e-08, "output_cost_per_token": 0.0, - "litellm_provider": "azure", + "litellm_provider": "azure", "mode": "image_generation" }, "azure/standard/1024-x-1792/dall-e-3": { - "input_cost_per_pixel": 0.00000004359, + "input_cost_per_pixel": 4.359e-08, "output_cost_per_token": 0.0, - "litellm_provider": "azure", + "litellm_provider": "azure", "mode": "image_generation" }, "azure/standard/1792-x-1024/dall-e-3": { - "input_cost_per_pixel": 0.00000004359, + "input_cost_per_pixel": 4.359e-08, "output_cost_per_token": 0.0, - "litellm_provider": "azure", + "litellm_provider": "azure", "mode": "image_generation" }, "azure/hd/1024-x-1792/dall-e-3": { - "input_cost_per_pixel": 0.00000006539, + "input_cost_per_pixel": 6.539e-08, "output_cost_per_token": 0.0, - "litellm_provider": "azure", + "litellm_provider": "azure", "mode": "image_generation" }, "azure/hd/1792-x-1024/dall-e-3": { - "input_cost_per_pixel": 0.00000006539, + "input_cost_per_pixel": 6.539e-08, "output_cost_per_token": 0.0, - "litellm_provider": "azure", + "litellm_provider": "azure", "mode": "image_generation" }, "azure/standard/1024-x-1024/dall-e-2": { "input_cost_per_pixel": 0.0, "output_cost_per_token": 0.0, - "litellm_provider": "azure", + "litellm_provider": "azure", "mode": "image_generation" }, "babbage-002": { "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 4096, - "input_cost_per_token": 0.0000004, - "output_cost_per_token": 0.0000004, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 4e-07, "litellm_provider": "text-completion-openai", "mode": "completion" }, @@ -655,17 +655,17 @@ "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 4096, - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000002, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 2e-06, "litellm_provider": "text-completion-openai", "mode": "completion" - }, + }, "gpt-3.5-turbo-instruct": { "max_tokens": 4096, "max_input_tokens": 8192, "max_output_tokens": 4096, - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.000002, + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 2e-06, "litellm_provider": "text-completion-openai", "mode": "completion" }, @@ -673,18 +673,17 @@ "max_tokens": 4097, "max_input_tokens": 8192, "max_output_tokens": 4097, - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.000002, + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 2e-06, "litellm_provider": "text-completion-openai", "mode": "completion" - }, "claude-instant-1": { "max_tokens": 8191, "max_input_tokens": 100000, "max_output_tokens": 8191, - "input_cost_per_token": 0.00000163, - "output_cost_per_token": 0.00000551, + "input_cost_per_token": 1.63e-06, + "output_cost_per_token": 5.51e-06, "litellm_provider": "anthropic", "mode": "chat" }, @@ -692,8 +691,8 @@ "max_tokens": 8191, "max_input_tokens": 32000, "max_output_tokens": 8191, - "input_cost_per_token": 0.00000015, - "output_cost_per_token": 0.00000046, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 4.6e-07, "litellm_provider": "mistral", "mode": "chat" }, @@ -701,8 +700,8 @@ "max_tokens": 8191, "max_input_tokens": 32000, "max_output_tokens": 8191, - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000006, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, "litellm_provider": "mistral", "supports_function_calling": true, "mode": "chat" @@ -711,8 +710,8 @@ "max_tokens": 8191, "max_input_tokens": 32000, "max_output_tokens": 8191, - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000006, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, "litellm_provider": "mistral", "supports_function_calling": true, "mode": "chat" @@ -721,8 +720,8 @@ "max_tokens": 8191, "max_input_tokens": 32000, "max_output_tokens": 8191, - "input_cost_per_token": 0.0000027, - "output_cost_per_token": 0.0000081, + "input_cost_per_token": 2.7e-06, + "output_cost_per_token": 8.1e-06, "litellm_provider": "mistral", "mode": "chat" }, @@ -730,8 +729,8 @@ "max_tokens": 8191, "max_input_tokens": 32000, "max_output_tokens": 8191, - "input_cost_per_token": 0.0000027, - "output_cost_per_token": 0.0000081, + "input_cost_per_token": 2.7e-06, + "output_cost_per_token": 8.1e-06, "litellm_provider": "mistral", "mode": "chat" }, @@ -739,8 +738,8 @@ "max_tokens": 8191, "max_input_tokens": 32000, "max_output_tokens": 8191, - "input_cost_per_token": 0.0000027, - "output_cost_per_token": 0.0000081, + "input_cost_per_token": 2.7e-06, + "output_cost_per_token": 8.1e-06, "litellm_provider": "mistral", "mode": "chat" }, @@ -748,8 +747,8 @@ "max_tokens": 8191, "max_input_tokens": 32000, "max_output_tokens": 8191, - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, "litellm_provider": "mistral", "mode": "chat", "supports_function_calling": true @@ -758,8 +757,8 @@ "max_tokens": 8191, "max_input_tokens": 32000, "max_output_tokens": 8191, - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, "litellm_provider": "mistral", "mode": "chat", "supports_function_calling": true @@ -768,8 +767,8 @@ "max_tokens": 8191, "max_input_tokens": 32000, "max_output_tokens": 8191, - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000006, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, "litellm_provider": "mistral", "mode": "chat", "supports_function_calling": true @@ -777,7 +776,7 @@ "mistral/mistral-embed": { "max_tokens": 8192, "max_input_tokens": 8192, - "input_cost_per_token": 0.000000111, + "input_cost_per_token": 1.11e-07, "litellm_provider": "mistral", "mode": "embedding" }, @@ -785,8 +784,8 @@ "max_tokens": 4096, "max_input_tokens": 32000, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000014, - "output_cost_per_token": 0.00000028, + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 2.8e-07, "litellm_provider": "deepseek", "mode": "chat" }, @@ -794,8 +793,8 @@ "max_tokens": 4096, "max_input_tokens": 16000, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000014, - "output_cost_per_token": 0.00000028, + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 2.8e-07, "litellm_provider": "deepseek", "mode": "chat" }, @@ -803,8 +802,8 @@ "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000070, - "output_cost_per_token": 0.00000080, + "input_cost_per_token": 7e-07, + "output_cost_per_token": 8e-07, "litellm_provider": "groq", "mode": "chat", "supports_function_calling": true @@ -813,8 +812,8 @@ "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, - "input_cost_per_token": 0.00000010, - "output_cost_per_token": 0.00000010, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 1e-07, "litellm_provider": "groq", "mode": "chat", "supports_function_calling": true @@ -823,8 +822,8 @@ "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, - "input_cost_per_token": 0.00000064, - "output_cost_per_token": 0.00000080, + "input_cost_per_token": 6.4e-07, + "output_cost_per_token": 8e-07, "litellm_provider": "groq", "mode": "chat", "supports_function_calling": true @@ -833,8 +832,8 @@ "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, - "input_cost_per_token": 0.00000027, - "output_cost_per_token": 0.00000027, + "input_cost_per_token": 2.7e-07, + "output_cost_per_token": 2.7e-07, "litellm_provider": "groq", "mode": "chat", "supports_function_calling": true @@ -843,8 +842,8 @@ "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, - "input_cost_per_token": 0.00000010, - "output_cost_per_token": 0.00000010, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 1e-07, "litellm_provider": "groq", "mode": "chat", "supports_function_calling": true @@ -853,8 +852,8 @@ "max_tokens": 8191, "max_input_tokens": 100000, "max_output_tokens": 8191, - "input_cost_per_token": 0.000000163, - "output_cost_per_token": 0.000000551, + "input_cost_per_token": 1.63e-07, + "output_cost_per_token": 5.51e-07, "litellm_provider": "anthropic", "mode": "chat" }, @@ -862,8 +861,8 @@ "max_tokens": 8191, "max_input_tokens": 100000, "max_output_tokens": 8191, - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, "litellm_provider": "anthropic", "mode": "chat" }, @@ -871,8 +870,8 @@ "max_tokens": 8191, "max_input_tokens": 200000, "max_output_tokens": 8191, - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, "litellm_provider": "anthropic", "mode": "chat" }, @@ -880,8 +879,8 @@ "max_tokens": 4096, "max_input_tokens": 200000, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000025, - "output_cost_per_token": 0.00000125, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 1.25e-06, "litellm_provider": "anthropic", "mode": "chat", "supports_function_calling": true, @@ -892,8 +891,8 @@ "max_tokens": 4096, "max_input_tokens": 200000, "max_output_tokens": 4096, - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.000075, + "input_cost_per_token": 1.5e-05, + "output_cost_per_token": 7.5e-05, "litellm_provider": "anthropic", "mode": "chat", "supports_function_calling": true, @@ -904,8 +903,8 @@ "max_tokens": 4096, "max_input_tokens": 200000, "max_output_tokens": 4096, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, "litellm_provider": "anthropic", "mode": "chat", "supports_function_calling": true, @@ -916,8 +915,8 @@ "max_tokens": 1024, "max_input_tokens": 8192, "max_output_tokens": 1024, - "input_cost_per_token": 0.000000125, - "output_cost_per_token": 0.000000125, + "input_cost_per_token": 1.25e-07, + "output_cost_per_token": 1.25e-07, "litellm_provider": "vertex_ai-text-models", "mode": "completion", "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" @@ -926,8 +925,8 @@ "max_tokens": 1024, "max_input_tokens": 8192, "max_output_tokens": 1024, - "input_cost_per_token": 0.000000125, - "output_cost_per_token": 0.000000125, + "input_cost_per_token": 1.25e-07, + "output_cost_per_token": 1.25e-07, "litellm_provider": "vertex_ai-text-models", "mode": "completion", "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" @@ -936,8 +935,8 @@ "max_tokens": 1024, "max_input_tokens": 8192, "max_output_tokens": 1024, - "input_cost_per_token": 0.00001, - "output_cost_per_token": 0.000028, + "input_cost_per_token": 1e-05, + "output_cost_per_token": 2.8e-05, "litellm_provider": "vertex_ai-text-models", "mode": "completion", "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" @@ -946,8 +945,8 @@ "max_tokens": 1024, "max_input_tokens": 8192, "max_output_tokens": 1024, - "input_cost_per_token": 0.00001, - "output_cost_per_token": 0.000028, + "input_cost_per_token": 1e-05, + "output_cost_per_token": 2.8e-05, "litellm_provider": "vertex_ai-text-models", "mode": "completion", "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" @@ -956,8 +955,8 @@ "max_tokens": 4096, "max_input_tokens": 8192, "max_output_tokens": 4096, - "input_cost_per_token": 0.000000125, - "output_cost_per_token": 0.000000125, + "input_cost_per_token": 1.25e-07, + "output_cost_per_token": 1.25e-07, "litellm_provider": "vertex_ai-chat-models", "mode": "chat", "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" @@ -966,8 +965,8 @@ "max_tokens": 4096, "max_input_tokens": 8192, "max_output_tokens": 4096, - "input_cost_per_token": 0.000000125, - "output_cost_per_token": 0.000000125, + "input_cost_per_token": 1.25e-07, + "output_cost_per_token": 1.25e-07, "litellm_provider": "vertex_ai-chat-models", "mode": "chat", "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" @@ -976,8 +975,8 @@ "max_tokens": 4096, "max_input_tokens": 8192, "max_output_tokens": 4096, - "input_cost_per_token": 0.000000125, - "output_cost_per_token": 0.000000125, + "input_cost_per_token": 1.25e-07, + "output_cost_per_token": 1.25e-07, "litellm_provider": "vertex_ai-chat-models", "mode": "chat", "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" @@ -986,8 +985,8 @@ "max_tokens": 8192, "max_input_tokens": 32000, "max_output_tokens": 8192, - "input_cost_per_token": 0.000000125, - "output_cost_per_token": 0.000000125, + "input_cost_per_token": 1.25e-07, + "output_cost_per_token": 1.25e-07, "litellm_provider": "vertex_ai-chat-models", "mode": "chat", "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" @@ -996,8 +995,8 @@ "max_tokens": 1024, "max_input_tokens": 6144, "max_output_tokens": 1024, - "input_cost_per_token": 0.000000125, - "output_cost_per_token": 0.000000125, + "input_cost_per_token": 1.25e-07, + "output_cost_per_token": 1.25e-07, "litellm_provider": "vertex_ai-code-text-models", "mode": "chat", "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" @@ -1006,8 +1005,8 @@ "max_tokens": 1024, "max_input_tokens": 6144, "max_output_tokens": 1024, - "input_cost_per_token": 0.000000125, - "output_cost_per_token": 0.000000125, + "input_cost_per_token": 1.25e-07, + "output_cost_per_token": 1.25e-07, "litellm_provider": "vertex_ai-code-text-models", "mode": "completion", "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" @@ -1016,8 +1015,8 @@ "max_tokens": 64, "max_input_tokens": 2048, "max_output_tokens": 64, - "input_cost_per_token": 0.000000125, - "output_cost_per_token": 0.000000125, + "input_cost_per_token": 1.25e-07, + "output_cost_per_token": 1.25e-07, "litellm_provider": "vertex_ai-code-text-models", "mode": "completion", "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" @@ -1026,8 +1025,8 @@ "max_tokens": 64, "max_input_tokens": 2048, "max_output_tokens": 64, - "input_cost_per_token": 0.000000125, - "output_cost_per_token": 0.000000125, + "input_cost_per_token": 1.25e-07, + "output_cost_per_token": 1.25e-07, "litellm_provider": "vertex_ai-code-text-models", "mode": "completion", "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" @@ -1036,8 +1035,8 @@ "max_tokens": 64, "max_input_tokens": 2048, "max_output_tokens": 64, - "input_cost_per_token": 0.000000125, - "output_cost_per_token": 0.000000125, + "input_cost_per_token": 1.25e-07, + "output_cost_per_token": 1.25e-07, "litellm_provider": "vertex_ai-code-text-models", "mode": "completion", "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" @@ -1046,8 +1045,8 @@ "max_tokens": 1024, "max_input_tokens": 6144, "max_output_tokens": 1024, - "input_cost_per_token": 0.000000125, - "output_cost_per_token": 0.000000125, + "input_cost_per_token": 1.25e-07, + "output_cost_per_token": 1.25e-07, "litellm_provider": "vertex_ai-code-chat-models", "mode": "chat", "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" @@ -1056,8 +1055,8 @@ "max_tokens": 1024, "max_input_tokens": 6144, "max_output_tokens": 1024, - "input_cost_per_token": 0.000000125, - "output_cost_per_token": 0.000000125, + "input_cost_per_token": 1.25e-07, + "output_cost_per_token": 1.25e-07, "litellm_provider": "vertex_ai-code-chat-models", "mode": "chat", "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" @@ -1066,8 +1065,8 @@ "max_tokens": 8192, "max_input_tokens": 32000, "max_output_tokens": 8192, - "input_cost_per_token": 0.000000125, - "output_cost_per_token": 0.000000125, + "input_cost_per_token": 1.25e-07, + "output_cost_per_token": 1.25e-07, "litellm_provider": "vertex_ai-code-chat-models", "mode": "chat", "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" @@ -1076,56 +1075,56 @@ "max_tokens": 8192, "max_input_tokens": 32760, "max_output_tokens": 8192, - "input_cost_per_token": 0.00000025, - "output_cost_per_token": 0.0000005, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 5e-07, "litellm_provider": "vertex_ai-language-models", "mode": "chat", "supports_function_calling": true, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" }, - "gemini-1.0-pro": { + "gemini-1.0-pro": { "max_tokens": 8192, "max_input_tokens": 32760, "max_output_tokens": 8192, - "input_cost_per_token": 0.00000025, - "output_cost_per_token": 0.0000005, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 5e-07, "litellm_provider": "vertex_ai-language-models", "mode": "chat", "supports_function_calling": true, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" }, - "gemini-1.0-pro-001": { + "gemini-1.0-pro-001": { "max_tokens": 8192, "max_input_tokens": 32760, "max_output_tokens": 8192, - "input_cost_per_token": 0.00000025, - "output_cost_per_token": 0.0000005, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 5e-07, "litellm_provider": "vertex_ai-language-models", "mode": "chat", "supports_function_calling": true, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" }, - "gemini-1.0-pro-002": { + "gemini-1.0-pro-002": { "max_tokens": 8192, "max_input_tokens": 32760, "max_output_tokens": 8192, - "input_cost_per_token": 0.00000025, - "output_cost_per_token": 0.0000005, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 5e-07, "litellm_provider": "vertex_ai-language-models", "mode": "chat", "supports_function_calling": true, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" }, - "gemini-1.5-pro": { + "gemini-1.5-pro": { "max_tokens": 8192, "max_input_tokens": 1000000, "max_output_tokens": 8192, - "input_cost_per_token": 0.000000625, - "output_cost_per_token": 0.000001875, + "input_cost_per_token": 6.25e-07, + "output_cost_per_token": 1.875e-06, "litellm_provider": "vertex_ai-language-models", "mode": "chat", "supports_function_calling": true, - "supports_tool_choice": true, + "supports_tool_choice": true, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" }, "gemini-1.5-flash-preview-0514": { @@ -1138,7 +1137,7 @@ "max_audio_length_hours": 8.4, "max_audio_per_prompt": 1, "max_pdf_size_mb": 30, - "input_cost_per_token": 0, + "input_cost_per_token": 0, "output_cost_per_token": 0, "litellm_provider": "vertex_ai-language-models", "mode": "chat", @@ -1146,40 +1145,40 @@ "supports_vision": true, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" }, - "gemini-1.5-pro-preview-0514": { + "gemini-1.5-pro-preview-0514": { "max_tokens": 8192, "max_input_tokens": 1000000, "max_output_tokens": 8192, - "input_cost_per_token": 0.000000625, - "output_cost_per_token": 0.000001875, + "input_cost_per_token": 6.25e-07, + "output_cost_per_token": 1.875e-06, "litellm_provider": "vertex_ai-language-models", "mode": "chat", "supports_function_calling": true, - "supports_tool_choice": true, + "supports_tool_choice": true, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" }, - "gemini-1.5-pro-preview-0215": { + "gemini-1.5-pro-preview-0215": { "max_tokens": 8192, "max_input_tokens": 1000000, "max_output_tokens": 8192, - "input_cost_per_token": 0.000000625, - "output_cost_per_token": 0.000001875, + "input_cost_per_token": 6.25e-07, + "output_cost_per_token": 1.875e-06, "litellm_provider": "vertex_ai-language-models", "mode": "chat", "supports_function_calling": true, - "supports_tool_choice": true, + "supports_tool_choice": true, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" }, "gemini-1.5-pro-preview-0409": { "max_tokens": 8192, "max_input_tokens": 1000000, "max_output_tokens": 8192, - "input_cost_per_token": 0.000000625, - "output_cost_per_token": 0.000001875, + "input_cost_per_token": 6.25e-07, + "output_cost_per_token": 1.875e-06, "litellm_provider": "vertex_ai-language-models", "mode": "chat", "supports_function_calling": true, - "supports_tool_choice": true, + "supports_tool_choice": true, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" }, "gemini-experimental": { @@ -1191,7 +1190,7 @@ "litellm_provider": "vertex_ai-language-models", "mode": "chat", "supports_function_calling": false, - "supports_tool_choice": true, + "supports_tool_choice": true, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" }, "gemini-pro-vision": { @@ -1201,8 +1200,8 @@ "max_images_per_prompt": 16, "max_videos_per_prompt": 1, "max_video_length": 2, - "input_cost_per_token": 0.00000025, - "output_cost_per_token": 0.0000005, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 5e-07, "litellm_provider": "vertex_ai-vision-models", "mode": "chat", "supports_function_calling": true, @@ -1216,8 +1215,8 @@ "max_images_per_prompt": 16, "max_videos_per_prompt": 1, "max_video_length": 2, - "input_cost_per_token": 0.00000025, - "output_cost_per_token": 0.0000005, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 5e-07, "litellm_provider": "vertex_ai-vision-models", "mode": "chat", "supports_function_calling": true, @@ -1231,8 +1230,8 @@ "max_images_per_prompt": 16, "max_videos_per_prompt": 1, "max_video_length": 2, - "input_cost_per_token": 0.00000025, - "output_cost_per_token": 0.0000005, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 5e-07, "litellm_provider": "vertex_ai-vision-models", "mode": "chat", "supports_function_calling": true, @@ -1243,19 +1242,19 @@ "max_tokens": 4096, "max_input_tokens": 200000, "max_output_tokens": 4096, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, "litellm_provider": "vertex_ai-anthropic_models", "mode": "chat", "supports_function_calling": true, "supports_vision": true }, "vertex_ai/claude-3-haiku@20240307": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 200000, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000025, - "output_cost_per_token": 0.00000125, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 1.25e-06, "litellm_provider": "vertex_ai-anthropic_models", "mode": "chat", "supports_function_calling": true, @@ -1265,8 +1264,8 @@ "max_tokens": 4096, "max_input_tokens": 200000, "max_output_tokens": 4096, - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.0000075, + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 7.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "mode": "chat", "supports_function_calling": true, @@ -1276,7 +1275,7 @@ "max_tokens": 3072, "max_input_tokens": 3072, "output_vector_size": 768, - "input_cost_per_token": 0.00000000625, + "input_cost_per_token": 6.25e-09, "output_cost_per_token": 0, "litellm_provider": "vertex_ai-embedding-models", "mode": "embedding", @@ -1286,7 +1285,7 @@ "max_tokens": 3072, "max_input_tokens": 3072, "output_vector_size": 768, - "input_cost_per_token": 0.00000000625, + "input_cost_per_token": 6.25e-09, "output_cost_per_token": 0, "litellm_provider": "vertex_ai-embedding-models", "mode": "embedding", @@ -1296,7 +1295,7 @@ "max_tokens": 3072, "max_input_tokens": 3072, "output_vector_size": 768, - "input_cost_per_token": 0.00000000625, + "input_cost_per_token": 6.25e-09, "output_cost_per_token": 0, "litellm_provider": "vertex_ai-embedding-models", "mode": "embedding", @@ -1306,7 +1305,7 @@ "max_tokens": 3072, "max_input_tokens": 3072, "output_vector_size": 768, - "input_cost_per_token": 0.00000000625, + "input_cost_per_token": 6.25e-09, "output_cost_per_token": 0, "litellm_provider": "vertex_ai-embedding-models", "mode": "embedding", @@ -1316,7 +1315,7 @@ "max_tokens": 3072, "max_input_tokens": 3072, "output_vector_size": 768, - "input_cost_per_token": 0.00000000625, + "input_cost_per_token": 6.25e-09, "output_cost_per_token": 0, "litellm_provider": "vertex_ai-embedding-models", "mode": "embedding", @@ -1326,18 +1325,18 @@ "max_tokens": 3072, "max_input_tokens": 3072, "output_vector_size": 768, - "input_cost_per_token": 0.00000000625, - "input_cost_per_token_batch_requests": 0.000000005, + "input_cost_per_token": 6.25e-09, + "input_cost_per_token_batch_requests": 5e-09, "output_cost_per_token": 0, "litellm_provider": "vertex_ai-embedding-models", "mode": "embedding", "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, - "text-multilingual-embedding-preview-0409":{ + "text-multilingual-embedding-preview-0409": { "max_tokens": 3072, "max_input_tokens": 3072, "output_vector_size": 768, - "input_cost_per_token": 0.00000000625, + "input_cost_per_token": 6.25e-09, "output_cost_per_token": 0, "litellm_provider": "vertex_ai-embedding-models", "mode": "embedding", @@ -1347,8 +1346,8 @@ "max_tokens": 4096, "max_input_tokens": 8192, "max_output_tokens": 4096, - "input_cost_per_token": 0.000000125, - "output_cost_per_token": 0.000000125, + "input_cost_per_token": 1.25e-07, + "output_cost_per_token": 1.25e-07, "litellm_provider": "palm", "mode": "chat", "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" @@ -1357,8 +1356,8 @@ "max_tokens": 4096, "max_input_tokens": 8192, "max_output_tokens": 4096, - "input_cost_per_token": 0.000000125, - "output_cost_per_token": 0.000000125, + "input_cost_per_token": 1.25e-07, + "output_cost_per_token": 1.25e-07, "litellm_provider": "palm", "mode": "chat", "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" @@ -1367,8 +1366,8 @@ "max_tokens": 1024, "max_input_tokens": 8192, "max_output_tokens": 1024, - "input_cost_per_token": 0.000000125, - "output_cost_per_token": 0.000000125, + "input_cost_per_token": 1.25e-07, + "output_cost_per_token": 1.25e-07, "litellm_provider": "palm", "mode": "completion", "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" @@ -1377,8 +1376,8 @@ "max_tokens": 1024, "max_input_tokens": 8192, "max_output_tokens": 1024, - "input_cost_per_token": 0.000000125, - "output_cost_per_token": 0.000000125, + "input_cost_per_token": 1.25e-07, + "output_cost_per_token": 1.25e-07, "litellm_provider": "palm", "mode": "completion", "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" @@ -1387,8 +1386,8 @@ "max_tokens": 1024, "max_input_tokens": 8192, "max_output_tokens": 1024, - "input_cost_per_token": 0.000000125, - "output_cost_per_token": 0.000000125, + "input_cost_per_token": 1.25e-07, + "output_cost_per_token": 1.25e-07, "litellm_provider": "palm", "mode": "completion", "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" @@ -1397,8 +1396,8 @@ "max_tokens": 1024, "max_input_tokens": 8192, "max_output_tokens": 1024, - "input_cost_per_token": 0.000000125, - "output_cost_per_token": 0.000000125, + "input_cost_per_token": 1.25e-07, + "output_cost_per_token": 1.25e-07, "litellm_provider": "palm", "mode": "completion", "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" @@ -1413,7 +1412,7 @@ "max_audio_length_hours": 8.4, "max_audio_per_prompt": 1, "max_pdf_size_mb": 30, - "input_cost_per_token": 0, + "input_cost_per_token": 0, "output_cost_per_token": 0, "litellm_provider": "vertex_ai-language-models", "mode": "chat", @@ -1425,7 +1424,7 @@ "max_tokens": 8192, "max_input_tokens": 32760, "max_output_tokens": 8192, - "input_cost_per_token": 0.0, + "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "litellm_provider": "gemini", "mode": "chat", @@ -1436,33 +1435,33 @@ "max_tokens": 8192, "max_input_tokens": 1000000, "max_output_tokens": 8192, - "input_cost_per_token": 0, + "input_cost_per_token": 0, "output_cost_per_token": 0, "litellm_provider": "gemini", "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "supports_tool_choice": true, + "supports_tool_choice": true, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" }, "gemini/gemini-1.5-pro-latest": { "max_tokens": 8192, "max_input_tokens": 1048576, "max_output_tokens": 8192, - "input_cost_per_token": 0, + "input_cost_per_token": 0, "output_cost_per_token": 0, "litellm_provider": "gemini", "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "supports_tool_choice": true, + "supports_tool_choice": true, "source": "https://ai.google.dev/models/gemini" }, "gemini/gemini-pro-vision": { "max_tokens": 2048, "max_input_tokens": 30720, "max_output_tokens": 2048, - "input_cost_per_token": 0.0, + "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "litellm_provider": "gemini", "mode": "chat", @@ -1471,76 +1470,76 @@ "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" }, "command-r": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 128000, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000050, - "output_cost_per_token": 0.0000015, + "input_cost_per_token": 5e-07, + "output_cost_per_token": 1.5e-06, "litellm_provider": "cohere_chat", "mode": "chat", "supports_function_calling": true }, "command-light": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.000015, + "input_cost_per_token": 1.5e-05, + "output_cost_per_token": 1.5e-05, "litellm_provider": "cohere_chat", "mode": "chat" }, "command-r-plus": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 128000, "max_output_tokens": 4096, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, "litellm_provider": "cohere_chat", "mode": "chat", "supports_function_calling": true }, "command-nightly": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.000015, + "input_cost_per_token": 1.5e-05, + "output_cost_per_token": 1.5e-05, "litellm_provider": "cohere", "mode": "completion" }, - "command": { - "max_tokens": 4096, + "command": { + "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.000015, + "input_cost_per_token": 1.5e-05, + "output_cost_per_token": 1.5e-05, "litellm_provider": "cohere", "mode": "completion" }, - "command-medium-beta": { - "max_tokens": 4096, + "command-medium-beta": { + "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.000015, + "input_cost_per_token": 1.5e-05, + "output_cost_per_token": 1.5e-05, "litellm_provider": "cohere", "mode": "completion" }, - "command-xlarge-beta": { - "max_tokens": 4096, + "command-xlarge-beta": { + "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.000015, + "input_cost_per_token": 1.5e-05, + "output_cost_per_token": 1.5e-05, "litellm_provider": "cohere", "mode": "completion" }, "replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.0000, - "output_cost_per_token": 0.0000, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, "litellm_provider": "replicate", "mode": "chat" }, @@ -1548,8 +1547,8 @@ "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.0000001, - "output_cost_per_token": 0.0000005, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 5e-07, "litellm_provider": "replicate", "mode": "chat" }, @@ -1557,8 +1556,8 @@ "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.0000001, - "output_cost_per_token": 0.0000005, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 5e-07, "litellm_provider": "replicate", "mode": "chat" }, @@ -1566,8 +1565,8 @@ "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000065, - "output_cost_per_token": 0.00000275, + "input_cost_per_token": 6.5e-07, + "output_cost_per_token": 2.75e-06, "litellm_provider": "replicate", "mode": "chat" }, @@ -1575,8 +1574,8 @@ "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000065, - "output_cost_per_token": 0.00000275, + "input_cost_per_token": 6.5e-07, + "output_cost_per_token": 2.75e-06, "litellm_provider": "replicate", "mode": "chat" }, @@ -1584,8 +1583,8 @@ "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000005, - "output_cost_per_token": 0.00000025, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 2.5e-07, "litellm_provider": "replicate", "mode": "chat" }, @@ -1593,8 +1592,8 @@ "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000005, - "output_cost_per_token": 0.00000025, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 2.5e-07, "litellm_provider": "replicate", "mode": "chat" }, @@ -1602,8 +1601,8 @@ "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000065, - "output_cost_per_token": 0.00000275, + "input_cost_per_token": 6.5e-07, + "output_cost_per_token": 2.75e-06, "litellm_provider": "replicate", "mode": "chat" }, @@ -1611,8 +1610,8 @@ "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000065, - "output_cost_per_token": 0.00000275, + "input_cost_per_token": 6.5e-07, + "output_cost_per_token": 2.75e-06, "litellm_provider": "replicate", "mode": "chat" }, @@ -1620,8 +1619,8 @@ "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000005, - "output_cost_per_token": 0.00000025, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 2.5e-07, "litellm_provider": "replicate", "mode": "chat" }, @@ -1629,8 +1628,8 @@ "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000005, - "output_cost_per_token": 0.00000025, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 2.5e-07, "litellm_provider": "replicate", "mode": "chat" }, @@ -1638,8 +1637,8 @@ "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000005, - "output_cost_per_token": 0.00000025, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 2.5e-07, "litellm_provider": "replicate", "mode": "chat" }, @@ -1647,8 +1646,8 @@ "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000005, - "output_cost_per_token": 0.00000025, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 2.5e-07, "litellm_provider": "replicate", "mode": "chat" }, @@ -1656,25 +1655,25 @@ "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.0000003, - "output_cost_per_token": 0.000001, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1e-06, "litellm_provider": "replicate", "mode": "chat" }, "openrouter/microsoft/wizardlm-2-8x22b:nitro": { "max_tokens": 65536, - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000001, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 1e-06, "litellm_provider": "openrouter", "mode": "chat" }, "openrouter/google/gemini-pro-1.5": { - "max_tokens": 8192, + "max_tokens": 2800000, "max_input_tokens": 1000000, "max_output_tokens": 8192, - "input_cost_per_token": 0.0000025, - "output_cost_per_token": 0.0000075, - "input_cost_per_image": 0.00265, + "input_cost_per_token": 2.5e-06, + "output_cost_per_token": 7.5e-06, + "input_cost_per_image": 0.00265, "litellm_provider": "openrouter", "mode": "chat", "supports_function_calling": true, @@ -1682,30 +1681,33 @@ }, "openrouter/mistralai/mixtral-8x22b-instruct": { "max_tokens": 65536, - "input_cost_per_token": 0.00000065, - "output_cost_per_token": 0.00000065, + "input_cost_per_token": 6.5e-07, + "output_cost_per_token": 6.5e-07, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/cohere/command-r-plus": { "max_tokens": 128000, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/databricks/dbrx-instruct": { "max_tokens": 32768, - "input_cost_per_token": 0.0000006, - "output_cost_per_token": 0.0000006, + "input_cost_per_token": 1.08e-06, + "output_cost_per_token": 1.08e-06, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/anthropic/claude-3-haiku": { "max_tokens": 200000, - "input_cost_per_token": 0.00000025, - "output_cost_per_token": 0.00000125, - "input_cost_per_image": 0.0004, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 1.25e-06, + "input_cost_per_image": 0.0004, "litellm_provider": "openrouter", "mode": "chat", "supports_function_calling": true, @@ -1713,9 +1715,9 @@ }, "openrouter/anthropic/claude-3-sonnet": { "max_tokens": 200000, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "input_cost_per_image": 0.0048, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "input_cost_per_image": 0.0048, "litellm_provider": "openrouter", "mode": "chat", "supports_function_calling": true, @@ -1723,23 +1725,25 @@ }, "openrouter/mistralai/mistral-large": { "max_tokens": 32000, - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/cognitivecomputations/dolphin-mixtral-8x7b": { - "max_tokens": 32769, - "input_cost_per_token": 0.0000005, - "output_cost_per_token": 0.0000005, + "max_tokens": 32768, + "input_cost_per_token": 5e-07, + "output_cost_per_token": 5e-07, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/google/gemini-pro-vision": { "max_tokens": 45875, - "input_cost_per_token": 0.000000125, - "output_cost_per_token": 0.000000375, - "input_cost_per_image": 0.0025, + "input_cost_per_token": 1.25e-07, + "output_cost_per_token": 3.75e-07, + "input_cost_per_image": 0.0025, "litellm_provider": "openrouter", "mode": "chat", "supports_function_calling": true, @@ -1747,219 +1751,244 @@ }, "openrouter/fireworks/firellava-13b": { "max_tokens": 4096, - "input_cost_per_token": 0.0000002, - "output_cost_per_token": 0.0000002, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 2e-07, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0001152, + "supports_vision": true }, "openrouter/meta-llama/llama-3-8b-instruct:free": { "max_tokens": 8192, "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/meta-llama/llama-3-8b-instruct:extended": { "max_tokens": 16384, - "input_cost_per_token": 0.000000225, - "output_cost_per_token": 0.00000225, + "input_cost_per_token": 2.25e-07, + "output_cost_per_token": 2.25e-06, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/meta-llama/llama-3-70b-instruct:nitro": { "max_tokens": 8192, - "input_cost_per_token": 0.0000009, - "output_cost_per_token": 0.0000009, + "input_cost_per_token": 9e-07, + "output_cost_per_token": 9e-07, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/meta-llama/llama-3-70b-instruct": { "max_tokens": 8192, - "input_cost_per_token": 0.00000059, - "output_cost_per_token": 0.00000079, + "input_cost_per_token": 5.9e-07, + "output_cost_per_token": 7.9e-07, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/openai/gpt-4o": { - "max_tokens": 4096, + "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 4096, - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000015, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 1.5e-05, "litellm_provider": "openrouter", "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_image": 0.002312 }, "openrouter/openai/gpt-4o-2024-05-13": { - "max_tokens": 4096, + "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 4096, - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000015, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 1.5e-05, "litellm_provider": "openrouter", "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_image": 0.002312 }, "openrouter/openai/gpt-4-vision-preview": { - "max_tokens": 130000, - "input_cost_per_token": 0.00001, - "output_cost_per_token": 0.00003, - "input_cost_per_image": 0.01445, + "max_tokens": 128000, + "input_cost_per_token": 1e-05, + "output_cost_per_token": 3e-05, + "input_cost_per_image": 0.01445, "litellm_provider": "openrouter", "mode": "chat", "supports_function_calling": true, "supports_vision": true }, "openrouter/openai/gpt-3.5-turbo": { - "max_tokens": 4095, - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.000002, + "max_tokens": 16385, + "input_cost_per_token": 5e-07, + "output_cost_per_token": 1.5e-06, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/openai/gpt-3.5-turbo-16k": { - "max_tokens": 16383, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000004, + "max_tokens": 16385, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 4e-06, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/openai/gpt-4": { - "max_tokens": 8192, - "input_cost_per_token": 0.00003, - "output_cost_per_token": 0.00006, + "max_tokens": 8191, + "input_cost_per_token": 3e-05, + "output_cost_per_token": 6e-05, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/anthropic/claude-instant-v1": { "max_tokens": 100000, "max_output_tokens": 8191, - "input_cost_per_token": 0.00000163, - "output_cost_per_token": 0.00000551, + "input_cost_per_token": 1.63e-06, + "output_cost_per_token": 5.51e-06, "litellm_provider": "openrouter", "mode": "chat" }, "openrouter/anthropic/claude-2": { - "max_tokens": 100000, + "max_tokens": 200000, "max_output_tokens": 8191, - "input_cost_per_token": 0.00001102, - "output_cost_per_token": 0.00003268, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/anthropic/claude-3-opus": { - "max_tokens": 4096, + "max_tokens": 200000, "max_input_tokens": 200000, "max_output_tokens": 4096, - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.000075, + "input_cost_per_token": 1.5e-05, + "output_cost_per_token": 7.5e-05, "litellm_provider": "openrouter", "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 395 + "tool_use_system_prompt_tokens": 395, + "input_cost_per_image": 0.024 }, "openrouter/google/palm-2-chat-bison": { "max_tokens": 25804, - "input_cost_per_token": 0.0000005, - "output_cost_per_token": 0.0000005, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 5e-07, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/google/palm-2-codechat-bison": { "max_tokens": 20070, - "input_cost_per_token": 0.0000005, - "output_cost_per_token": 0.0000005, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 5e-07, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/meta-llama/llama-2-13b-chat": { "max_tokens": 4096, - "input_cost_per_token": 0.0000002, - "output_cost_per_token": 0.0000002, + "input_cost_per_token": 1.3e-07, + "output_cost_per_token": 1.3e-07, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/meta-llama/llama-2-70b-chat": { "max_tokens": 4096, - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.0000015, + "input_cost_per_token": 6.4e-07, + "output_cost_per_token": 8e-07, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/meta-llama/codellama-34b-instruct": { - "max_tokens": 8096, - "input_cost_per_token": 0.0000005, - "output_cost_per_token": 0.0000005, + "max_tokens": 8192, + "input_cost_per_token": 7.2e-07, + "output_cost_per_token": 7.2e-07, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/nousresearch/nous-hermes-llama2-13b": { "max_tokens": 4096, - "input_cost_per_token": 0.0000002, - "output_cost_per_token": 0.0000002, + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 1.8e-07, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/mancer/weaver": { "max_tokens": 8000, - "input_cost_per_token": 0.000005625, - "output_cost_per_token": 0.000005625, + "input_cost_per_token": 3.375e-06, + "output_cost_per_token": 3.375e-06, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/gryphe/mythomax-l2-13b": { - "max_tokens": 8192, - "input_cost_per_token": 0.000001875, - "output_cost_per_token": 0.000001875, + "max_tokens": 4096, + "input_cost_per_token": 1.3e-07, + "output_cost_per_token": 1.3e-07, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/jondurbin/airoboros-l2-70b-2.1": { "max_tokens": 4096, - "input_cost_per_token": 0.000013875, - "output_cost_per_token": 0.000013875, + "input_cost_per_token": 1.3875e-05, + "output_cost_per_token": 1.3875e-05, "litellm_provider": "openrouter", "mode": "chat" }, "openrouter/undi95/remm-slerp-l2-13b": { - "max_tokens": 6144, - "input_cost_per_token": 0.000001875, - "output_cost_per_token": 0.000001875, + "max_tokens": 4096, + "input_cost_per_token": 2.7e-07, + "output_cost_per_token": 2.7e-07, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/pygmalionai/mythalion-13b": { - "max_tokens": 4096, - "input_cost_per_token": 0.000001875, - "output_cost_per_token": 0.000001875, + "max_tokens": 8192, + "input_cost_per_token": 1.125e-06, + "output_cost_per_token": 1.125e-06, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/mistralai/mistral-7b-instruct": { - "max_tokens": 8192, - "input_cost_per_token": 0.00000013, - "output_cost_per_token": 0.00000013, + "max_tokens": 32768, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 7e-08, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "openrouter/mistralai/mistral-7b-instruct:free": { - "max_tokens": 8192, + "max_tokens": 32768, "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "litellm_provider": "openrouter", - "mode": "chat" + "mode": "chat", + "input_cost_per_image": 0.0 }, "j2-ultra": { "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.000015, + "input_cost_per_token": 1.5e-05, + "output_cost_per_token": 1.5e-05, "litellm_provider": "ai21", "mode": "completion" }, @@ -1967,8 +1996,8 @@ "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, - "input_cost_per_token": 0.00001, - "output_cost_per_token": 0.00001, + "input_cost_per_token": 1e-05, + "output_cost_per_token": 1e-05, "litellm_provider": "ai21", "mode": "completion" }, @@ -1976,8 +2005,8 @@ "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000003, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 3e-06, "litellm_provider": "ai21", "mode": "completion" }, @@ -1985,8 +2014,8 @@ "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, - "input_cost_per_token": 0.0000005, - "output_cost_per_token": 0.0000005, + "input_cost_per_token": 5e-07, + "output_cost_per_token": 5e-07, "litellm_provider": "nlp_cloud", "mode": "completion" }, @@ -1994,113 +2023,113 @@ "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, - "input_cost_per_token": 0.0000005, - "output_cost_per_token": 0.0000005, + "input_cost_per_token": 5e-07, + "output_cost_per_token": 5e-07, "litellm_provider": "nlp_cloud", "mode": "chat" }, "luminous-base": { - "max_tokens": 2048, - "input_cost_per_token": 0.00003, - "output_cost_per_token": 0.000033, + "max_tokens": 2048, + "input_cost_per_token": 3e-05, + "output_cost_per_token": 3.3e-05, "litellm_provider": "aleph_alpha", "mode": "completion" }, "luminous-base-control": { - "max_tokens": 2048, - "input_cost_per_token": 0.0000375, - "output_cost_per_token": 0.00004125, + "max_tokens": 2048, + "input_cost_per_token": 3.75e-05, + "output_cost_per_token": 4.125e-05, "litellm_provider": "aleph_alpha", "mode": "chat" }, "luminous-extended": { - "max_tokens": 2048, - "input_cost_per_token": 0.000045, - "output_cost_per_token": 0.0000495, + "max_tokens": 2048, + "input_cost_per_token": 4.5e-05, + "output_cost_per_token": 4.95e-05, "litellm_provider": "aleph_alpha", "mode": "completion" }, "luminous-extended-control": { - "max_tokens": 2048, - "input_cost_per_token": 0.00005625, - "output_cost_per_token": 0.000061875, + "max_tokens": 2048, + "input_cost_per_token": 5.625e-05, + "output_cost_per_token": 6.1875e-05, "litellm_provider": "aleph_alpha", "mode": "chat" }, "luminous-supreme": { - "max_tokens": 2048, + "max_tokens": 2048, "input_cost_per_token": 0.000175, "output_cost_per_token": 0.0001925, "litellm_provider": "aleph_alpha", "mode": "completion" }, "luminous-supreme-control": { - "max_tokens": 2048, + "max_tokens": 2048, "input_cost_per_token": 0.00021875, "output_cost_per_token": 0.000240625, "litellm_provider": "aleph_alpha", "mode": "chat" }, "ai21.j2-mid-v1": { - "max_tokens": 8191, - "max_input_tokens": 8191, - "max_output_tokens": 8191, - "input_cost_per_token": 0.0000125, - "output_cost_per_token": 0.0000125, + "max_tokens": 8191, + "max_input_tokens": 8191, + "max_output_tokens": 8191, + "input_cost_per_token": 1.25e-05, + "output_cost_per_token": 1.25e-05, "litellm_provider": "bedrock", "mode": "chat" }, "ai21.j2-ultra-v1": { - "max_tokens": 8191, - "max_input_tokens": 8191, - "max_output_tokens": 8191, - "input_cost_per_token": 0.0000188, - "output_cost_per_token": 0.0000188, + "max_tokens": 8191, + "max_input_tokens": 8191, + "max_output_tokens": 8191, + "input_cost_per_token": 1.88e-05, + "output_cost_per_token": 1.88e-05, "litellm_provider": "bedrock", "mode": "chat" }, "amazon.titan-text-lite-v1": { - "max_tokens": 4000, + "max_tokens": 4000, "max_input_tokens": 42000, - "max_output_tokens": 4000, - "input_cost_per_token": 0.0000003, - "output_cost_per_token": 0.0000004, + "max_output_tokens": 4000, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 4e-07, "litellm_provider": "bedrock", "mode": "chat" }, "amazon.titan-text-express-v1": { - "max_tokens": 8000, + "max_tokens": 8000, "max_input_tokens": 42000, - "max_output_tokens": 8000, - "input_cost_per_token": 0.0000013, - "output_cost_per_token": 0.0000017, + "max_output_tokens": 8000, + "input_cost_per_token": 1.3e-06, + "output_cost_per_token": 1.7e-06, "litellm_provider": "bedrock", "mode": "chat" }, "amazon.titan-embed-text-v1": { - "max_tokens": 8192, - "max_input_tokens": 8192, + "max_tokens": 8192, + "max_input_tokens": 8192, "output_vector_size": 1536, - "input_cost_per_token": 0.0000001, + "input_cost_per_token": 1e-07, "output_cost_per_token": 0.0, - "litellm_provider": "bedrock", + "litellm_provider": "bedrock", "mode": "embedding" }, "amazon.titan-embed-text-v2:0": { - "max_tokens": 8192, - "max_input_tokens": 8192, + "max_tokens": 8192, + "max_input_tokens": 8192, "output_vector_size": 1024, - "input_cost_per_token": 0.0000002, + "input_cost_per_token": 2e-07, "output_cost_per_token": 0.0, - "litellm_provider": "bedrock", + "litellm_provider": "bedrock", "mode": "embedding" }, "mistral.mistral-7b-instruct-v0:2": { "max_tokens": 8191, "max_input_tokens": 32000, "max_output_tokens": 8191, - "input_cost_per_token": 0.00000015, - "output_cost_per_token": 0.0000002, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 2e-07, "litellm_provider": "bedrock", "mode": "chat" }, @@ -2108,8 +2137,8 @@ "max_tokens": 8191, "max_input_tokens": 32000, "max_output_tokens": 8191, - "input_cost_per_token": 0.00000045, - "output_cost_per_token": 0.0000007, + "input_cost_per_token": 4.5e-07, + "output_cost_per_token": 7e-07, "litellm_provider": "bedrock", "mode": "chat" }, @@ -2117,8 +2146,8 @@ "max_tokens": 8191, "max_input_tokens": 32000, "max_output_tokens": 8191, - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, "litellm_provider": "bedrock", "mode": "chat" }, @@ -2126,8 +2155,8 @@ "max_tokens": 8191, "max_input_tokens": 32000, "max_output_tokens": 8191, - "input_cost_per_token": 0.00000045, - "output_cost_per_token": 0.0000007, + "input_cost_per_token": 4.5e-07, + "output_cost_per_token": 7e-07, "litellm_provider": "bedrock", "mode": "chat" }, @@ -2135,8 +2164,8 @@ "max_tokens": 8191, "max_input_tokens": 32000, "max_output_tokens": 8191, - "input_cost_per_token": 0.00000045, - "output_cost_per_token": 0.0000007, + "input_cost_per_token": 4.5e-07, + "output_cost_per_token": 7e-07, "litellm_provider": "bedrock", "mode": "chat" }, @@ -2144,8 +2173,8 @@ "max_tokens": 8191, "max_input_tokens": 32000, "max_output_tokens": 8191, - "input_cost_per_token": 0.00000059, - "output_cost_per_token": 0.00000091, + "input_cost_per_token": 5.9e-07, + "output_cost_per_token": 9.1e-07, "litellm_provider": "bedrock", "mode": "chat" }, @@ -2153,8 +2182,8 @@ "max_tokens": 8191, "max_input_tokens": 32000, "max_output_tokens": 8191, - "input_cost_per_token": 0.00000015, - "output_cost_per_token": 0.0000002, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 2e-07, "litellm_provider": "bedrock", "mode": "chat" }, @@ -2162,8 +2191,8 @@ "max_tokens": 8191, "max_input_tokens": 32000, "max_output_tokens": 8191, - "input_cost_per_token": 0.00000015, - "output_cost_per_token": 0.0000002, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 2e-07, "litellm_provider": "bedrock", "mode": "chat" }, @@ -2171,8 +2200,8 @@ "max_tokens": 8191, "max_input_tokens": 32000, "max_output_tokens": 8191, - "input_cost_per_token": 0.0000002, - "output_cost_per_token": 0.00000026, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 2.6e-07, "litellm_provider": "bedrock", "mode": "chat" }, @@ -2180,8 +2209,8 @@ "max_tokens": 8191, "max_input_tokens": 32000, "max_output_tokens": 8191, - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, "litellm_provider": "bedrock", "mode": "chat" }, @@ -2189,8 +2218,8 @@ "max_tokens": 8191, "max_input_tokens": 32000, "max_output_tokens": 8191, - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, "litellm_provider": "bedrock", "mode": "chat" }, @@ -2198,28 +2227,28 @@ "max_tokens": 8191, "max_input_tokens": 32000, "max_output_tokens": 8191, - "input_cost_per_token": 0.0000104, - "output_cost_per_token": 0.0000312, + "input_cost_per_token": 1.04e-05, + "output_cost_per_token": 3.12e-05, "litellm_provider": "bedrock", "mode": "chat" }, "anthropic.claude-3-sonnet-20240229-v1:0": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 200000, "max_output_tokens": 4096, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, "litellm_provider": "bedrock", "mode": "chat", "supports_function_calling": true, "supports_vision": true }, "anthropic.claude-3-haiku-20240307-v1:0": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 200000, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000025, - "output_cost_per_token": 0.00000125, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 1.25e-06, "litellm_provider": "bedrock", "mode": "chat", "supports_function_calling": true, @@ -2229,51 +2258,51 @@ "max_tokens": 4096, "max_input_tokens": 200000, "max_output_tokens": 4096, - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.000075, + "input_cost_per_token": 1.5e-05, + "output_cost_per_token": 7.5e-05, "litellm_provider": "bedrock", "mode": "chat", "supports_function_calling": true, "supports_vision": true }, "anthropic.claude-v1": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, "max_output_tokens": 8191, - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/us-east-1/anthropic.claude-v1": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, "max_output_tokens": 8191, - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/us-west-2/anthropic.claude-v1": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, "max_output_tokens": 8191, - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/ap-northeast-1/anthropic.claude-v1": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, "max_output_tokens": 8191, - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/ap-northeast-1/1-month-commitment/anthropic.claude-v1": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.0455, @@ -2282,7 +2311,7 @@ "mode": "chat" }, "bedrock/ap-northeast-1/6-month-commitment/anthropic.claude-v1": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.02527, @@ -2291,224 +2320,224 @@ "mode": "chat" }, "bedrock/eu-central-1/anthropic.claude-v1": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, + "max_output_tokens": 8191, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/eu-central-1/1-month-commitment/anthropic.claude-v1": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_second": 0.0415, "output_cost_per_second": 0.0415, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/eu-central-1/6-month-commitment/anthropic.claude-v1": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_second": 0.02305, "output_cost_per_second": 0.02305, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/us-east-1/1-month-commitment/anthropic.claude-v1": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_second": 0.0175, "output_cost_per_second": 0.0175, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/us-east-1/6-month-commitment/anthropic.claude-v1": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_second": 0.00972, "output_cost_per_second": 0.00972, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/us-west-2/1-month-commitment/anthropic.claude-v1": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_second": 0.0175, "output_cost_per_second": 0.0175, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/us-west-2/6-month-commitment/anthropic.claude-v1": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_second": 0.00972, "output_cost_per_second": 0.00972, "litellm_provider": "bedrock", "mode": "chat" }, "anthropic.claude-v2": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, + "max_output_tokens": 8191, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/us-east-1/anthropic.claude-v2": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, + "max_output_tokens": 8191, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/us-west-2/anthropic.claude-v2": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, + "max_output_tokens": 8191, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/ap-northeast-1/anthropic.claude-v2": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, + "max_output_tokens": 8191, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/ap-northeast-1/1-month-commitment/anthropic.claude-v2": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_second": 0.0455, "output_cost_per_second": 0.0455, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/ap-northeast-1/6-month-commitment/anthropic.claude-v2": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_second": 0.02527, "output_cost_per_second": 0.02527, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/eu-central-1/anthropic.claude-v2": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, + "max_output_tokens": 8191, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/eu-central-1/1-month-commitment/anthropic.claude-v2": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_second": 0.0415, "output_cost_per_second": 0.0415, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/eu-central-1/6-month-commitment/anthropic.claude-v2": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_second": 0.02305, "output_cost_per_second": 0.02305, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/us-east-1/1-month-commitment/anthropic.claude-v2": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_second": 0.0175, "output_cost_per_second": 0.0175, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/us-east-1/6-month-commitment/anthropic.claude-v2": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_second": 0.00972, "output_cost_per_second": 0.00972, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/us-west-2/1-month-commitment/anthropic.claude-v2": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_second": 0.0175, "output_cost_per_second": 0.0175, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/us-west-2/6-month-commitment/anthropic.claude-v2": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_second": 0.00972, "output_cost_per_second": 0.00972, "litellm_provider": "bedrock", "mode": "chat" }, "anthropic.claude-v2:1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/us-east-1/anthropic.claude-v2:1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/us-west-2/anthropic.claude-v2:1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/ap-northeast-1/anthropic.claude-v2:1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/ap-northeast-1/1-month-commitment/anthropic.claude-v2:1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.0455, "output_cost_per_second": 0.0455, @@ -2516,8 +2545,8 @@ "mode": "chat" }, "bedrock/ap-northeast-1/6-month-commitment/anthropic.claude-v2:1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.02527, "output_cost_per_second": 0.02527, @@ -2525,17 +2554,17 @@ "mode": "chat" }, "bedrock/eu-central-1/anthropic.claude-v2:1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/eu-central-1/1-month-commitment/anthropic.claude-v2:1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.0415, "output_cost_per_second": 0.0415, @@ -2543,8 +2572,8 @@ "mode": "chat" }, "bedrock/eu-central-1/6-month-commitment/anthropic.claude-v2:1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.02305, "output_cost_per_second": 0.02305, @@ -2552,8 +2581,8 @@ "mode": "chat" }, "bedrock/us-east-1/1-month-commitment/anthropic.claude-v2:1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.0175, "output_cost_per_second": 0.0175, @@ -2561,8 +2590,8 @@ "mode": "chat" }, "bedrock/us-east-1/6-month-commitment/anthropic.claude-v2:1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.00972, "output_cost_per_second": 0.00972, @@ -2570,8 +2599,8 @@ "mode": "chat" }, "bedrock/us-west-2/1-month-commitment/anthropic.claude-v2:1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.0175, "output_cost_per_second": 0.0175, @@ -2579,8 +2608,8 @@ "mode": "chat" }, "bedrock/us-west-2/6-month-commitment/anthropic.claude-v2:1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.00972, "output_cost_per_second": 0.00972, @@ -2588,26 +2617,26 @@ "mode": "chat" }, "anthropic.claude-instant-v1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, - "input_cost_per_token": 0.00000163, - "output_cost_per_token": 0.00000551, + "input_cost_per_token": 1.63e-06, + "output_cost_per_token": 5.51e-06, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/us-east-1/anthropic.claude-instant-v1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, - "input_cost_per_token": 0.0000008, - "output_cost_per_token": 0.0000024, + "input_cost_per_token": 8e-07, + "output_cost_per_token": 2.4e-06, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/us-east-1/1-month-commitment/anthropic.claude-instant-v1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.011, "output_cost_per_second": 0.011, @@ -2615,8 +2644,8 @@ "mode": "chat" }, "bedrock/us-east-1/6-month-commitment/anthropic.claude-instant-v1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.00611, "output_cost_per_second": 0.00611, @@ -2624,8 +2653,8 @@ "mode": "chat" }, "bedrock/us-west-2/1-month-commitment/anthropic.claude-instant-v1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.011, "output_cost_per_second": 0.011, @@ -2633,8 +2662,8 @@ "mode": "chat" }, "bedrock/us-west-2/6-month-commitment/anthropic.claude-instant-v1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.00611, "output_cost_per_second": 0.00611, @@ -2642,26 +2671,26 @@ "mode": "chat" }, "bedrock/us-west-2/anthropic.claude-instant-v1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, - "input_cost_per_token": 0.0000008, - "output_cost_per_token": 0.0000024, + "input_cost_per_token": 8e-07, + "output_cost_per_token": 2.4e-06, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/ap-northeast-1/anthropic.claude-instant-v1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, - "input_cost_per_token": 0.00000223, - "output_cost_per_token": 0.00000755, + "input_cost_per_token": 2.23e-06, + "output_cost_per_token": 7.55e-06, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/ap-northeast-1/1-month-commitment/anthropic.claude-instant-v1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.01475, "output_cost_per_second": 0.01475, @@ -2669,8 +2698,8 @@ "mode": "chat" }, "bedrock/ap-northeast-1/6-month-commitment/anthropic.claude-instant-v1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.008194, "output_cost_per_second": 0.008194, @@ -2678,17 +2707,17 @@ "mode": "chat" }, "bedrock/eu-central-1/anthropic.claude-instant-v1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, - "input_cost_per_token": 0.00000248, - "output_cost_per_token": 0.00000838, + "input_cost_per_token": 2.48e-06, + "output_cost_per_token": 8.38e-06, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/eu-central-1/1-month-commitment/anthropic.claude-instant-v1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.01635, "output_cost_per_second": 0.01635, @@ -2696,8 +2725,8 @@ "mode": "chat" }, "bedrock/eu-central-1/6-month-commitment/anthropic.claude-instant-v1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.009083, "output_cost_per_second": 0.009083, @@ -2705,254 +2734,254 @@ "mode": "chat" }, "cohere.command-text-v14": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 4096, - "max_output_tokens": 4096, - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.0000020, + "max_output_tokens": 4096, + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 2e-06, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/*/1-month-commitment/cohere.command-text-v14": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 4096, - "max_output_tokens": 4096, + "max_output_tokens": 4096, "input_cost_per_second": 0.011, "output_cost_per_second": 0.011, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/*/6-month-commitment/cohere.command-text-v14": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 4096, - "max_output_tokens": 4096, + "max_output_tokens": 4096, "input_cost_per_second": 0.0066027, "output_cost_per_second": 0.0066027, "litellm_provider": "bedrock", "mode": "chat" }, "cohere.command-light-text-v14": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 4096, - "max_output_tokens": 4096, - "input_cost_per_token": 0.0000003, - "output_cost_per_token": 0.0000006, + "max_output_tokens": 4096, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 6e-07, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/*/1-month-commitment/cohere.command-light-text-v14": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 4096, - "max_output_tokens": 4096, + "max_output_tokens": 4096, "input_cost_per_second": 0.001902, "output_cost_per_second": 0.001902, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/*/6-month-commitment/cohere.command-light-text-v14": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 4096, - "max_output_tokens": 4096, + "max_output_tokens": 4096, "input_cost_per_second": 0.0011416, "output_cost_per_second": 0.0011416, "litellm_provider": "bedrock", "mode": "chat" }, "cohere.command-r-plus-v1:0": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 128000, "max_output_tokens": 4096, - "input_cost_per_token": 0.0000030, - "output_cost_per_token": 0.000015, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, "litellm_provider": "bedrock", "mode": "chat" }, "cohere.command-r-v1:0": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 128000, "max_output_tokens": 4096, - "input_cost_per_token": 0.0000005, - "output_cost_per_token": 0.0000015, + "input_cost_per_token": 5e-07, + "output_cost_per_token": 1.5e-06, "litellm_provider": "bedrock", "mode": "chat" }, "cohere.embed-english-v3": { - "max_tokens": 512, - "max_input_tokens": 512, - "input_cost_per_token": 0.0000001, - "output_cost_per_token": 0.000000, + "max_tokens": 512, + "max_input_tokens": 512, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 0.0, "litellm_provider": "bedrock", "mode": "embedding" }, "cohere.embed-multilingual-v3": { - "max_tokens": 512, - "max_input_tokens": 512, - "input_cost_per_token": 0.0000001, - "output_cost_per_token": 0.000000, + "max_tokens": 512, + "max_input_tokens": 512, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 0.0, "litellm_provider": "bedrock", "mode": "embedding" }, "meta.llama2-13b-chat-v1": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "input_cost_per_token": 0.00000075, - "output_cost_per_token": 0.000001, + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 7.5e-07, + "output_cost_per_token": 1e-06, "litellm_provider": "bedrock", "mode": "chat" }, "meta.llama2-70b-chat-v1": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "input_cost_per_token": 0.00000195, - "output_cost_per_token": 0.00000256, + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 1.95e-06, + "output_cost_per_token": 2.56e-06, "litellm_provider": "bedrock", "mode": "chat" }, "meta.llama3-8b-instruct-v1:0": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "input_cost_per_token": 0.0000004, - "output_cost_per_token": 0.0000006, + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 6e-07, "litellm_provider": "bedrock", "mode": "chat" }, "meta.llama3-70b-instruct-v1:0": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "input_cost_per_token": 0.00000265, - "output_cost_per_token": 0.0000035, + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 2.65e-06, + "output_cost_per_token": 3.5e-06, "litellm_provider": "bedrock", "mode": "chat" }, "512-x-512/50-steps/stability.stable-diffusion-xl-v0": { - "max_tokens": 77, - "max_input_tokens": 77, + "max_tokens": 77, + "max_input_tokens": 77, "output_cost_per_image": 0.018, "litellm_provider": "bedrock", "mode": "image_generation" }, "512-x-512/max-steps/stability.stable-diffusion-xl-v0": { - "max_tokens": 77, - "max_input_tokens": 77, + "max_tokens": 77, + "max_input_tokens": 77, "output_cost_per_image": 0.036, "litellm_provider": "bedrock", "mode": "image_generation" }, "max-x-max/50-steps/stability.stable-diffusion-xl-v0": { - "max_tokens": 77, - "max_input_tokens": 77, + "max_tokens": 77, + "max_input_tokens": 77, "output_cost_per_image": 0.036, "litellm_provider": "bedrock", "mode": "image_generation" }, "max-x-max/max-steps/stability.stable-diffusion-xl-v0": { - "max_tokens": 77, - "max_input_tokens": 77, + "max_tokens": 77, + "max_input_tokens": 77, "output_cost_per_image": 0.072, "litellm_provider": "bedrock", "mode": "image_generation" }, "1024-x-1024/50-steps/stability.stable-diffusion-xl-v1": { - "max_tokens": 77, - "max_input_tokens": 77, + "max_tokens": 77, + "max_input_tokens": 77, "output_cost_per_image": 0.04, "litellm_provider": "bedrock", "mode": "image_generation" }, "1024-x-1024/max-steps/stability.stable-diffusion-xl-v1": { - "max_tokens": 77, - "max_input_tokens": 77, + "max_tokens": 77, + "max_input_tokens": 77, "output_cost_per_image": 0.08, "litellm_provider": "bedrock", "mode": "image_generation" }, "sagemaker/meta-textgeneration-llama-2-7b": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "input_cost_per_token": 0.000, - "output_cost_per_token": 0.000, + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, "litellm_provider": "sagemaker", "mode": "completion" }, "sagemaker/meta-textgeneration-llama-2-7b-f": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "input_cost_per_token": 0.000, - "output_cost_per_token": 0.000, + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, "litellm_provider": "sagemaker", "mode": "chat" }, "sagemaker/meta-textgeneration-llama-2-13b": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "input_cost_per_token": 0.000, - "output_cost_per_token": 0.000, + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, "litellm_provider": "sagemaker", "mode": "completion" }, "sagemaker/meta-textgeneration-llama-2-13b-f": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "input_cost_per_token": 0.000, - "output_cost_per_token": 0.000, + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, "litellm_provider": "sagemaker", "mode": "chat" }, "sagemaker/meta-textgeneration-llama-2-70b": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "input_cost_per_token": 0.000, - "output_cost_per_token": 0.000, + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, "litellm_provider": "sagemaker", "mode": "completion" }, "sagemaker/meta-textgeneration-llama-2-70b-b-f": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "input_cost_per_token": 0.000, - "output_cost_per_token": 0.000, + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, "litellm_provider": "sagemaker", "mode": "chat" }, "together-ai-up-to-3b": { - "input_cost_per_token": 0.0000001, - "output_cost_per_token": 0.0000001, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 1e-07, "litellm_provider": "together_ai" }, "together-ai-3.1b-7b": { - "input_cost_per_token": 0.0000002, - "output_cost_per_token": 0.0000002, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 2e-07, "litellm_provider": "together_ai" }, "together-ai-7.1b-20b": { "max_tokens": 1000, - "input_cost_per_token": 0.0000004, - "output_cost_per_token": 0.0000004, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 4e-07, "litellm_provider": "together_ai" }, "together-ai-20.1b-40b": { - "input_cost_per_token": 0.0000008, - "output_cost_per_token": 0.0000008, + "input_cost_per_token": 8e-07, + "output_cost_per_token": 8e-07, "litellm_provider": "together_ai" }, "together-ai-40.1b-70b": { - "input_cost_per_token": 0.0000009, - "output_cost_per_token": 0.0000009, + "input_cost_per_token": 9e-07, + "output_cost_per_token": 9e-07, "litellm_provider": "together_ai" }, "together_ai/mistralai/Mixtral-8x7B-Instruct-v0.1": { - "input_cost_per_token": 0.0000006, - "output_cost_per_token": 0.0000006, + "input_cost_per_token": 6e-07, + "output_cost_per_token": 6e-07, "litellm_provider": "together_ai", "supports_function_calling": true, "supports_parallel_function_calling": true @@ -2968,36 +2997,36 @@ "supports_parallel_function_calling": true }, "ollama/llama2": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "litellm_provider": "ollama", "mode": "completion" }, "ollama/llama2:13b": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "litellm_provider": "ollama", "mode": "completion" }, "ollama/llama2:70b": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "litellm_provider": "ollama", "mode": "completion" }, "ollama/llama2-uncensored": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "litellm_provider": "ollama", @@ -3067,8 +3096,8 @@ "mode": "chat" }, "ollama/codellama": { - "max_tokens": 4096, - "max_input_tokens": 4096, + "max_tokens": 4096, + "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, @@ -3076,8 +3105,8 @@ "mode": "completion" }, "ollama/orca-mini": { - "max_tokens": 4096, - "max_input_tokens": 4096, + "max_tokens": 4096, + "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, @@ -3097,8 +3126,8 @@ "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000070, - "output_cost_per_token": 0.00000090, + "input_cost_per_token": 7e-07, + "output_cost_per_token": 9e-07, "litellm_provider": "deepinfra", "mode": "chat" }, @@ -3106,8 +3135,8 @@ "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000022, - "output_cost_per_token": 0.00000022, + "input_cost_per_token": 2.2e-07, + "output_cost_per_token": 2.2e-07, "litellm_provider": "deepinfra", "mode": "chat" }, @@ -3115,8 +3144,8 @@ "max_tokens": 8191, "max_input_tokens": 32768, "max_output_tokens": 8191, - "input_cost_per_token": 0.00000013, - "output_cost_per_token": 0.00000013, + "input_cost_per_token": 1.3e-07, + "output_cost_per_token": 1.3e-07, "litellm_provider": "deepinfra", "mode": "chat" }, @@ -3124,8 +3153,8 @@ "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000070, - "output_cost_per_token": 0.00000090, + "input_cost_per_token": 7e-07, + "output_cost_per_token": 9e-07, "litellm_provider": "deepinfra", "mode": "chat" }, @@ -3133,8 +3162,8 @@ "max_tokens": 8191, "max_input_tokens": 32768, "max_output_tokens": 8191, - "input_cost_per_token": 0.00000027, - "output_cost_per_token": 0.00000027, + "input_cost_per_token": 2.7e-07, + "output_cost_per_token": 2.7e-07, "litellm_provider": "deepinfra", "mode": "chat" }, @@ -3142,8 +3171,8 @@ "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000060, - "output_cost_per_token": 0.00000060, + "input_cost_per_token": 6e-07, + "output_cost_per_token": 6e-07, "litellm_provider": "deepinfra", "mode": "chat" }, @@ -3151,8 +3180,8 @@ "max_tokens": 4096, "max_input_tokens": 32000, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000027, - "output_cost_per_token": 0.00000027, + "input_cost_per_token": 2.7e-07, + "output_cost_per_token": 2.7e-07, "litellm_provider": "deepinfra", "mode": "completion" }, @@ -3160,8 +3189,8 @@ "max_tokens": 4096, "max_input_tokens": 16384, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000060, - "output_cost_per_token": 0.00000060, + "input_cost_per_token": 6e-07, + "output_cost_per_token": 6e-07, "litellm_provider": "deepinfra", "mode": "chat" }, @@ -3169,8 +3198,8 @@ "max_tokens": 8191, "max_input_tokens": 32768, "max_output_tokens": 8191, - "input_cost_per_token": 0.00000027, - "output_cost_per_token": 0.00000027, + "input_cost_per_token": 2.7e-07, + "output_cost_per_token": 2.7e-07, "litellm_provider": "deepinfra", "mode": "chat" }, @@ -3178,8 +3207,8 @@ "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000070, - "output_cost_per_token": 0.00000090, + "input_cost_per_token": 7e-07, + "output_cost_per_token": 9e-07, "litellm_provider": "deepinfra", "mode": "chat" }, @@ -3187,8 +3216,8 @@ "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000060, - "output_cost_per_token": 0.00000060, + "input_cost_per_token": 6e-07, + "output_cost_per_token": 6e-07, "litellm_provider": "deepinfra", "mode": "chat" }, @@ -3196,8 +3225,8 @@ "max_tokens": 4096, "max_input_tokens": 200000, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000013, - "output_cost_per_token": 0.00000013, + "input_cost_per_token": 1.3e-07, + "output_cost_per_token": 1.3e-07, "litellm_provider": "deepinfra", "mode": "completion" }, @@ -3205,8 +3234,8 @@ "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000070, - "output_cost_per_token": 0.00000090, + "input_cost_per_token": 7e-07, + "output_cost_per_token": 9e-07, "litellm_provider": "deepinfra", "mode": "chat" }, @@ -3214,8 +3243,8 @@ "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000022, - "output_cost_per_token": 0.00000022, + "input_cost_per_token": 2.2e-07, + "output_cost_per_token": 2.2e-07, "litellm_provider": "deepinfra", "mode": "chat" }, @@ -3223,8 +3252,8 @@ "max_tokens": 8191, "max_input_tokens": 32768, "max_output_tokens": 8191, - "input_cost_per_token": 0.00000020, - "output_cost_per_token": 0.00000020, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 2e-07, "litellm_provider": "deepinfra", "mode": "chat" }, @@ -3232,8 +3261,8 @@ "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000013, - "output_cost_per_token": 0.00000013, + "input_cost_per_token": 1.3e-07, + "output_cost_per_token": 1.3e-07, "litellm_provider": "deepinfra", "mode": "chat" }, @@ -3241,8 +3270,8 @@ "max_tokens": 4096, "max_input_tokens": 200000, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000060, - "output_cost_per_token": 0.00000060, + "input_cost_per_token": 6e-07, + "output_cost_per_token": 6e-07, "litellm_provider": "deepinfra", "mode": "completion" }, @@ -3250,91 +3279,91 @@ "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000013, - "output_cost_per_token": 0.00000013, + "input_cost_per_token": 1.3e-07, + "output_cost_per_token": 1.3e-07, "litellm_provider": "deepinfra", "mode": "chat" }, - "perplexity/codellama-34b-instruct": { + "perplexity/codellama-34b-instruct": { "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, - "input_cost_per_token": 0.00000035, - "output_cost_per_token": 0.00000140, - "litellm_provider": "perplexity", - "mode": "chat" + "input_cost_per_token": 3.5e-07, + "output_cost_per_token": 1.4e-06, + "litellm_provider": "perplexity", + "mode": "chat" }, - "perplexity/codellama-70b-instruct": { + "perplexity/codellama-70b-instruct": { "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, - "input_cost_per_token": 0.00000070, - "output_cost_per_token": 0.00000280, - "litellm_provider": "perplexity", - "mode": "chat" + "input_cost_per_token": 7e-07, + "output_cost_per_token": 2.8e-06, + "litellm_provider": "perplexity", + "mode": "chat" }, - "perplexity/pplx-7b-chat": { + "perplexity/pplx-7b-chat": { "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, - "input_cost_per_token": 0.00000007, - "output_cost_per_token": 0.00000028, - "litellm_provider": "perplexity", - "mode": "chat" + "input_cost_per_token": 7e-08, + "output_cost_per_token": 2.8e-07, + "litellm_provider": "perplexity", + "mode": "chat" }, - "perplexity/pplx-70b-chat": { + "perplexity/pplx-70b-chat": { "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000070, - "output_cost_per_token": 0.00000280, - "litellm_provider": "perplexity", - "mode": "chat" + "input_cost_per_token": 7e-07, + "output_cost_per_token": 2.8e-06, + "litellm_provider": "perplexity", + "mode": "chat" }, - "perplexity/pplx-7b-online": { + "perplexity/pplx-7b-online": { "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.0000000, - "output_cost_per_token": 0.00000028, + "input_cost_per_token": 0.0, + "output_cost_per_token": 2.8e-07, "input_cost_per_request": 0.005, - "litellm_provider": "perplexity", - "mode": "chat" + "litellm_provider": "perplexity", + "mode": "chat" }, - "perplexity/pplx-70b-online": { + "perplexity/pplx-70b-online": { "max_tokens": 4096, "max_input_tokens": 4096, - "max_output_tokens": 4096, - "input_cost_per_token": 0.0000000, - "output_cost_per_token": 0.00000280, + "max_output_tokens": 4096, + "input_cost_per_token": 0.0, + "output_cost_per_token": 2.8e-06, "input_cost_per_request": 0.005, - "litellm_provider": "perplexity", - "mode": "chat" + "litellm_provider": "perplexity", + "mode": "chat" }, - "perplexity/llama-2-70b-chat": { + "perplexity/llama-2-70b-chat": { "max_tokens": 4096, "max_input_tokens": 4096, - "max_output_tokens": 4096, - "input_cost_per_token": 0.00000070, - "output_cost_per_token": 0.00000280, - "litellm_provider": "perplexity", - "mode": "chat" + "max_output_tokens": 4096, + "input_cost_per_token": 7e-07, + "output_cost_per_token": 2.8e-06, + "litellm_provider": "perplexity", + "mode": "chat" }, - "perplexity/mistral-7b-instruct": { + "perplexity/mistral-7b-instruct": { "max_tokens": 4096, "max_input_tokens": 4096, - "max_output_tokens": 4096, - "input_cost_per_token": 0.00000007, - "output_cost_per_token": 0.00000028, - "litellm_provider": "perplexity", - "mode": "chat" + "max_output_tokens": 4096, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 2.8e-07, + "litellm_provider": "perplexity", + "mode": "chat" }, "perplexity/mixtral-8x7b-instruct": { "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000007, - "output_cost_per_token": 0.00000028, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 2.8e-07, "litellm_provider": "perplexity", "mode": "chat" }, @@ -3342,8 +3371,8 @@ "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, - "input_cost_per_token": 0.00000007, - "output_cost_per_token": 0.00000028, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 2.8e-07, "litellm_provider": "perplexity", "mode": "chat" }, @@ -3352,7 +3381,7 @@ "max_input_tokens": 12000, "max_output_tokens": 12000, "input_cost_per_token": 0, - "output_cost_per_token": 0.00000028, + "output_cost_per_token": 2.8e-07, "input_cost_per_request": 0.005, "litellm_provider": "perplexity", "mode": "chat" @@ -3361,8 +3390,8 @@ "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, - "input_cost_per_token": 0.0000006, - "output_cost_per_token": 0.0000018, + "input_cost_per_token": 6e-07, + "output_cost_per_token": 1.8e-06, "litellm_provider": "perplexity", "mode": "chat" }, @@ -3371,167 +3400,1118 @@ "max_input_tokens": 12000, "max_output_tokens": 12000, "input_cost_per_token": 0, - "output_cost_per_token": 0.0000018, + "output_cost_per_token": 1.8e-06, "input_cost_per_request": 0.005, "litellm_provider": "perplexity", "mode": "chat" }, - "anyscale/mistralai/Mistral-7B-Instruct-v0.1": { + "anyscale/mistralai/Mistral-7B-Instruct-v0.1": { "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, - "input_cost_per_token": 0.00000015, - "output_cost_per_token": 0.00000015, - "litellm_provider": "anyscale", + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 1.5e-07, + "litellm_provider": "anyscale", "mode": "chat", "supports_function_calling": true - }, - "anyscale/Mixtral-8x7B-Instruct-v0.1": { + }, + "anyscale/Mixtral-8x7B-Instruct-v0.1": { "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, - "input_cost_per_token": 0.00000015, - "output_cost_per_token": 0.00000015, - "litellm_provider": "anyscale", + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 1.5e-07, + "litellm_provider": "anyscale", "mode": "chat", "supports_function_calling": true - }, - "anyscale/HuggingFaceH4/zephyr-7b-beta": { + }, + "anyscale/HuggingFaceH4/zephyr-7b-beta": { "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, - "input_cost_per_token": 0.00000015, - "output_cost_per_token": 0.00000015, - "litellm_provider": "anyscale", + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 1.5e-07, + "litellm_provider": "anyscale", "mode": "chat" - }, - "anyscale/meta-llama/Llama-2-7b-chat-hf": { + }, + "anyscale/meta-llama/Llama-2-7b-chat-hf": { "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000015, - "output_cost_per_token": 0.00000015, - "litellm_provider": "anyscale", + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 1.5e-07, + "litellm_provider": "anyscale", "mode": "chat" - }, - "anyscale/meta-llama/Llama-2-13b-chat-hf": { + }, + "anyscale/meta-llama/Llama-2-13b-chat-hf": { "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000025, - "output_cost_per_token": 0.00000025, - "litellm_provider": "anyscale", + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 2.5e-07, + "litellm_provider": "anyscale", "mode": "chat" - }, - "anyscale/meta-llama/Llama-2-70b-chat-hf": { + }, + "anyscale/meta-llama/Llama-2-70b-chat-hf": { "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000001, - "litellm_provider": "anyscale", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 1e-06, + "litellm_provider": "anyscale", "mode": "chat" - }, - "anyscale/codellama/CodeLlama-34b-Instruct-hf": { + }, + "anyscale/codellama/CodeLlama-34b-Instruct-hf": { "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000001, - "litellm_provider": "anyscale", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 1e-06, + "litellm_provider": "anyscale", "mode": "chat" - }, - "cloudflare/@cf/meta/llama-2-7b-chat-fp16": { - "max_tokens": 3072, - "max_input_tokens": 3072, - "max_output_tokens": 3072, - "input_cost_per_token": 0.000001923, - "output_cost_per_token": 0.000001923, - "litellm_provider": "cloudflare", + }, + "cloudflare/@cf/meta/llama-2-7b-chat-fp16": { + "max_tokens": 3072, + "max_input_tokens": 3072, + "max_output_tokens": 3072, + "input_cost_per_token": 1.923e-06, + "output_cost_per_token": 1.923e-06, + "litellm_provider": "cloudflare", "mode": "chat" - }, - "cloudflare/@cf/meta/llama-2-7b-chat-int8": { - "max_tokens": 2048, - "max_input_tokens": 2048, - "max_output_tokens": 2048, - "input_cost_per_token": 0.000001923, - "output_cost_per_token": 0.000001923, - "litellm_provider": "cloudflare", + }, + "cloudflare/@cf/meta/llama-2-7b-chat-int8": { + "max_tokens": 2048, + "max_input_tokens": 2048, + "max_output_tokens": 2048, + "input_cost_per_token": 1.923e-06, + "output_cost_per_token": 1.923e-06, + "litellm_provider": "cloudflare", "mode": "chat" - }, - "cloudflare/@cf/mistral/mistral-7b-instruct-v0.1": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "input_cost_per_token": 0.000001923, - "output_cost_per_token": 0.000001923, - "litellm_provider": "cloudflare", + }, + "cloudflare/@cf/mistral/mistral-7b-instruct-v0.1": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 1.923e-06, + "output_cost_per_token": 1.923e-06, + "litellm_provider": "cloudflare", "mode": "chat" - }, - "cloudflare/@hf/thebloke/codellama-7b-instruct-awq": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "input_cost_per_token": 0.000001923, - "output_cost_per_token": 0.000001923, - "litellm_provider": "cloudflare", - "mode": "chat" - }, - "voyage/voyage-01": { + }, + "cloudflare/@hf/thebloke/codellama-7b-instruct-awq": { "max_tokens": 4096, "max_input_tokens": 4096, - "input_cost_per_token": 0.0000001, - "output_cost_per_token": 0.000000, + "max_output_tokens": 4096, + "input_cost_per_token": 1.923e-06, + "output_cost_per_token": 1.923e-06, + "litellm_provider": "cloudflare", + "mode": "chat" + }, + "voyage/voyage-01": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 0.0, "litellm_provider": "voyage", "mode": "embedding" }, "voyage/voyage-lite-01": { "max_tokens": 4096, "max_input_tokens": 4096, - "input_cost_per_token": 0.0000001, - "output_cost_per_token": 0.000000, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 0.0, "litellm_provider": "voyage", "mode": "embedding" }, "voyage/voyage-large-2": { "max_tokens": 16000, "max_input_tokens": 16000, - "input_cost_per_token": 0.00000012, - "output_cost_per_token": 0.000000, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 0.0, "litellm_provider": "voyage", "mode": "embedding" }, "voyage/voyage-law-2": { "max_tokens": 16000, "max_input_tokens": 16000, - "input_cost_per_token": 0.00000012, - "output_cost_per_token": 0.000000, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 0.0, "litellm_provider": "voyage", "mode": "embedding" }, "voyage/voyage-code-2": { "max_tokens": 16000, "max_input_tokens": 16000, - "input_cost_per_token": 0.00000012, - "output_cost_per_token": 0.000000, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 0.0, "litellm_provider": "voyage", "mode": "embedding" }, "voyage/voyage-2": { "max_tokens": 4000, "max_input_tokens": 4000, - "input_cost_per_token": 0.0000001, - "output_cost_per_token": 0.000000, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 0.0, "litellm_provider": "voyage", "mode": "embedding" }, "voyage/voyage-lite-02-instruct": { "max_tokens": 4000, "max_input_tokens": 4000, - "input_cost_per_token": 0.0000001, - "output_cost_per_token": 0.000000, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 0.0, "litellm_provider": "voyage", "mode": "embedding" + }, + "openrouter/liuhaotian/llava-yi-34b": { + "max_tokens": 4096, + "input_cost_per_token": 9e-07, + "output_cost_per_token": 9e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0005184, + "supports_vision": true + }, + "openrouter/meta-llama/llama-2-70b-chat:nitro": { + "max_tokens": 4096, + "input_cost_per_token": 9e-07, + "output_cost_per_token": 9e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/open-orca/mistral-7b-openorca": { + "max_tokens": 8192, + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 1.8e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/jondurbin/bagel-34b": { + "max_tokens": 8000, + "input_cost_per_token": -1.0, + "output_cost_per_token": -1.0, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": -1.0 + }, + "openrouter/anthropic/claude-2.1": { + "max_tokens": 200000, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/undi95/toppy-m-7b:nitro": { + "max_tokens": 4096, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 7e-08, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/mistralai/mistral-tiny": { + "max_tokens": 32000, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 2.5e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/jebcarter/psyfighter-13b": { + "max_tokens": 4096, + "input_cost_per_token": -1.0, + "output_cost_per_token": -1.0, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": -1.0 + }, + "openrouter/mistralai/mixtral-8x7b": { + "max_tokens": 32768, + "input_cost_per_token": 5.4e-07, + "output_cost_per_token": 5.4e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/undi95/remm-slerp-l2-13b:extended": { + "max_tokens": 6144, + "input_cost_per_token": 1.125e-06, + "output_cost_per_token": 1.125e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/phind/phind-codellama-34b": { + "max_tokens": 4096, + "input_cost_per_token": 7.2e-07, + "output_cost_per_token": 7.2e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/lynn/soliloquy-l3": { + "max_tokens": 24576, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 5e-08, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/anthropic/claude-2.0:beta": { + "max_tokens": 100000, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/undi95/toppy-m-7b": { + "max_tokens": 4096, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 7e-08, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/togethercomputer/stripedhyena-hessian-7b": { + "max_tokens": 32768, + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 1.8e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/openai/gpt-4-32k-0314": { + "max_tokens": 32767, + "input_cost_per_token": 6e-05, + "output_cost_per_token": 0.00012, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/openai/gpt-3.5-turbo-0613": { + "max_tokens": 4095, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/openai/gpt-4-32k": { + "max_tokens": 32767, + "input_cost_per_token": 6e-05, + "output_cost_per_token": 0.00012, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/anthropic/claude-3-sonnet:beta": { + "max_tokens": 200000, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0048, + "supports_vision": true + }, + "openrouter/togethercomputer/stripedhyena-nous-7b": { + "max_tokens": 32768, + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 1.8e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/perplexity/llama-3-sonar-large-32k-online": { + "max_tokens": 28000, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 1e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/gryphe/mythomax-l2-13b:extended": { + "max_tokens": 8192, + "input_cost_per_token": 1.125e-06, + "output_cost_per_token": 1.125e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/01-ai/yi-34b": { + "max_tokens": 4096, + "input_cost_per_token": 7.2e-07, + "output_cost_per_token": 7.2e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/allenai/olmo-7b-instruct": { + "max_tokens": 2048, + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 1.8e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/meta-llama/llama-3-8b": { + "max_tokens": 8192, + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 1.8e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/google/gemma-7b-it:nitro": { + "max_tokens": 8192, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/neversleep/noromaid-mixtral-8x7b-instruct": { + "max_tokens": 8000, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 8e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/rwkv/rwkv-5-world-3b": { + "max_tokens": 10000, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/qwen/qwen-4b-chat": { + "max_tokens": 32768, + "input_cost_per_token": 9e-08, + "output_cost_per_token": 9e-08, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/nousresearch/nous-capybara-34b": { + "max_tokens": 32768, + "input_cost_per_token": 9e-07, + "output_cost_per_token": 9e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/microsoft/wizardlm-2-8x22b": { + "max_tokens": 65536, + "input_cost_per_token": 6.5e-07, + "output_cost_per_token": 6.5e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/cohere/command": { + "max_tokens": 4096, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/neversleep/noromaid-20b": { + "max_tokens": 8192, + "input_cost_per_token": 2.25e-06, + "output_cost_per_token": 2.25e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/openai/gpt-4-1106-preview": { + "max_tokens": 128000, + "input_cost_per_token": 1e-05, + "output_cost_per_token": 3e-05, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/perplexity/llama-3-sonar-small-32k-online": { + "max_tokens": 28000, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/deepseek/deepseek-chat": { + "max_tokens": 128000, + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 2.8e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/mistralai/mistral-7b-instruct:nitro": { + "max_tokens": 32768, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/google/gemma-7b-it:free": { + "max_tokens": 8192, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/neversleep/llama-3-lumimaid-8b:extended": { + "max_tokens": 24576, + "input_cost_per_token": 2.25e-07, + "output_cost_per_token": 2.25e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/deepseek/deepseek-coder": { + "max_tokens": 16000, + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 2.8e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/openai/gpt-4-0314": { + "max_tokens": 8191, + "input_cost_per_token": 3e-05, + "output_cost_per_token": 6e-05, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/sao10k/fimbulvetr-11b-v2": { + "max_tokens": 8192, + "input_cost_per_token": 5.499375e-07, + "output_cost_per_token": 2.825625e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/sophosympatheia/midnight-rose-70b": { + "max_tokens": 4096, + "input_cost_per_token": 9e-06, + "output_cost_per_token": 9e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/qwen/qwen-72b-chat": { + "max_tokens": 32768, + "input_cost_per_token": 9e-07, + "output_cost_per_token": 9e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/huggingfaceh4/zephyr-7b-beta:free": { + "max_tokens": 4096, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/anthropic/claude-3-opus:beta": { + "max_tokens": 200000, + "input_cost_per_token": 1.5e-05, + "output_cost_per_token": 7.5e-05, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.024, + "supports_vision": true + }, + "openrouter/openchat/openchat-7b": { + "max_tokens": 8192, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 7e-08, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/google/palm-2-codechat-bison-32k": { + "max_tokens": 91750, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 5e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-sft": { + "max_tokens": 32768, + "input_cost_per_token": 5.4e-07, + "output_cost_per_token": 5.4e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/perplexity/llama-3-sonar-small-32k-chat": { + "max_tokens": 32768, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/01-ai/yi-6b": { + "max_tokens": 4096, + "input_cost_per_token": 1.26e-07, + "output_cost_per_token": 1.26e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/anthropic/claude-3-haiku:beta": { + "max_tokens": 200000, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 1.25e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0004, + "supports_vision": true + }, + "openrouter/openai/gpt-3.5-turbo-1106": { + "max_tokens": 16385, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/snowflake/snowflake-arctic-instruct": { + "max_tokens": 4096, + "input_cost_per_token": 2.16e-06, + "output_cost_per_token": 2.16e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/qwen/qwen-32b-chat": { + "max_tokens": 32768, + "input_cost_per_token": 7.2e-07, + "output_cost_per_token": 7.2e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/recursal/eagle-7b": { + "max_tokens": 10000, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/gryphe/mythomist-7b:free": { + "max_tokens": 32768, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/meta-llama/llama-3-8b-instruct": { + "max_tokens": 8192, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 7e-08, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/google/palm-2-chat-bison-32k": { + "max_tokens": 91750, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 5e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/meta-llama/llama-3-70b": { + "max_tokens": 8192, + "input_cost_per_token": 8.1e-07, + "output_cost_per_token": 8.1e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/openrouter/cinematika-7b:free": { + "max_tokens": 8000, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/lizpreciatior/lzlv-70b-fp16-hf": { + "max_tokens": 4096, + "input_cost_per_token": 5.9e-07, + "output_cost_per_token": 7.9e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/meta-llama/llama-guard-2-8b": { + "max_tokens": 8192, + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 1.8e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/cohere/command-r": { + "max_tokens": 128000, + "input_cost_per_token": 5e-07, + "output_cost_per_token": 1.5e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/google/gemini-pro": { + "max_tokens": 91728, + "input_cost_per_token": 1.25e-07, + "output_cost_per_token": 3.75e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0025 + }, + "openrouter/recursal/rwkv-5-3b-ai-town": { + "max_tokens": 10000, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/openai/gpt-3.5-turbo-0301": { + "max_tokens": 4095, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/mistralai/mistral-small": { + "max_tokens": 32000, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/teknium/openhermes-2-mistral-7b": { + "max_tokens": 4096, + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 1.8e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/anthropic/claude-2.1:beta": { + "max_tokens": 200000, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/intel/neural-chat-7b": { + "max_tokens": 4096, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 5e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/undi95/toppy-m-7b:free": { + "max_tokens": 4096, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/google/gemini-flash-1.5": { + "max_tokens": 2800000, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 7.5e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.000265, + "supports_vision": true + }, + "openrouter/neversleep/llama-3-lumimaid-8b": { + "max_tokens": 24576, + "input_cost_per_token": 2.25e-07, + "output_cost_per_token": 2.25e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/qwen/qwen-14b-chat": { + "max_tokens": 32768, + "input_cost_per_token": 2.7e-07, + "output_cost_per_token": 2.7e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/nousresearch/nous-hermes-2-mistral-7b-dpo": { + "max_tokens": 8192, + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 1.8e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/teknium/openhermes-2.5-mistral-7b": { + "max_tokens": 4096, + "input_cost_per_token": 1.7e-07, + "output_cost_per_token": 1.7e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/anthropic/claude-instant-1.0": { + "max_tokens": 100000, + "input_cost_per_token": 8e-07, + "output_cost_per_token": 2.4e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/gryphe/mythomist-7b": { + "max_tokens": 32768, + "input_cost_per_token": 3.75e-07, + "output_cost_per_token": 3.75e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/qwen/qwen-7b-chat": { + "max_tokens": 32768, + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 1.8e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/liuhaotian/llava-13b": { + "max_tokens": 2048, + "input_cost_per_token": 1e-05, + "output_cost_per_token": 1e-05, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0, + "supports_vision": true + }, + "openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo": { + "max_tokens": 32768, + "input_cost_per_token": 2.7e-07, + "output_cost_per_token": 2.7e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/anthropic/claude-2:beta": { + "max_tokens": 200000, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/anthropic/claude-instant-1:beta": { + "max_tokens": 100000, + "input_cost_per_token": 8e-07, + "output_cost_per_token": 2.4e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/nousresearch/nous-capybara-7b": { + "max_tokens": 4096, + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 1.8e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/mistralai/mixtral-8x22b": { + "max_tokens": 65536, + "input_cost_per_token": 9e-07, + "output_cost_per_token": 9e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/openrouter/cinematika-7b": { + "max_tokens": 8000, + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 1.8e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/mistralai/mixtral-8x7b-instruct": { + "max_tokens": 32768, + "input_cost_per_token": 2.4e-07, + "output_cost_per_token": 2.4e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/google/gemma-7b-it": { + "max_tokens": 8192, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 7e-08, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/nousresearch/nous-capybara-7b:free": { + "max_tokens": 4096, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/openrouter/auto": { + "max_tokens": 200000, + "input_cost_per_token": -1.0, + "output_cost_per_token": -1.0, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": -1.0 + }, + "openrouter/anthropic/claude-2.0": { + "max_tokens": 100000, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/codellama/codellama-70b-instruct": { + "max_tokens": 2048, + "input_cost_per_token": 8.1e-07, + "output_cost_per_token": 8.1e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/alpindale/goliath-120b": { + "max_tokens": 6144, + "input_cost_per_token": 9.375e-06, + "output_cost_per_token": 9.375e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/nousresearch/nous-hermes-yi-34b": { + "max_tokens": 4096, + "input_cost_per_token": 7.2e-07, + "output_cost_per_token": 7.2e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/xwin-lm/xwin-lm-70b": { + "max_tokens": 8192, + "input_cost_per_token": 3.75e-06, + "output_cost_per_token": 3.75e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/microsoft/wizardlm-2-7b": { + "max_tokens": 32000, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 7e-08, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/gryphe/mythomax-l2-13b:nitro": { + "max_tokens": 4096, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/perplexity/llama-3-sonar-large-32k-chat": { + "max_tokens": 32768, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 1e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/koboldai/psyfighter-13b-2": { + "max_tokens": 4096, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 1e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/anthropic/claude-1": { + "max_tokens": 100000, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/nousresearch/nous-hermes-2-vision-7b": { + "max_tokens": 4096, + "input_cost_per_token": 1e-05, + "output_cost_per_token": 1e-05, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0, + "supports_vision": true + }, + "openrouter/anthropic/claude-1.2": { + "max_tokens": 100000, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 2.4e-05, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/openai/gpt-4-turbo": { + "max_tokens": 128000, + "input_cost_per_token": 1e-05, + "output_cost_per_token": 3e-05, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.01445, + "supports_vision": true + }, + "openrouter/openchat/openchat-7b:free": { + "max_tokens": 8192, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/jondurbin/airoboros-l2-70b": { + "max_tokens": 4096, + "input_cost_per_token": 7e-07, + "output_cost_per_token": 9e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/qwen/qwen-110b-chat": { + "max_tokens": 32768, + "input_cost_per_token": 1.62e-06, + "output_cost_per_token": 1.62e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/meta-llama/llama-3-8b-instruct:nitro": { + "max_tokens": 8192, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/mistralai/mistral-medium": { + "max_tokens": 32000, + "input_cost_per_token": 2.7e-06, + "output_cost_per_token": 8.1e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/anthropic/claude-instant-1.1": { + "max_tokens": 100000, + "input_cost_per_token": 8e-07, + "output_cost_per_token": 2.4e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/anthropic/claude-instant-1": { + "max_tokens": 100000, + "input_cost_per_token": 8e-07, + "output_cost_per_token": 2.4e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/austism/chronos-hermes-13b": { + "max_tokens": 4096, + "input_cost_per_token": 1.3e-07, + "output_cost_per_token": 1.3e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/neversleep/llama-3-lumimaid-70b": { + "max_tokens": 8192, + "input_cost_per_token": 8e-06, + "output_cost_per_token": 8e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/openai/gpt-4-turbo-preview": { + "max_tokens": 128000, + "input_cost_per_token": 1e-05, + "output_cost_per_token": 3e-05, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/openai/gpt-3.5-turbo-0125": { + "max_tokens": 16385, + "input_cost_per_token": 5e-07, + "output_cost_per_token": 1.5e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/openai/gpt-3.5-turbo-instruct": { + "max_tokens": 4095, + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 2e-06, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/01-ai/yi-34b-chat": { + "max_tokens": 4096, + "input_cost_per_token": 7.2e-07, + "output_cost_per_token": 7.2e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 + }, + "openrouter/mistralai/mixtral-8x7b-instruct:nitro": { + "max_tokens": 32768, + "input_cost_per_token": 5.4e-07, + "output_cost_per_token": 5.4e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_image": 0.0 } - -} +} \ No newline at end of file From b81fcf24826d79e0a573d4f16e20abb00d9dd22a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 21 May 2024 22:00:36 -0700 Subject: [PATCH 017/192] feat -add failure callbacks from DB to proxy --- litellm/proxy/proxy_server.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index cd9251942b8..c7bf509266c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2805,11 +2805,18 @@ class ProxyConfig: config_data = await proxy_config.get_config() litellm_settings = config_data.get("litellm_settings", {}) or {} success_callbacks = litellm_settings.get("success_callback", None) + failure_callbacks = litellm_settings.get("failure_callback", None) if success_callbacks is not None and isinstance(success_callbacks, list): for success_callback in success_callbacks: if success_callback not in litellm.success_callback: litellm.success_callback.append(success_callback) + + # Add failure callbacks from DB to litellm + if failure_callbacks is not None and isinstance(failure_callbacks, list): + for failure_callback in failure_callbacks: + if failure_callback not in litellm.failure_callback: + litellm.failure_callback.append(failure_callback) # we need to set env variables too environment_variables = config_data.get("environment_variables", {}) for k, v in environment_variables.items(): From f1ae9f0acd19569d9c5a0c6cd01af93aec0a8a60 Mon Sep 17 00:00:00 2001 From: vietpham1911 Date: Wed, 22 May 2024 16:10:39 +0930 Subject: [PATCH 018/192] Added the requested features of Merlinvt ("1.If the model is not multimodal, there is no info needed for input_cost_per_image. Right now it's 0.0 for a lot of models. 2. Could you add the field max_output_tokens (is it max_completion_tokens in OpenRouter?).") --- ...to_update_price_and_context_window_file.py | 31 +++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/.github/workflows/auto_update_price_and_context_window_file.py b/.github/workflows/auto_update_price_and_context_window_file.py index 06b08bd3081..3e0731b94bd 100644 --- a/.github/workflows/auto_update_price_and_context_window_file.py +++ b/.github/workflows/auto_update_price_and_context_window_file.py @@ -47,28 +47,41 @@ def write_to_file(file_path, data): def transform_remote_data(data): transformed = {} for row in data: - # Create a new dictionary with transformed data + # Add the fields 'max_tokens' and 'input_cost_per_token' obj = { "max_tokens": row["context_length"], "input_cost_per_token": float(row["pricing"]["prompt"]), - "output_cost_per_token": float(row["pricing"]["completion"]), - "litellm_provider": "openrouter", - "mode": "chat" } - # Add an field 'input_cost_per_image' - if "pricing" in row and "image" in row["pricing"]: + # Add 'max_output_tokens' as a field if it is not None + if "top_provider" in row and "max_completion_tokens" in row["top_provider"] and row["top_provider"]["max_completion_tokens"] is not None: + obj['max_output_tokens'] = int(row["top_provider"]["max_completion_tokens"]) + + # Add the field 'output_cost_per_token' + obj.update({ + "output_cost_per_token": float(row["pricing"]["completion"]), + }) + + # Add field 'input_cost_per_image' if it exists and is non-zero + if "pricing" in row and "image" in row["pricing"] and float(row["pricing"]["image"]) != 0.0: obj['input_cost_per_image'] = float(row["pricing"]["image"]) - # Add an additional field if the modality is 'multimodal' + # Add the fields 'litellm_provider' and 'mode' + obj.update({ + "litellm_provider": "openrouter", + "mode": "chat" + }) + + # Add the 'supports_vision' field if the modality is 'multimodal' if row.get('architecture', {}).get('modality') == 'multimodal': obj['supports_vision'] = True - + # Use a composite key to store the transformed object transformed[f'openrouter/{row["id"]}'] = obj return transformed + # Load local data from a specified file def load_local_data(file_path): try: @@ -105,4 +118,4 @@ def main(): # Entry point of the script if __name__ == "__main__": - main() + main() \ No newline at end of file From 815dcd9187d7faa758212d09594ff999e2d9d096 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 22 May 2024 07:48:34 -0700 Subject: [PATCH 019/192] fix - don't use gunicorn on litellm helm --- deploy/charts/litellm-helm/templates/deployment.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/deploy/charts/litellm-helm/templates/deployment.yaml b/deploy/charts/litellm-helm/templates/deployment.yaml index 736f35680e5..07e617581ed 100644 --- a/deploy/charts/litellm-helm/templates/deployment.yaml +++ b/deploy/charts/litellm-helm/templates/deployment.yaml @@ -161,7 +161,6 @@ spec: args: - --config - /etc/litellm/config.yaml - - --run_gunicorn ports: - name: http containerPort: {{ .Values.service.port }} From 778b41cea52ea39628048ac1cecbc001ffb1f247 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 22 May 2024 12:22:23 -0700 Subject: [PATCH 020/192] fix - update migration script to ensure api_key in script --- cookbook/misc/migrate_proxy_config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cookbook/misc/migrate_proxy_config.py b/cookbook/misc/migrate_proxy_config.py index f1d736dc8c2..53551a0ce75 100644 --- a/cookbook/misc/migrate_proxy_config.py +++ b/cookbook/misc/migrate_proxy_config.py @@ -54,6 +54,9 @@ def migrate_models(config_file, proxy_base_url): new_value = input(f"Enter value for {value}: ") _in_memory_os_variables[value] = new_value litellm_params[param] = new_value + if "api_key" not in litellm_params: + new_value = input(f"Enter api key for {model_name}: ") + litellm_params["api_key"] = new_value print("\nlitellm_params: ", litellm_params) # Confirm before sending POST request From 9609df16d308a203811ed0fb460d03b8d7c19f14 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 22 May 2024 13:16:37 -0700 Subject: [PATCH 021/192] feat(proxy_server.py): new 'add budget' endpoint create a budget object (max budget, tpm /rpm limits, etc.) and assign that to a user/team/etc. --- litellm/proxy/_types.py | 25 +++++++++++++++++++++++ litellm/proxy/proxy_server.py | 37 +++++++++++++++++++++++++++++++++-- 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index a6640e4d3db..8bfa56004f3 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -676,6 +676,31 @@ class OrganizationRequest(LiteLLMBase): organizations: List[str] +class BudgetNew(LiteLLMBase): + budget_id: str = Field(default=None, description="The unique budget id.") + max_budget: Optional[float] = Field( + default=None, + description="Requests will fail if this budget (in USD) is exceeded.", + ) + soft_budget: Optional[float] = Field( + default=None, + description="Requests will NOT fail if this is exceeded. Will fire alerting though.", + ) + max_parallel_requests: Optional[int] = Field( + default=None, description="Max concurrent requests allowed for this budget id." + ) + tpm_limit: Optional[int] = Field( + default=None, description="Max tokens per minute, allowed for this budget id." + ) + rpm_limit: Optional[int] = Field( + default=None, description="Max requests per minute, allowed for this budget id." + ) + budget_duration: Optional[str] = Field( + default=None, + description="Max duration budget should be set for (e.g. '1hr', '1d', '28d')", + ) + + class BudgetRequest(LiteLLMBase): budgets: List[str] diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c7bf509266c..f157f420cce 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3470,7 +3470,9 @@ async def startup_event(): if prisma_client is not None and master_key is not None: # add master key to db if os.getenv("PROXY_ADMIN_ID", None) is not None: - litellm_proxy_admin_name = os.getenv("PROXY_ADMIN_ID") + litellm_proxy_admin_name = os.getenv( + "PROXY_ADMIN_ID", litellm_proxy_admin_name + ) asyncio.create_task( generate_key_helper_fn( duration=None, @@ -8187,9 +8189,40 @@ async def info_organization(data: OrganizationRequest): #### BUDGET TABLE MANAGEMENT #### +@router.post( + "/budget/new", + tags=["budget management"], + dependencies=[Depends(user_api_key_auth)], +) +async def new_budget( + budget_obj: BudgetNew, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Create a new budget object. Can apply this to teams, orgs, end-users, keys. + """ + global prisma_client + + if prisma_client is None: + raise HTTPException( + status_code=500, + detail={"error": CommonProxyErrors.db_not_connected_error.value}, + ) + + response = await prisma_client.db.litellm_budgettable.create( + data={ + **budget_obj.model_dump(exclude_none=True), # type: ignore + "created_by": user_api_key_dict.user_id or litellm_proxy_admin_name, + "updated_by": user_api_key_dict.user_id or litellm_proxy_admin_name, + } # type: ignore + ) + + return response + + @router.post( "/budget/info", - tags=["organization management"], + tags=["budget management"], dependencies=[Depends(user_api_key_auth)], ) async def info_budget(data: BudgetRequest): From eac7e70dcaf26a7484f8374c970cb665c6992efd Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 22 May 2024 13:30:08 -0700 Subject: [PATCH 022/192] feat - include litellm_budget table when getting end_user --- litellm/proxy/auth/auth_checks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 08da25556d6..fce6d4254b8 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -219,7 +219,8 @@ async def get_end_user_object( # else, check db try: response = await prisma_client.db.litellm_endusertable.find_unique( - where={"user_id": end_user_id} + where={"user_id": end_user_id}, + include={"litellm_budget_table": True}, ) if response is None: From 106910cecf3e839c78bd6310468c3aa29001469b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 22 May 2024 14:01:57 -0700 Subject: [PATCH 023/192] feat - add end user rate limiting --- litellm/proxy/_types.py | 5 ++ .../proxy/hooks/parallel_request_limiter.py | 70 ++++++++++++++++++- 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 8bfa56004f3..ce38f4f9edd 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -938,6 +938,11 @@ class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken): soft_budget: Optional[float] = None team_model_aliases: Optional[Dict] = None + # End User Params + end_user_id: Optional[str] = None + end_user_tpm_limit: Optional[int] = None + end_user_rpm_limit: Optional[int] = None + class UserAPIKeyAuth( LiteLLM_VerificationTokenView diff --git a/litellm/proxy/hooks/parallel_request_limiter.py b/litellm/proxy/hooks/parallel_request_limiter.py index 26238b6c089..0558cdf05b3 100644 --- a/litellm/proxy/hooks/parallel_request_limiter.py +++ b/litellm/proxy/hooks/parallel_request_limiter.py @@ -64,7 +64,8 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger): cache.set_cache(request_count_api_key, new_val) else: raise HTTPException( - status_code=429, detail="Max parallel request limit reached." + status_code=429, + detail=f"LiteLLM Rate Limit Handler: Crossed TPM, RPM Limit. current rpm: {current['current_rpm']}, rpm limit: {rpm_limit}, current tpm: {current['current_tpm']}, tpm limit: {tpm_limit}", ) async def async_pre_call_hook( @@ -223,6 +224,38 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger): rpm_limit=team_rpm_limit, ) + # End-User Rate Limits + # Only enforce if user passed `user` to /chat, /completions, /embeddings + if user_api_key_dict.end_user_id: + end_user_tpm_limit = getattr( + user_api_key_dict, "end_user_tpm_limit", sys.maxsize + ) + end_user_rpm_limit = getattr( + user_api_key_dict, "end_user_rpm_limit", sys.maxsize + ) + + if end_user_tpm_limit is None: + end_user_tpm_limit = sys.maxsize + if end_user_rpm_limit is None: + end_user_rpm_limit = sys.maxsize + + # now do the same tpm/rpm checks + request_count_api_key = ( + f"{user_api_key_dict.end_user_id}::{precise_minute}::request_count" + ) + + # print(f"Checking if {request_count_api_key} is allowed to make request for minute {precise_minute}") + await self.check_key_in_limits( + user_api_key_dict=user_api_key_dict, + cache=cache, + data=data, + call_type=call_type, + max_parallel_requests=sys.maxsize, # TODO: Support max parallel requests for an End-User + request_count_api_key=request_count_api_key, + tpm_limit=end_user_tpm_limit, + rpm_limit=end_user_rpm_limit, + ) + return async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -238,6 +271,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger): user_api_key_team_id = kwargs["litellm_params"]["metadata"].get( "user_api_key_team_id", None ) + user_api_key_end_user_id = kwargs.get("user") if self.user_api_key_cache is None: return @@ -362,6 +396,40 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger): request_count_api_key, new_val, ttl=60 ) # store in cache for 1 min. + # ------------ + # Update usage - End User + # ------------ + if user_api_key_end_user_id is not None: + total_tokens = 0 + + if isinstance(response_obj, ModelResponse): + total_tokens = response_obj.usage.total_tokens + + request_count_api_key = ( + f"{user_api_key_end_user_id}::{precise_minute}::request_count" + ) + + current = self.user_api_key_cache.get_cache( + key=request_count_api_key + ) or { + "current_requests": 1, + "current_tpm": total_tokens, + "current_rpm": 1, + } + + new_val = { + "current_requests": max(current["current_requests"] - 1, 0), + "current_tpm": current["current_tpm"] + total_tokens, + "current_rpm": current["current_rpm"] + 1, + } + + self.print_verbose( + f"updated_value in success call: {new_val}, precise_minute: {precise_minute}" + ) + self.user_api_key_cache.set_cache( + request_count_api_key, new_val, ttl=60 + ) # store in cache for 1 min. + except Exception as e: self.print_verbose(e) # noqa From bef10f4b01acbb4c9eb61f05b218afceacd27d06 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 22 May 2024 15:42:41 -0700 Subject: [PATCH 024/192] test - end user tpm / rpm limiting --- tests/test_end_users.py | 86 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 85 insertions(+), 1 deletion(-) diff --git a/tests/test_end_users.py b/tests/test_end_users.py index 83bffdc12be..9c8b597531a 100644 --- a/tests/test_end_users.py +++ b/tests/test_end_users.py @@ -99,7 +99,12 @@ async def generate_key( async def new_end_user( - session, i, user_id=str(uuid.uuid4()), model_region=None, default_model=None + session, + i, + user_id=str(uuid.uuid4()), + model_region=None, + default_model=None, + budget_id=None, ): url = "http://0.0.0.0:4000/end_user/new" headers = {"Authorization": "Bearer sk-1234", "Content-Type": "application/json"} @@ -109,6 +114,10 @@ async def new_end_user( "default_model": default_model, } + if budget_id is not None: + data["budget_id"] = budget_id + print("end user data: {}".format(data)) + async with session.post(url, headers=headers, json=data) as response: status = response.status response_text = await response.text() @@ -123,6 +132,23 @@ async def new_end_user( return await response.json() +async def new_budget(session, i, budget_id=None): + url = "http://0.0.0.0:4000/budget/new" + headers = {"Authorization": "Bearer sk-1234", "Content-Type": "application/json"} + data = { + "budget_id": budget_id, + "tpm_limit": 2, + } + + async with session.post(url, headers=headers, json=data) as response: + status = response.status + response_text = await response.text() + + print(f"Response {i} (Status code: {status}):") + print(response_text) + print() + + @pytest.mark.asyncio async def test_end_user_new(): """ @@ -170,3 +196,61 @@ async def test_end_user_specific_region(): ) assert result.headers.get("x-litellm-model-region") == "eu" + + +@pytest.mark.asyncio +async def test_end_tpm_limits(): + """ + 1. budget_id = Create Budget with tpm_limit = 10 + 2. create end_user with budget_id + 3. Make /chat/completions calls + 4. Sleep 1 second + 4. Make /chat/completions call -> expect this to fail because rate limit hit + """ + async with aiohttp.ClientSession() as session: + # create a budget with budget_id = "free-tier" + budget_id = f"free-tier-{uuid.uuid4()}" + await new_budget(session, 0, budget_id=budget_id) + await asyncio.sleep(2) + + end_user_id = str(uuid.uuid4()) + + await new_end_user( + session=session, i=0, user_id=end_user_id, budget_id=budget_id + ) + + ## MAKE CALL ## + key_gen = await generate_key(session=session, i=0, models=[]) + + key = key_gen["key"] + + # chat completion 1 + client = AsyncOpenAI(api_key=key, base_url="http://0.0.0.0:4000") + + result = await client.chat.completions.create( + model="fake-openai-endpoint", + messages=[{"role": "user", "content": "Hey!"}], + user=end_user_id, + ) + + print("\nchat completion result 1=", result) + + await asyncio.sleep(1) + + # chat completion 2 + try: + result = await client.chat.completions.create( + model="fake-openai-endpoint", + messages=[{"role": "user", "content": "Hey!"}], + user=end_user_id, + ) + pytest.fail( + "User crossed their limit - this should have failed. instead got result = {}".format( + result + ) + ) + except Exception as e: + print("got exception 2 =", e) + assert "Crossed TPM, RPM Limit" in str( + e + ), f"Expected 'Crossed TPM, RPM Limit' but got {str(e)}" From e6b406d739e5d7fdf8554a17f418b31e8f789c42 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 22 May 2024 15:45:30 -0700 Subject: [PATCH 025/192] feat - enforce end user tpm / rpm limits --- litellm/proxy/proxy_server.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f157f420cce..80b7c84d502 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -683,6 +683,17 @@ async def user_api_key_auth( end_user_params["allowed_model_region"] = ( _end_user_object.allowed_model_region ) + if _end_user_object.litellm_budget_table is not None: + budget_info = _end_user_object.litellm_budget_table + end_user_params["end_user_id"] = _end_user_object.user_id + if budget_info.tpm_limit is not None: + end_user_params["end_user_tpm_limit"] = ( + budget_info.tpm_limit + ) + if budget_info.rpm_limit is not None: + end_user_params["end_user_rpm_limit"] = ( + budget_info.rpm_limit + ) except Exception as e: verbose_proxy_logger.debug( "Unable to find user in db. Error - {}".format(str(e)) @@ -1148,10 +1159,7 @@ async def user_api_key_auth( valid_token_dict.pop("token", None) if _end_user_object is not None: - valid_token_dict["allowed_model_region"] = ( - _end_user_object.allowed_model_region - ) - + valid_token_dict.update(end_user_params) """ asyncio create task to update the user api key cache with the user db table as well From 42078ac285783bf559355ed4da7bd7143e250073 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 22 May 2024 16:15:09 -0700 Subject: [PATCH 026/192] fix - run tpm / rpm checks on proxy admin keys too --- litellm/proxy/proxy_server.py | 45 ++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 80b7c84d502..fbcff31c28f 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -655,19 +655,6 @@ async def user_api_key_auth( detail="'allow_user_auth' not set or set to False", ) - ### CHECK IF ADMIN ### - # note: never string compare api keys, this is vulenerable to a time attack. Use secrets.compare_digest instead - ### CHECK IF ADMIN ### - # note: never string compare api keys, this is vulenerable to a time attack. Use secrets.compare_digest instead - ## Check CACHE - valid_token = user_api_key_cache.get_cache(key=hash_token(api_key)) - if ( - valid_token is not None - and isinstance(valid_token, UserAPIKeyAuth) - and valid_token.user_role == "proxy_admin" - ): - return valid_token - ## Check END-USER OBJECT request_data = await _read_request_body(request=request) _end_user_object = None @@ -700,6 +687,27 @@ async def user_api_key_auth( ) pass + ### CHECK IF ADMIN ### + # note: never string compare api keys, this is vulenerable to a time attack. Use secrets.compare_digest instead + ### CHECK IF ADMIN ### + # note: never string compare api keys, this is vulenerable to a time attack. Use secrets.compare_digest instead + ## Check CACHE + valid_token = user_api_key_cache.get_cache(key=hash_token(api_key)) + if ( + valid_token is not None + and isinstance(valid_token, UserAPIKeyAuth) + and valid_token.user_role == "proxy_admin" + ): + # update end-user params on valid token + valid_token.end_user_id = end_user_params.get("end_user_id") + valid_token.end_user_tpm_limit = end_user_params.get("end_user_tpm_limit") + valid_token.end_user_rpm_limit = end_user_params.get("end_user_rpm_limit") + valid_token.allowed_model_region = end_user_params.get( + "allowed_model_region" + ) + + return valid_token + try: is_master_key_valid = secrets.compare_digest(api_key, master_key) # type: ignore except Exception as e: @@ -772,9 +780,18 @@ async def user_api_key_auth( key=original_api_key, table_name="key" ) verbose_proxy_logger.debug("Token from db: %s", valid_token) - elif valid_token is not None: + elif valid_token is not None and isinstance(valid_token, UserAPIKeyAuth): verbose_proxy_logger.debug("API Key Cache Hit!") + # update end-user params on valid token + # These can change per request - it's important to update them here + valid_token.end_user_id = end_user_params.get("end_user_id") + valid_token.end_user_tpm_limit = end_user_params.get("end_user_tpm_limit") + valid_token.end_user_rpm_limit = end_user_params.get("end_user_rpm_limit") + valid_token.allowed_model_region = end_user_params.get( + "allowed_model_region" + ) + user_id_information = None if valid_token: # Got Valid Token from Cache, DB From 7f0d85f12a94593dd6e0241617b500402eb56486 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 22 May 2024 16:20:06 -0700 Subject: [PATCH 027/192] feat(proxy_server.py): enable admin to create/delete budgets for end-users --- litellm/proxy/_types.py | 4 + litellm/proxy/proxy_server.py | 136 ++++++++++++ ui/litellm-dashboard/src/app/page.tsx | 32 +-- .../src/components/budgets/budget_modal.tsx | 140 ++++++++++++ .../src/components/budgets/budget_panel.tsx | 201 ++++++++++++++++++ .../components/budgets/budget_settings.tsx | 195 +++++++++++++++++ .../src/components/create_user_button.tsx | 31 ++- .../src/components/leftnav.tsx | 111 ++++------ .../src/components/networking.tsx | 141 ++++++++++++ .../src/components/view_users.tsx | 62 +++--- 10 files changed, 931 insertions(+), 122 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/budgets/budget_modal.tsx create mode 100644 ui/litellm-dashboard/src/components/budgets/budget_panel.tsx create mode 100644 ui/litellm-dashboard/src/components/budgets/budget_settings.tsx diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 8bfa56004f3..8cdf6de725e 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -705,6 +705,10 @@ class BudgetRequest(LiteLLMBase): budgets: List[str] +class BudgetDeleteRequest(LiteLLMBase): + id: str + + class KeyManagementSystem(enum.Enum): GOOGLE_KMS = "google_kms" AZURE_KEY_VAULT = "azure_key_vault" diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f157f420cce..e9edd383639 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -8248,6 +8248,142 @@ async def info_budget(data: BudgetRequest): return response +@router.get( + "/budget/settings", + tags=["budget management"], + dependencies=[Depends(user_api_key_auth)], +) +async def budget_settings( + budget_id: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Get list of configurable params + current value for a budget item + description of each field + + Used on Admin UI. + """ + if prisma_client is None: + raise HTTPException( + status_code=400, + detail={"error": CommonProxyErrors.db_not_connected_error.value}, + ) + + if user_api_key_dict.user_role != "proxy_admin": + raise HTTPException( + status_code=400, + detail={ + "error": "{}, your role={}".format( + CommonProxyErrors.not_allowed_access.value, + user_api_key_dict.user_role, + ) + }, + ) + + ## get budget item from db + db_budget_row = await prisma_client.db.litellm_budgettable.find_first( + where={"budget_id": budget_id} + ) + + if db_budget_row is not None: + db_budget_row_dict = db_budget_row.model_dump(exclude_none=True) + else: + db_budget_row_dict = {} + + allowed_args = { + "max_parallel_requests": {"type": "Integer"}, + "tpm_limit": {"type": "Integer"}, + "rpm_limit": {"type": "Integer"}, + "budget_duration": {"type": "String"}, + "max_budget": {"type": "Float"}, + "soft_budget": {"type": "Float"}, + } + + return_val = [] + + for field_name, field_info in BudgetNew.model_fields.items(): + if field_name in allowed_args: + + _stored_in_db = True + + _response_obj = ConfigList( + field_name=field_name, + field_type=allowed_args[field_name]["type"], + field_description=field_info.description or "", + field_value=db_budget_row_dict.get(field_name, None), + stored_in_db=_stored_in_db, + ) + return_val.append(_response_obj) + + return return_val + + +@router.get( + "/budget/list", + tags=["budget management"], + dependencies=[Depends(user_api_key_auth)], +) +async def list_budget( + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """List all the created budgets in proxy db. Used on Admin UI.""" + if prisma_client is None: + raise HTTPException( + status_code=400, + detail={"error": CommonProxyErrors.db_not_connected_error.value}, + ) + + if user_api_key_dict.user_role != "proxy_admin": + raise HTTPException( + status_code=400, + detail={ + "error": "{}, your role={}".format( + CommonProxyErrors.not_allowed_access.value, + user_api_key_dict.user_role, + ) + }, + ) + + response = await prisma_client.db.litellm_budgettable.find_many() + + return response + + +@router.post( + "/budget/delete", + tags=["budget management"], + dependencies=[Depends(user_api_key_auth)], +) +async def delete_budget( + data: BudgetDeleteRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """Delete budget""" + global prisma_client + + if prisma_client is None: + raise HTTPException( + status_code=500, + detail={"error": CommonProxyErrors.db_not_connected_error.value}, + ) + + if user_api_key_dict.user_role != "proxy_admin": + raise HTTPException( + status_code=400, + detail={ + "error": "{}, your role={}".format( + CommonProxyErrors.not_allowed_access.value, + user_api_key_dict.user_role, + ) + }, + ) + + response = await prisma_client.db.litellm_budgettable.delete( + where={"budget_id": data.id} + ) + + return response + + #### MODEL MANAGEMENT #### diff --git a/ui/litellm-dashboard/src/app/page.tsx b/ui/litellm-dashboard/src/app/page.tsx index ce692508151..7df196d9b26 100644 --- a/ui/litellm-dashboard/src/app/page.tsx +++ b/ui/litellm-dashboard/src/app/page.tsx @@ -9,6 +9,7 @@ import Teams from "@/components/teams"; import AdminPanel from "@/components/admins"; import Settings from "@/components/settings"; import GeneralSettings from "@/components/general_settings"; +import BudgetPanel from "@/components/budgets/budget_panel"; import APIRef from "@/components/api_ref"; import ChatUI from "@/components/chat_ui"; import Sidebar from "../components/leftnav"; @@ -106,14 +107,13 @@ const CreateKeyPage = () => { />
- - +
- + {page == "api-keys" ? ( { showSSOBanner={showSSOBanner} /> ) : page == "api_ref" ? ( - + ) : page == "settings" ? ( - ) : page == "general-settings" ? ( - + ) : page == "budgets" ? ( + + ) : page == "general-settings" ? ( + ) : ( >; + setBudgetList: React.Dispatch>; +} +const BudgetModal: React.FC = ({ + isModalVisible, + accessToken, + setIsModalVisible, + setBudgetList, +}) => { + const [form] = Form.useForm(); + const handleOk = () => { + setIsModalVisible(false); + form.resetFields(); + }; + + const handleCancel = () => { + setIsModalVisible(false); + form.resetFields(); + }; + + const handleCreate = async (formValues: Record) => { + if (accessToken == null || accessToken == undefined) { + return; + } + try { + message.info("Making API Call"); + // setIsModalVisible(true); + const response = await budgetCreateCall(accessToken, formValues); + console.log("key create Response:", response); + setBudgetList((prevData) => + prevData ? [...prevData, response] : [response] + ); // Check if prevData is null + message.success("API Key Created"); + form.resetFields(); + } catch (error) { + console.error("Error creating the key:", error); + message.error(`Error creating the key: ${error}`, 20); + } + }; + + return ( + +
+ <> + + + + + + + + + + + + + Optional Settings + + + + + + + + + + + + +
+ Create Budget +
+
+ + ); +}; + +export default BudgetModal; diff --git a/ui/litellm-dashboard/src/components/budgets/budget_panel.tsx b/ui/litellm-dashboard/src/components/budgets/budget_panel.tsx new file mode 100644 index 00000000000..705597ab26d --- /dev/null +++ b/ui/litellm-dashboard/src/components/budgets/budget_panel.tsx @@ -0,0 +1,201 @@ +/** + * The parent pane, showing list of budgets + * + */ + +import React, { useState, useEffect } from "react"; +import BudgetSettings from "./budget_settings"; +import BudgetModal from "./budget_modal"; +import { + Table, + TableBody, + TableCell, + TableFoot, + TableHead, + TableHeaderCell, + TableRow, + Card, + Button, + Icon, + Text, + Tab, + TabGroup, + TabList, + TabPanel, + TabPanels, + Grid, +} from "@tremor/react"; +import { + InformationCircleIcon, + PencilAltIcon, + PencilIcon, + StatusOnlineIcon, + TrashIcon, + RefreshIcon, + CheckCircleIcon, + XCircleIcon, + QuestionMarkCircleIcon, +} from "@heroicons/react/outline"; +import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; +import { getBudgetList, budgetDeleteCall } from "../networking"; +import { message } from "antd"; +interface BudgetSettingsPageProps { + accessToken: string | null; +} + +interface budgetItem { + budget_id: string; + max_budget: string | null; + rpm_limit: number | null; + tpm_limit: number | null; +} + +const BudgetPanel: React.FC = ({ accessToken }) => { + const [isModalVisible, setIsModalVisible] = useState(false); + const [budgetList, setBudgetList] = useState([]); + useEffect(() => { + if (!accessToken) { + return; + } + getBudgetList(accessToken).then((data) => { + setBudgetList(data); + }); + }, [accessToken]); + + const handleDeleteCall = async (budget_id: string, index: number) => { + if (accessToken == null) { + return; + } + + message.info("Request made"); + + await budgetDeleteCall(accessToken, budget_id); + + const newBudgetList = [...budgetList]; + newBudgetList.splice(index, 1); + setBudgetList(newBudgetList); + + message.success("Budget Deleted."); + }; + + return ( +
+ + + + Create a budget to assign to customers. + + + + Budget ID + Max Budget + TPM + RPM + + + + + {budgetList.map((value: budgetItem, index: number) => ( + + {value.budget_id} + + {value.max_budget ? value.max_budget : "n/a"} + + + {value.tpm_limit ? value.tpm_limit : "n/a"} + + + {value.rpm_limit ? value.rpm_limit : "n/a"} + + handleDeleteCall(value.budget_id, index)} + /> + + ))} + +
+ +
+ How to use budget id + + + Assign Budget to Customer + Test it (Curl) + + Test it (OpenAI SDK) + + + + + {` +curl -X POST --location '/end_user/new' \ + +-H 'Authorization: Bearer ' \ + +-H 'Content-Type: application/json' \ + +-d '{"user_id": "my-customer-id', "budget_id": ""}' # 👈 KEY CHANGE + + `} + + + + + {` +curl -X POST --location '/chat/completions' \ + +-H 'Authorization: Bearer ' \ + +-H 'Content-Type: application/json' \ + +-d '{ + "model": "gpt-3.5-turbo', + "messages":[{"role": "user", "content": "Hey, how's it going?"}], + "user": "my-customer-id" +}' # 👈 KEY CHANGE + + `} + + + + + {`from openai import OpenAI +client = OpenAI( + base_url=" + + + +
+
+ ); +}; + +export default BudgetPanel; diff --git a/ui/litellm-dashboard/src/components/budgets/budget_settings.tsx b/ui/litellm-dashboard/src/components/budgets/budget_settings.tsx new file mode 100644 index 00000000000..7f31ac248aa --- /dev/null +++ b/ui/litellm-dashboard/src/components/budgets/budget_settings.tsx @@ -0,0 +1,195 @@ +import React, { useState, useEffect } from "react"; +import { + Card, + Title, + Subtitle, + Table, + TableHead, + TableRow, + Badge, + TableHeaderCell, + TableCell, + TableBody, + Metric, + Text, + Grid, + Button, + TextInput, + Select as Select2, + SelectItem, + Col, + Accordion, + AccordionBody, + AccordionHeader, + AccordionList, +} from "@tremor/react"; +import { + TabPanel, + TabPanels, + TabGroup, + TabList, + Tab, + Icon, +} from "@tremor/react"; +import { getBudgetSettings } from "../networking"; +import { + Modal, + Form, + Input, + Select, + Button as Button2, + message, + InputNumber, +} from "antd"; +import { + InformationCircleIcon, + PencilAltIcon, + PencilIcon, + StatusOnlineIcon, + TrashIcon, + RefreshIcon, + CheckCircleIcon, + XCircleIcon, + QuestionMarkCircleIcon, +} from "@heroicons/react/outline"; +import StaticGenerationSearchParamsBailoutProvider from "next/dist/client/components/static-generation-searchparams-bailout-provider"; +import AddFallbacks from "../add_fallbacks"; +import openai from "openai"; +import Paragraph from "antd/es/skeleton/Paragraph"; + +interface BudgetSettingsPageProps { + accessToken: string | null; +} + +interface budgetSettingsItem { + field_name: string; + field_type: string; + field_value: any; + field_description: string; +} + +const BudgetSettings: React.FC = ({ accessToken }) => { + const [budgetSettings, setBudgetSettings] = useState( + [] + ); + useEffect(() => { + if (!accessToken) { + return; + } + getBudgetSettings(accessToken).then((data) => { + console.log("budget settings", data); + let budget_settings = data.budget_settings; + setBudgetSettings(budget_settings); + }); + }, [accessToken]); + + const handleInputChange = (fieldName: string, newValue: any) => { + // Update the value in the state + const updatedSettings = budgetSettings.map((setting) => + setting.field_name === fieldName + ? { ...setting, field_value: newValue } + : setting + ); + setBudgetSettings(updatedSettings); + }; + + const handleUpdateField = (fieldName: string, idx: number) => { + if (!accessToken) { + return; + } + + let fieldValue = budgetSettings[idx].field_value; + + if (fieldValue == null || fieldValue == undefined) { + return; + } + try { + const updatedSettings = budgetSettings.map((setting) => + setting.field_name === fieldName + ? { ...setting, stored_in_db: true } + : setting + ); + setBudgetSettings(updatedSettings); + } catch (error) { + // do something + } + }; + + const handleResetField = (fieldName: string, idx: number) => { + if (!accessToken) { + return; + } + + try { + const updatedSettings = budgetSettings.map((setting) => + setting.field_name === fieldName + ? { ...setting, stored_in_db: null, field_value: null } + : setting + ); + setBudgetSettings(updatedSettings); + } catch (error) { + // do something + } + }; + + return ( +
+ + + + + Setting + Value + + + + {budgetSettings.map((value, index) => ( + + + {value.field_name} +

+ {value.field_description} +

+ + + {value.field_type == "Integer" ? ( + + handleInputChange(value.field_name, newValue) + } // Handle value change + /> + ) : null} + + + + handleResetField(value.field_name, index)} + > + Reset + + + + ))} + +
+ +
+ ); +}; + +export default BudgetSettings; diff --git a/ui/litellm-dashboard/src/components/create_user_button.tsx b/ui/litellm-dashboard/src/components/create_user_button.tsx index c0332b35c02..512ff23c789 100644 --- a/ui/litellm-dashboard/src/components/create_user_button.tsx +++ b/ui/litellm-dashboard/src/components/create_user_button.tsx @@ -10,7 +10,11 @@ interface CreateuserProps { teams: any[] | null; } -const Createuser: React.FC = ({ userID, accessToken, teams }) => { +const Createuser: React.FC = ({ + userID, + accessToken, + teams, +}) => { const [form] = Form.useForm(); const [isModalVisible, setIsModalVisible] = useState(false); const [apiuser, setApiuser] = useState(null); @@ -73,7 +77,7 @@ const Createuser: React.FC = ({ userID, accessToken, teams }) = return (
- setIsModalVisible(true)}> + setIsModalVisible(true)}> + Invite User = ({ userID, accessToken, teams }) = onOk={handleOk} onCancel={handleCancel} > - Invite a user to login to the Admin UI and create Keys - Note: SSO Setup Required for this + + Invite a user to login to the Admin UI and create Keys + + + Note: SSO Setup Required for this +
= ({ userID, accessToken, teams }) = - {teams ? ( teams.map((team: any) => (
diff --git a/ui/litellm-dashboard/src/components/leftnav.tsx b/ui/litellm-dashboard/src/components/leftnav.tsx index c4296f18e62..ec2f6b47805 100644 --- a/ui/litellm-dashboard/src/components/leftnav.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.tsx @@ -1,7 +1,7 @@ import { Layout, Menu } from "antd"; import Link from "next/link"; import { List } from "postcss/lib/list"; -import { Text } from "@tremor/react" +import { Text } from "@tremor/react"; const { Sider } = Layout; @@ -54,88 +54,61 @@ const Sidebar: React.FC = ({ style={{ height: "100%", borderRight: 0 }} > setPage("api-keys")}> - - API Keys - + API Keys setPage("llm-playground")}> - - Test Key - + Test Key - - - { - userRole == "Admin" ? ( - setPage("models")}> - - Models - - - ) : null - } - { - userRole == "Admin" ? ( - setPage("usage")}> - - Usage - + {userRole == "Admin" ? ( + setPage("models")}> + Models - - ) : null - } + ) : null} + {userRole == "Admin" ? ( + setPage("usage")}> + Usage + + ) : null} {userRole == "Admin" ? ( setPage("teams")}> - - Teams - + Teams ) : null} - - - {userRole == "Admin" ? ( - setPage("users")}> - - Users - - - ) : null} - - { - userRole == "Admin" ? ( - setPage("settings")}> - - Logging & Alerts - - - ) : null - } - - { - userRole == "Admin" ? ( - setPage("general-settings")}> - - Router Settings - - - ) : null - } - {userRole == "Admin" ? ( - setPage("admin-panel")}> - - Admin - + setPage("users")}> + Users ) : null} - setPage("api_ref")}> - - API Reference - - + + {userRole == "Admin" ? ( + setPage("settings")}> + Logging & Alerts + + ) : null} + + {userRole == "Admin" ? ( + setPage("budgets")}> + Rate Limits + + ) : null} + + {userRole == "Admin" ? ( + setPage("general-settings")}> + Router Settings + + ) : null} + + {userRole == "Admin" ? ( + setPage("admin-panel")}> + Admin + + ) : null} + setPage("api_ref")}> + API Reference + diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index bb7899755b9..06e0a06924f 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -97,6 +97,82 @@ export const modelDeleteCall = async ( } }; +export const budgetDeleteCall = async ( + accessToken: string | null, + budget_id: string +) => { + console.log(`budget_id in budget delete call: ${budget_id}`); + + if (accessToken == null) { + return; + } + + try { + const url = proxyBaseUrl + ? `${proxyBaseUrl}/budget/delete` + : `/budget/delete`; + const response = await fetch(url, { + method: "POST", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + id: budget_id, + }), + }); + + if (!response.ok) { + const errorData = await response.text(); + message.error("Failed to create key: " + errorData, 10); + console.error("Error response from the server:", errorData); + throw new Error("Network response was not ok"); + } + const data = await response.json(); + console.log("API Response:", data); + return data; + } catch (error) { + console.error("Failed to create key:", error); + throw error; + } +}; + +export const budgetCreateCall = async ( + accessToken: string, + formValues: Record // Assuming formValues is an object +) => { + try { + console.log("Form Values in budgetCreateCall:", formValues); // Log the form values before making the API call + + console.log("Form Values after check:", formValues); + const url = proxyBaseUrl ? `${proxyBaseUrl}/budget/new` : `/budget/new`; + const response = await fetch(url, { + method: "POST", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + ...formValues, // Include formValues in the request body + }), + }); + + if (!response.ok) { + const errorData = await response.text(); + message.error("Failed to create key: " + errorData, 10); + console.error("Error response from the server:", errorData); + throw new Error("Network response was not ok"); + } + + const data = await response.json(); + console.log("API Response:", data); + return data; + // Handle success - you might want to update some state or UI based on the created key + } catch (error) { + console.error("Failed to create key:", error); + throw error; + } +}; export const keyCreateCall = async ( accessToken: string, userID: string, @@ -1382,6 +1458,71 @@ export const serviceHealthCheck = async ( } }; +export const getBudgetList = async (accessToken: String) => { + /** + * Get all configurable params for setting a budget + */ + try { + let url = proxyBaseUrl ? `${proxyBaseUrl}/budget/list` : `/budget/list`; + + //message.info("Requesting model data"); + const response = await fetch(url, { + method: "GET", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + + if (!response.ok) { + const errorData = await response.text(); + message.error(errorData, 10); + throw new Error("Network response was not ok"); + } + + const data = await response.json(); + //message.info("Received model data"); + return data; + // Handle success - you might want to update some state or UI based on the created key + } catch (error) { + console.error("Failed to get callbacks:", error); + throw error; + } +}; +export const getBudgetSettings = async (accessToken: String) => { + /** + * Get all configurable params for setting a budget + */ + try { + let url = proxyBaseUrl + ? `${proxyBaseUrl}/budget/settings` + : `/budget/settings`; + + //message.info("Requesting model data"); + const response = await fetch(url, { + method: "GET", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + + if (!response.ok) { + const errorData = await response.text(); + message.error(errorData, 10); + throw new Error("Network response was not ok"); + } + + const data = await response.json(); + //message.info("Received model data"); + return data; + // Handle success - you might want to update some state or UI based on the created key + } catch (error) { + console.error("Failed to get callbacks:", error); + throw error; + } +}; + export const getCallbacksCall = async ( accessToken: String, userID: String, diff --git a/ui/litellm-dashboard/src/components/view_users.tsx b/ui/litellm-dashboard/src/components/view_users.tsx index 41c15be9c56..3faf4c02687 100644 --- a/ui/litellm-dashboard/src/components/view_users.tsx +++ b/ui/litellm-dashboard/src/components/view_users.tsx @@ -19,7 +19,7 @@ import { TabPanel, Select, SelectItem, - Dialog, + Dialog, DialogPanel, Icon, TextInput, @@ -82,8 +82,6 @@ const ViewUserDashboard: React.FC = ({ if (accessToken && token && userRole && userID) { fetchData(); } - - }, [accessToken, token, userRole, userID, currentPage]); if (!userData) { @@ -102,7 +100,7 @@ const ViewUserDashboard: React.FC = ({ return (
- Showing Page {currentPage+1} of {totalPages} + Showing Page {currentPage + 1} of {totalPages}