From 96dda40f45b3d12ea03e861d060ec81460b7759e Mon Sep 17 00:00:00 2001 From: Muhammad Sannan Nasir Date: Mon, 30 Jun 2025 20:48:22 +0500 Subject: [PATCH] Add digitalocean provider --- .circleci/config.yml | 62 +- README.md | 13 +- docs/my-website/Dockerfile | 4 +- docs/my-website/sidebars.js | 3 +- litellm/__init__.py | 8 +- litellm/constants.py | 1 + .../get_llm_provider_logic.py | 9 + .../llms/digitalocean/chat/transformation.py | 191 +++ litellm/main.py | 20 + litellm/types/utils.py | 1 + litellm/utils.py | 13 + model_prices_and_context_window.json | 1500 +++++++++-------- proxy_server_config.yaml | 27 +- .../test_digitalocean_chat_transformation.py | 60 + .../out/assets/logos/digitalocean.svg | 229 +++ .../add_model/provider_specific_fields.tsx | 32 +- .../src/components/provider_info_helpers.tsx | 19 +- 17 files changed, 1396 insertions(+), 796 deletions(-) create mode 100644 litellm/llms/digitalocean/chat/transformation.py create mode 100644 tests/litellm/llms/digitalocean/chat/test_digitalocean_chat_transformation.py create mode 100644 ui/litellm-dashboard/out/assets/logos/digitalocean.svg diff --git a/.circleci/config.yml b/.circleci/config.yml index feb425a38e0..f552c7b0911 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1154,7 +1154,7 @@ jobs: name: Install Helm command: | curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash - + # Install kind - run: name: Install Kind @@ -1162,7 +1162,7 @@ jobs: curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 chmod +x ./kind sudo mv ./kind /usr/local/bin/kind - + # Install kubectl - run: name: Install kubectl @@ -1170,19 +1170,19 @@ jobs: curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" chmod +x kubectl sudo mv kubectl /usr/local/bin/ - + # Create kind cluster - run: name: Create Kind Cluster command: | kind create cluster --name litellm-test - + # Run helm lint - run: name: Run helm lint command: | helm lint ./deploy/charts/litellm-helm - + # Run helm tests - run: name: Run helm tests @@ -1191,15 +1191,15 @@ jobs: # Wait for pod to be ready echo "Waiting 30 seconds for pod to be ready..." sleep 30 - + # Print pod logs before running tests echo "Printing pod logs..." kubectl logs $(kubectl get pods -l app.kubernetes.io/name=litellm -o jsonpath="{.items[0].metadata.name}") - + # Run the helm tests helm test litellm --logs helm test litellm --logs - + # Cleanup - run: name: Cleanup @@ -1309,7 +1309,7 @@ jobs: echo "=== Printing Full Container Startup Logs ===" docker logs my-app echo "=== End of Full Container Startup Logs ===" - + if docker logs my-app 2>&1 | grep -q "prisma schema out of sync with db. Consider running these sql_commands to sync the two"; then echo "Expected message found in logs. Test passed." else @@ -1427,6 +1427,7 @@ jobs: -e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \ -e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \ -e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \ + -e DIGITALOCEAN_API_KEY=$DIGITALOCEAN_API_KEY \ --name my-app \ -v $(pwd)/proxy_server_config.yaml:/app/config.yaml \ my-app:latest \ @@ -1552,6 +1553,7 @@ jobs: -e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \ -e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \ -e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \ + -e DIGITALOCEAN_API_KEY=$DIGITALOCEAN_API_KEY \ --name my-app \ -v $(pwd)/litellm/proxy/example_config_yaml/oai_misc_config.yaml:/app/config.yaml \ my-app:latest \ @@ -2030,7 +2032,7 @@ jobs: no_output_timeout: 120m # Clean up first container - + proxy_build_from_pip_tests: # Change from docker to machine executor machine: @@ -2235,17 +2237,17 @@ jobs: curl -sSL https://rvm.io/mpapis.asc | gpg --import - curl -sSL https://rvm.io/pkuczynski.asc | gpg --import - } - + # Install Ruby version manager (RVM) curl -sSL https://get.rvm.io | bash -s stable - + # Source RVM from the correct location source $HOME/.rvm/scripts/rvm - + # Install Ruby 3.2.2 rvm install 3.2.2 rvm use 3.2.2 --default - + # Install latest Bundler gem install bundler @@ -2408,32 +2410,32 @@ jobs: python -m pip install toml # Get current version from pyproject.toml CURRENT_VERSION=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])") - + # Get last published version from PyPI LAST_VERSION=$(curl -s https://pypi.org/pypi/litellm-proxy-extras/json | python -c "import json, sys; print(json.load(sys.stdin)['info']['version'])") - + echo "Current version: $CURRENT_VERSION" echo "Last published version: $LAST_VERSION" - + # Compare versions using Python's packaging.version VERSION_COMPARE=$(python -c "from packaging import version; print(1 if version.parse('$CURRENT_VERSION') < version.parse('$LAST_VERSION') else 0)") - + echo "Version compare: $VERSION_COMPARE" if [ "$VERSION_COMPARE" = "1" ]; then echo "Error: Current version ($CURRENT_VERSION) is less than last published version ($LAST_VERSION)" exit 1 fi - + # If versions are equal or current is greater, check contents pip download --no-deps litellm-proxy-extras==$LAST_VERSION -d /tmp - + echo "Contents of /tmp directory:" ls -la /tmp - + # Find the downloaded file (could be .whl or .tar.gz) DOWNLOADED_FILE=$(ls /tmp/litellm_proxy_extras-*) echo "Downloaded file: $DOWNLOADED_FILE" - + # Extract based on file extension if [[ "$DOWNLOADED_FILE" == *.whl ]]; then echo "Extracting wheel file..." @@ -2444,10 +2446,10 @@ jobs: tar -xzf "$DOWNLOADED_FILE" -C /tmp EXTRACTED_DIR="/tmp/litellm_proxy_extras-$LAST_VERSION" fi - + echo "Contents of extracted package:" ls -R "$EXTRACTED_DIR" - + # Compare contents if ! diff -r "$EXTRACTED_DIR/litellm_proxy_extras" ./litellm_proxy_extras; then if [ "$CURRENT_VERSION" = "$LAST_VERSION" ]; then @@ -2513,16 +2515,16 @@ jobs: export NVM_DIR="/opt/circleci/.nvm" source "$NVM_DIR/nvm.sh" source "$NVM_DIR/bash_completion" - + # Install and use Node version nvm install v18.17.0 nvm use v18.17.0 - + cd ui/litellm-dashboard - + # Install dependencies first npm install - + # Now source the build script source ./build_ui.sh - run: @@ -2569,7 +2571,7 @@ jobs: name: Install Playwright Browsers command: | npx playwright install - + - run: name: Build Docker image command: docker build -t my-app:latest -f ./docker/Dockerfile.database . @@ -2933,4 +2935,4 @@ workflows: - proxy_pass_through_endpoint_tests - check_code_and_doc_quality - publish_proxy_extras - + diff --git a/README.md b/README.md index 1c4e1484437..41725cab366 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature # Usage ([**Docs**](https://docs.litellm.ai/docs/)) > [!IMPORTANT] -> LiteLLM v1.0.0 now requires `openai>=1.0.0`. Migration guide [here](https://docs.litellm.ai/docs/migration) +> LiteLLM v1.0.0 now requires `openai>=1.0.0`. Migration guide [here](https://docs.litellm.ai/docs/migration) > LiteLLM v1.40.14+ now requires `pydantic>=2.0.0`. No changes required. @@ -129,7 +129,7 @@ print(response) ## Streaming ([Docs](https://docs.litellm.ai/docs/completion/stream)) -liteLLM supports streaming the model response back, pass `stream=True` to get a streaming iterator in response. +liteLLM supports streaming the model response back, pass `stream=True` to get a streaming iterator in response. Streaming is supported for all models (Bedrock, Huggingface, TogetherAI, Azure, OpenAI, etc.) ```python @@ -227,7 +227,7 @@ $ litellm --model huggingface/bigcode/starcoder > [!IMPORTANT] -> πŸ’‘ [Use LiteLLM Proxy with Langchain (Python, JS), OpenAI SDK (Python, JS) Anthropic SDK, Mistral SDK, LlamaIndex, Instructor, Curl](https://docs.litellm.ai/docs/proxy/user_keys) +> πŸ’‘ [Use LiteLLM Proxy with Langchain (Python, JS), OpenAI SDK (Python, JS) Anthropic SDK, Mistral SDK, LlamaIndex, Instructor, Curl](https://docs.litellm.ai/docs/proxy/user_keys) ```python import openai # openai v1.0.0+ @@ -259,7 +259,7 @@ echo 'LITELLM_MASTER_KEY="sk-1234"' > .env # Add the litellm salt key - you cannot change this after adding a model # It is used to encrypt / decrypt your LLM API Key credentials -# We recommend - https://1password.com/password-generator/ +# We recommend - https://1password.com/password-generator/ # password generator to get a random hash for litellm salt key echo 'LITELLM_SALT_KEY="sk-1234"' > .env @@ -332,6 +332,7 @@ curl 'http://0.0.0.0:4000/key/generate' \ | [xinference [Xorbits Inference]](https://docs.litellm.ai/docs/providers/xinference) | | | | | βœ… | | | [FriendliAI](https://docs.litellm.ai/docs/providers/friendliai) | βœ… | βœ… | βœ… | βœ… | | | | [Galadriel](https://docs.litellm.ai/docs/providers/galadriel) | βœ… | βœ… | βœ… | βœ… | | | +| [DigitalOcean](https://docs.litellm.ai/docs/providers/digitalocean) | βœ… | βœ… | | | | | [**Read the Docs**](https://docs.litellm.ai/docs/) @@ -344,7 +345,7 @@ For companies that need better security, user management and professional suppor [Talk to founders](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) -This covers: +This covers: - βœ… **Features under the [LiteLLM Commercial License](https://docs.litellm.ai/docs/proxy/enterprise):** - βœ… **Feature Prioritization** - βœ… **Custom Integrations** @@ -356,7 +357,7 @@ This covers: LiteLLM follows the [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html). -We run: +We run: - Ruff for [formatting and linting checks](https://github.com/BerriAI/litellm/blob/e19bb55e3b4c6a858b6e364302ebbf6633a51de5/.circleci/config.yml#L320) - Mypy + Pyright for typing [1](https://github.com/BerriAI/litellm/blob/e19bb55e3b4c6a858b6e364302ebbf6633a51de5/.circleci/config.yml#L90), [2](https://github.com/BerriAI/litellm/blob/e19bb55e3b4c6a858b6e364302ebbf6633a51de5/.pre-commit-config.yaml#L4) - Black for [formatting](https://github.com/BerriAI/litellm/blob/e19bb55e3b4c6a858b6e364302ebbf6633a51de5/.circleci/config.yml#L79) diff --git a/docs/my-website/Dockerfile b/docs/my-website/Dockerfile index 87d1537237d..a1989a9dd97 100644 --- a/docs/my-website/Dockerfile +++ b/docs/my-website/Dockerfile @@ -4,6 +4,6 @@ COPY . /app WORKDIR /app RUN pip install -r requirements.txt -EXPOSE $PORT +EXPOSE 3000 -CMD litellm --host 0.0.0.0 --port $PORT --workers 10 --config config.yaml \ No newline at end of file +CMD litellm --host 0.0.0.0 --port 3000 --workers 10 --config config.yaml \ No newline at end of file diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 60030a59bb3..87f25827512 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -245,7 +245,8 @@ const sidebars = { "providers/sambanova", "providers/custom_llm_server", "providers/petals", - "providers/snowflake" + "providers/snowflake", + "providers/digitalocean", ], }, { diff --git a/litellm/__init__.py b/litellm/__init__.py index 59c8c78eb9e..6b9c3636b2a 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -193,6 +193,7 @@ baseten_key: Optional[str] = None aleph_alpha_key: Optional[str] = None nlp_cloud_key: Optional[str] = None snowflake_key: Optional[str] = None +digitalocean_api_key: Optional[str] = None common_cloud_provider_auth_params: dict = { "params": ["project", "region_name", "token"], "providers": ["vertex_ai", "bedrock", "watsonx", "azure", "vertex_ai_beta"], @@ -431,6 +432,7 @@ galadriel_models: List = [] sambanova_models: List = [] assemblyai_models: List = [] snowflake_models: List = [] +digitalocean_models: List = [] def is_bedrock_pricing_only_model(key: str) -> bool: @@ -588,6 +590,8 @@ def add_known_models(): jina_ai_models.append(key) elif value.get("litellm_provider") == "snowflake": snowflake_models.append(key) + elif value.get("litellm_provider") == "digitalocean": + digitalocean_models.append(key) add_known_models() @@ -664,6 +668,7 @@ model_list = ( + assemblyai_models + jina_ai_models + snowflake_models + + digitalocean_models ) model_list_set = set(model_list) @@ -721,6 +726,7 @@ models_by_provider: dict = { "assemblyai": assemblyai_models, "jina_ai": jina_ai_models, "snowflake": snowflake_models, + "digitalocean": digitalocean_models, } # mapping for those models which have larger equivalents @@ -963,7 +969,7 @@ from .llms.openai.chat.o_series_transformation import ( ) from .llms.snowflake.chat.transformation import SnowflakeConfig - +from .llms.digitalocean.chat.transformation import DigitalOceanConfig openaiOSeriesConfig = OpenAIOSeriesConfig() from .llms.openai.chat.gpt_transformation import ( OpenAIGPTConfig, diff --git a/litellm/constants.py b/litellm/constants.py index f48ce97afe6..e8ba4a7d24b 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -157,6 +157,7 @@ LITELLM_CHAT_PROVIDERS = [ "hosted_vllm", "lm_studio", "galadriel", + "digitalocean" ] diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 13103c85a00..98d537011f0 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -321,6 +321,8 @@ def get_llm_provider( # noqa: PLR0915 custom_llm_provider = "openai" elif model in litellm.empower_models: custom_llm_provider = "empower" + elif model in litellm.digitalocean_models: + custom_llm_provider = "digitalocean" elif model == "*": custom_llm_provider = "openai" if not custom_llm_provider: @@ -580,6 +582,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 or f"https://{get_secret('SNOWFLAKE_ACCOUNT_ID')}.snowflakecomputing.com/api/v2/cortex/inference:complete" ) # type: ignore dynamic_api_key = api_key or get_secret_str("SNOWFLAKE_JWT") + elif custom_llm_provider == "digitalocean": + ( + api_base, + dynamic_api_key, + ) = litellm.DigitalOceanConfig()._get_openai_compatible_provider_info( + api_base, api_key + ) if api_base is not None and not isinstance(api_base, str): raise Exception("api base needs to be a string. api_base={}".format(api_base)) diff --git a/litellm/llms/digitalocean/chat/transformation.py b/litellm/llms/digitalocean/chat/transformation.py new file mode 100644 index 00000000000..f89a0ab8767 --- /dev/null +++ b/litellm/llms/digitalocean/chat/transformation.py @@ -0,0 +1,191 @@ +""" +Translate from OpenAI's `/v1/chat/completions` to DigitalOcean AI Agent's `/v1/chat/completions` +""" + +from typing import List, Optional, Tuple, Union, Dict + +from pydantic import BaseModel + +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionAssistantMessage, + ChatCompletionToolParam, + ChatCompletionToolParamFunctionChunk, +) + +from ...openai_like.chat.transformation import OpenAILikeChatConfig + + +class DigitalOceanConfig(OpenAILikeChatConfig): + + # DigitalOcean-specific parameters: + stream_options: Optional[Dict] = None + kb_filters: Optional[List[Dict]] = None + filter_kb_content_by_query_metadata: Optional[bool] = None + instruction_override: Optional[str] = None + include_functions_info: Optional[bool] = None + include_retrieval_info: Optional[bool] = None + include_guardrails_info: Optional[bool] = None + provide_citations: Optional[bool] = None + + def __init__( + self, + frequency_penalty: Optional[float] = None, + function_call: Optional[Union[str, Dict]] = None, + functions: Optional[List] = None, + logit_bias: Optional[Dict] = None, + max_tokens: Optional[int] = None, + max_completion_tokens: Optional[int] = None, # Additional field; interchangeable with max_tokens per spec + n: Optional[int] = None, + presence_penalty: Optional[float] = None, + retrieval_method: Optional[str] = None, # expected values: "rewrite", "step_back", "sub_queries", "none" + stop: Optional[Union[str, List[str]]] = None, + stream: Optional[bool] = None, + temperature: Optional[float] = None, + top_p: Optional[float] = None, + response_format: Optional[Dict] = None, + tools: Optional[List] = None, + tool_choice: Optional[Union[str, Dict]] = None, + ) -> None: + locals_ = locals().copy() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + + @classmethod + def get_config(cls): + return super().get_config() + + def get_supported_openai_params(self, model: str) -> list: + base_params = super().get_supported_openai_params(model) + try: + base_params.remove("max_retries") + except ValueError: + pass + return base_params + + def validate_environment(self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None): + api_key = api_key or get_secret_str("DIGITALOCEAN_API_KEY") + if api_key is None: + raise ValueError("DigitalOcean API key not found") + if headers is None: + headers = {} + headers["Authorization"] = f"Bearer {api_key}" + headers["Content-Type"] = "application/json" + return headers + + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + + complete_url = f"{api_base}/api/v1/chat/completions" + return complete_url + + def _transform_messages(self, messages: List[AllMessageValues], model: str) -> List: + for idx, message in enumerate(messages): + """ + 1. Don't pass 'null' function_call assistant message to groq - https://github.com/BerriAI/litellm/issues/5839 + """ + if isinstance(message, BaseModel): + _message = message.model_dump() + else: + _message = message + assistant_message = _message.get("role") == "assistant" + if assistant_message: + new_message = ChatCompletionAssistantMessage(role="assistant") + for k, v in _message.items(): + if v is not None: + new_message[k] = v # type: ignore + messages[idx] = new_message + + return messages + + def _get_openai_compatible_provider_info( + self, api_base: Optional[str], api_key: Optional[str] + ) -> Tuple[Optional[str], Optional[str]]: + api_base = ( + api_base + or get_secret_str("DIGITALOCEAN_AGENT_ENDPOINT") + or get_secret_str("DO_AGENT_ENDPOINT") + ) # type: ignore + dynamic_api_key = api_key or get_secret_str("DO_API_KEY") or get_secret_str("DIGITALOCEAN_API_KEY") + return api_base, dynamic_api_key + + + + def _create_json_tool_call_for_response_format( + self, + json_schema: dict, + ): + """ + Handles creating a tool call for getting responses in JSON format. + + Args: + json_schema (Optional[dict]): The JSON schema the response should be in + + Returns: + AnthropicMessagesTool: The tool call to send to Anthropic API to get responses in JSON format + """ + return ChatCompletionToolParam( + type="function", + function=ChatCompletionToolParamFunctionChunk( + name="json_tool_call", + parameters=json_schema, + ), + ) + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool = False, + replace_max_completion_tokens_with_max_tokens: bool = False, # groq supports max_completion_tokens + ) -> dict: + _response_format = non_default_params.get("response_format") + if _response_format is not None and isinstance(_response_format, dict): + json_schema: Optional[dict] = None + if "response_schema" in _response_format: + json_schema = _response_format["response_schema"] + elif "json_schema" in _response_format: + json_schema = _response_format["json_schema"]["schema"] + """ + When using tools in this way: - https://docs.anthropic.com/en/docs/build-with-claude/tool-use#json-mode + - You usually want to provide a single tool + - You should set tool_choice (see Forcing tool use) to instruct the model to explicitly use that tool + - Remember that the model will pass the input to the tool, so the name of the tool and description should be from the model’s perspective. + """ + if json_schema is not None: + _tool_choice = { + "type": "function", + "function": {"name": "json_tool_call"}, + } + _tool = self._create_json_tool_call_for_response_format( + json_schema=json_schema, + ) + optional_params["tools"] = [_tool] + optional_params["tool_choice"] = _tool_choice + optional_params["json_mode"] = True + non_default_params.pop( + "response_format", None + ) # only remove if it's a json_schema - handled via using groq's tool calling params. + optional_params = super().map_openai_params( + non_default_params, optional_params, model, drop_params + ) + + return optional_params diff --git a/litellm/main.py b/litellm/main.py index de0716fd963..32e945a7896 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -3065,6 +3065,26 @@ def completion( # type: ignore # noqa: PLR0915 additional_args={"headers": headers}, ) raise e + elif custom_llm_provider == "digitalocean": + + api_base = litellm.api_base or api_base + response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + custom_llm_provider="digitalocean", + timeout=timeout, + headers=headers, + encoding=encoding, + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) elif custom_llm_provider == "custom": url = litellm.api_base or api_base or "" diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 532162e60f8..40c8b4187d8 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2102,6 +2102,7 @@ class LlmProviders(str, Enum): TOPAZ = "topaz" ASSEMBLYAI = "assemblyai" SNOWFLAKE = "snowflake" + DIGITALOCEAN = "digitalocean" # Create a set of all provider values for quick lookup diff --git a/litellm/utils.py b/litellm/utils.py index 98a9c34b470..154fca85b6a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3705,6 +3705,17 @@ def get_optional_params( # noqa: PLR0915 else False ), ) + elif custom_llm_provider == "digitalocean": + optional_params = litellm.DigitalOceanConfig().map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=( + drop_params + if drop_params is not None and isinstance(drop_params, bool) + else False + ), + ) elif custom_llm_provider == "openai": optional_params = litellm.OpenAIConfig().map_openai_params( non_default_params=non_default_params, @@ -6569,6 +6580,8 @@ class ProviderConfigManager: return litellm.LiteLLMProxyChatConfig() elif litellm.LlmProviders.OPENAI == provider: return litellm.OpenAIGPTConfig() + elif litellm.LlmProviders.DIGITALOCEAN == provider: + return litellm.DigitalOceanConfig() return None @staticmethod diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 55052761c78..f5ce53131eb 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1,8 +1,8 @@ { "sample_spec": { - "max_tokens": "LEGACY parameter. set to max_output_tokens if provider specifies it. IF not set to max_input_tokens, if provider specifies it.", + "max_tokens": "LEGACY parameter. set to max_output_tokens if provider specifies it. IF not set to max_input_tokens, if provider specifies it.", "max_input_tokens": "max input tokens, if the provider specifies it. if not default to max_tokens", - "max_output_tokens": "max output tokens, if the provider specifies it. if not default to max_tokens", + "max_output_tokens": "max output tokens, if the provider specifies it. if not default to max_tokens", "input_cost_per_token": 0.0000, "output_cost_per_token": 0.000, "output_cost_per_reasoning_token": 0.000, @@ -11,7 +11,7 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_vision": true, - "supports_audio_input": true, + "supports_audio_input": true, "supports_audio_output": true, "supports_prompt_caching": true, "supports_response_schema": true, @@ -28,7 +28,7 @@ "omni-moderation-latest": { "max_tokens": 32768, "max_input_tokens": 32768, - "max_output_tokens": 0, + "max_output_tokens": 0, "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "litellm_provider": "openai", @@ -37,7 +37,7 @@ "omni-moderation-latest-intents": { "max_tokens": 32768, "max_input_tokens": 32768, - "max_output_tokens": 0, + "max_output_tokens": 0, "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "litellm_provider": "openai", @@ -46,16 +46,16 @@ "omni-moderation-2024-09-26": { "max_tokens": 32768, "max_input_tokens": 32768, - "max_output_tokens": 0, + "max_output_tokens": 0, "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "litellm_provider": "openai", "mode": "moderation" }, "gpt-4": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 8192, - "max_output_tokens": 4096, + "max_output_tokens": 4096, "input_cost_per_token": 0.00003, "output_cost_per_token": 0.00006, "litellm_provider": "openai", @@ -253,21 +253,21 @@ } }, "watsonx/ibm/granite-3-8b-instruct": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 1024, - "input_cost_per_token": 0.0002, - "output_cost_per_token": 0.0002, - "litellm_provider": "watsonx", - "mode": "chat", - "supports_function_calling": true, + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 1024, + "input_cost_per_token": 0.0002, + "output_cost_per_token": 0.0002, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": true, "supports_tool_choice": true, - "supports_parallel_function_calling": false, - "supports_vision": false, - "supports_audio_input": false, - "supports_audio_output": false, - "supports_prompt_caching": true, - "supports_response_schema": true, + "supports_parallel_function_calling": false, + "supports_vision": false, + "supports_audio_input": false, + "supports_audio_output": false, + "supports_prompt_caching": true, + "supports_response_schema": true, "supports_system_messages": true }, "gpt-4o-search-preview-2025-03-11": { @@ -1329,7 +1329,7 @@ "text-embedding-3-small": { "max_tokens": 8191, "max_input_tokens": 8191, - "output_vector_size": 1536, + "output_vector_size": 1536, "input_cost_per_token": 0.00000002, "output_cost_per_token": 0.000000, "input_cost_per_token_batches": 0.000000010, @@ -1340,7 +1340,7 @@ "text-embedding-ada-002": { "max_tokens": 8191, "max_input_tokens": 8191, - "output_vector_size": 1536, + "output_vector_size": 1536, "input_cost_per_token": 0.0000001, "output_cost_per_token": 0.000000, "litellm_provider": "openai", @@ -1511,33 +1511,33 @@ "mode": "audio_transcription", "input_cost_per_token": 0.0000025, "input_cost_per_audio_token": 0.000006, - "output_cost_per_token": 0.00001, + "output_cost_per_token": 0.00001, "litellm_provider": "openai", "supported_endpoints": ["/v1/audio/transcriptions"] - }, + }, "gpt-4o-mini-transcribe": { "mode": "audio_transcription", "input_cost_per_token": 0.00000125, "input_cost_per_audio_token": 0.000003, - "output_cost_per_token": 0.000005, + "output_cost_per_token": 0.000005, "litellm_provider": "openai", "supported_endpoints": ["/v1/audio/transcriptions"] - }, + }, "whisper-1": { "mode": "audio_transcription", "input_cost_per_second": 0.0001, - "output_cost_per_second": 0.0001, + "output_cost_per_second": 0.0001, "litellm_provider": "openai", "supported_endpoints": ["/v1/audio/transcriptions"] - }, + }, "tts-1": { - "mode": "audio_speech", + "mode": "audio_speech", "input_cost_per_character": 0.000015, "litellm_provider": "openai", "supported_endpoints": ["/v1/audio/speech"] }, "tts-1-hd": { - "mode": "audio_speech", + "mode": "audio_speech", "input_cost_per_character": 0.000030, "litellm_provider": "openai", "supported_endpoints": ["/v1/audio/speech"] @@ -2070,19 +2070,19 @@ "supports_tool_choice": true }, "azure/tts-1": { - "mode": "audio_speech", + "mode": "audio_speech", "input_cost_per_character": 0.000015, "litellm_provider": "azure" }, "azure/tts-1-hd": { - "mode": "audio_speech", + "mode": "audio_speech", "input_cost_per_character": 0.000030, "litellm_provider": "azure" }, "azure/whisper-1": { "mode": "audio_transcription", - "input_cost_per_second": 0.0001, - "output_cost_per_second": 0.0001, + "input_cost_per_second": 0.0001, + "output_cost_per_second": 0.0001, "litellm_provider": "azure" }, "azure/o3-mini": { @@ -2650,7 +2650,7 @@ "max_output_tokens": 4096, "input_cost_per_token": 0.00001, "output_cost_per_token": 0.00003, - "litellm_provider": "azure", + "litellm_provider": "azure", "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -2662,7 +2662,7 @@ "max_output_tokens": 4096, "input_cost_per_token": 0.00001, "output_cost_per_token": 0.00003, - "litellm_provider": "azure", + "litellm_provider": "azure", "mode": "chat", "supports_vision": true, "supports_tool_choice": true @@ -2818,7 +2818,7 @@ "supports_function_calling": true }, "azure/command-r-plus": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 128000, "max_output_tokens": 4096, "input_cost_per_token": 0.000003, @@ -2858,47 +2858,47 @@ "output_cost_per_token": 0.000000, "litellm_provider": "azure", "mode": "embedding" - }, + }, "azure/standard/1024-x-1024/dall-e-3": { "input_cost_per_pixel": 0.0000000381469, "output_cost_per_token": 0.0, - "litellm_provider": "azure", + "litellm_provider": "azure", "mode": "image_generation" }, "azure/hd/1024-x-1024/dall-e-3": { "input_cost_per_pixel": 0.00000007629, "output_cost_per_token": 0.0, - "litellm_provider": "azure", + "litellm_provider": "azure", "mode": "image_generation" }, "azure/standard/1024-x-1792/dall-e-3": { "input_cost_per_pixel": 0.00000004359, "output_cost_per_token": 0.0, - "litellm_provider": "azure", + "litellm_provider": "azure", "mode": "image_generation" }, "azure/standard/1792-x-1024/dall-e-3": { "input_cost_per_pixel": 0.00000004359, "output_cost_per_token": 0.0, - "litellm_provider": "azure", + "litellm_provider": "azure", "mode": "image_generation" }, "azure/hd/1024-x-1792/dall-e-3": { "input_cost_per_pixel": 0.00000006539, "output_cost_per_token": 0.0, - "litellm_provider": "azure", + "litellm_provider": "azure", "mode": "image_generation" }, "azure/hd/1792-x-1024/dall-e-3": { "input_cost_per_pixel": 0.00000006539, "output_cost_per_token": 0.0, - "litellm_provider": "azure", + "litellm_provider": "azure", "mode": "image_generation" }, "azure/standard/1024-x-1024/dall-e-2": { "input_cost_per_pixel": 0.0, "output_cost_per_token": 0.0, - "litellm_provider": "azure", + "litellm_provider": "azure", "mode": "image_generation" }, "azure_ai/deepseek-r1": { @@ -3014,7 +3014,7 @@ "mode": "chat", "source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/000-000.ministral-3b-2410-offer?tab=Overview", "supports_tool_choice": true - }, + }, "azure_ai/Llama-3.2-11B-Vision-Instruct": { "max_tokens": 2048, "max_input_tokens": 128000, @@ -3303,7 +3303,7 @@ "output_cost_per_token": 0.000002, "litellm_provider": "text-completion-openai", "mode": "completion" - }, + }, "gpt-3.5-turbo-instruct": { "max_tokens": 4096, "max_input_tokens": 8192, @@ -3608,7 +3608,7 @@ "output_cost_per_token": 0.00000219, "litellm_provider": "deepseek", "mode": "chat", - "supports_function_calling": true, + "supports_function_calling": true, "supports_assistant_prefill": true, "supports_tool_choice": true, "supports_reasoning": true, @@ -3625,7 +3625,7 @@ "output_cost_per_token": 0.0000011, "litellm_provider": "deepseek", "mode": "chat", - "supports_function_calling": true, + "supports_function_calling": true, "supports_assistant_prefill": true, "supports_tool_choice": true, "supports_prompt_caching": true @@ -3861,7 +3861,7 @@ "output_cost_per_token": 0.00000028, "litellm_provider": "deepseek", "mode": "chat", - "supports_function_calling": true, + "supports_function_calling": true, "supports_assistant_prefill": true, "supports_tool_choice": true, "supports_prompt_caching": true @@ -3875,7 +3875,7 @@ "litellm_provider": "groq", "mode": "chat", "supports_system_messages": false, - "supports_function_calling": false, + "supports_function_calling": false, "supports_reasoning": true, "supports_response_schema": false, "supports_tool_choice": true @@ -4757,8 +4757,8 @@ "max_output_tokens": 8192, "input_cost_per_image": 0.0025, "input_cost_per_video_per_second": 0.002, - "input_cost_per_token": 0.0000005, - "input_cost_per_character": 0.000000125, + "input_cost_per_token": 0.0000005, + "input_cost_per_character": 0.000000125, "output_cost_per_token": 0.0000015, "output_cost_per_character": 0.000000375, "litellm_provider": "vertex_ai-language-models", @@ -4767,14 +4767,14 @@ "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "supports_tool_choice": true }, - "gemini-1.0-pro": { + "gemini-1.0-pro": { "max_tokens": 8192, "max_input_tokens": 32760, "max_output_tokens": 8192, "input_cost_per_image": 0.0025, "input_cost_per_video_per_second": 0.002, - "input_cost_per_token": 0.0000005, - "input_cost_per_character": 0.000000125, + "input_cost_per_token": 0.0000005, + "input_cost_per_character": 0.000000125, "output_cost_per_token": 0.0000015, "output_cost_per_character": 0.000000375, "litellm_provider": "vertex_ai-language-models", @@ -4783,14 +4783,14 @@ "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#google_models", "supports_tool_choice": true }, - "gemini-1.0-pro-001": { + "gemini-1.0-pro-001": { "max_tokens": 8192, "max_input_tokens": 32760, "max_output_tokens": 8192, "input_cost_per_image": 0.0025, "input_cost_per_video_per_second": 0.002, - "input_cost_per_token": 0.0000005, - "input_cost_per_character": 0.000000125, + "input_cost_per_token": 0.0000005, + "input_cost_per_character": 0.000000125, "output_cost_per_token": 0.0000015, "output_cost_per_character": 0.000000375, "litellm_provider": "vertex_ai-language-models", @@ -4806,8 +4806,8 @@ "max_output_tokens": 2048, "input_cost_per_image": 0.0025, "input_cost_per_video_per_second": 0.002, - "input_cost_per_token": 0.0000005, - "input_cost_per_character": 0.000000125, + "input_cost_per_token": 0.0000005, + "input_cost_per_character": 0.000000125, "output_cost_per_token": 0.0000015, "output_cost_per_character": 0.000000375, "litellm_provider": "vertex_ai-language-models", @@ -4822,8 +4822,8 @@ "max_output_tokens": 2048, "input_cost_per_image": 0.0025, "input_cost_per_video_per_second": 0.002, - "input_cost_per_token": 0.0000005, - "input_cost_per_character": 0.000000125, + "input_cost_per_token": 0.0000005, + "input_cost_per_character": 0.000000125, "output_cost_per_token": 0.0000015, "output_cost_per_character": 0.000000375, "litellm_provider": "vertex_ai-language-models", @@ -4832,14 +4832,14 @@ "source": "As of Jun, 2024. There is no available doc on vertex ai pricing gemini-1.0-ultra-001. Using gemini-1.0-pro pricing. Got max_tokens info here: https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models", "supports_tool_choice": true }, - "gemini-1.0-pro-002": { + "gemini-1.0-pro-002": { "max_tokens": 8192, "max_input_tokens": 32760, "max_output_tokens": 8192, "input_cost_per_image": 0.0025, "input_cost_per_video_per_second": 0.002, - "input_cost_per_token": 0.0000005, - "input_cost_per_character": 0.000000125, + "input_cost_per_token": 0.0000005, + "input_cost_per_character": 0.000000125, "output_cost_per_token": 0.0000015, "output_cost_per_character": 0.000000375, "litellm_provider": "vertex_ai-language-models", @@ -4849,7 +4849,7 @@ "deprecation_date": "2025-04-09", "supports_tool_choice": true }, - "gemini-1.5-pro": { + "gemini-1.5-pro": { "max_tokens": 8192, "max_input_tokens": 2097152, "max_output_tokens": 8192, @@ -4858,10 +4858,10 @@ "input_cost_per_video_per_second": 0.00032875, "input_cost_per_token": 0.00000125, "input_cost_per_character": 0.0000003125, - "input_cost_per_image_above_128k_tokens": 0.0006575, - "input_cost_per_video_per_second_above_128k_tokens": 0.0006575, - "input_cost_per_audio_per_second_above_128k_tokens": 0.0000625, - "input_cost_per_token_above_128k_tokens": 0.0000025, + "input_cost_per_image_above_128k_tokens": 0.0006575, + "input_cost_per_video_per_second_above_128k_tokens": 0.0006575, + "input_cost_per_audio_per_second_above_128k_tokens": 0.0000625, + "input_cost_per_token_above_128k_tokens": 0.0000025, "input_cost_per_character_above_128k_tokens": 0.000000625, "output_cost_per_token": 0.000005, "output_cost_per_character": 0.00000125, @@ -4873,8 +4873,8 @@ "supports_pdf_input": true, "supports_system_messages": true, "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, + "supports_tool_choice": true, + "supports_response_schema": true, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" }, "gemini-1.5-pro-002": { @@ -4886,10 +4886,10 @@ "input_cost_per_video_per_second": 0.00032875, "input_cost_per_token": 0.00000125, "input_cost_per_character": 0.0000003125, - "input_cost_per_image_above_128k_tokens": 0.0006575, - "input_cost_per_video_per_second_above_128k_tokens": 0.0006575, - "input_cost_per_audio_per_second_above_128k_tokens": 0.0000625, - "input_cost_per_token_above_128k_tokens": 0.0000025, + "input_cost_per_image_above_128k_tokens": 0.0006575, + "input_cost_per_video_per_second_above_128k_tokens": 0.0006575, + "input_cost_per_audio_per_second_above_128k_tokens": 0.0000625, + "input_cost_per_token_above_128k_tokens": 0.0000025, "input_cost_per_character_above_128k_tokens": 0.000000625, "output_cost_per_token": 0.000005, "output_cost_per_character": 0.00000125, @@ -4900,12 +4900,12 @@ "supports_vision": true, "supports_system_messages": true, "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, + "supports_tool_choice": true, + "supports_response_schema": true, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-1.5-pro", "deprecation_date": "2025-09-24" }, - "gemini-1.5-pro-001": { + "gemini-1.5-pro-001": { "max_tokens": 8192, "max_input_tokens": 1000000, "max_output_tokens": 8192, @@ -4914,10 +4914,10 @@ "input_cost_per_video_per_second": 0.00032875, "input_cost_per_token": 0.00000125, "input_cost_per_character": 0.0000003125, - "input_cost_per_image_above_128k_tokens": 0.0006575, - "input_cost_per_video_per_second_above_128k_tokens": 0.0006575, - "input_cost_per_audio_per_second_above_128k_tokens": 0.0000625, - "input_cost_per_token_above_128k_tokens": 0.0000025, + "input_cost_per_image_above_128k_tokens": 0.0006575, + "input_cost_per_video_per_second_above_128k_tokens": 0.0006575, + "input_cost_per_audio_per_second_above_128k_tokens": 0.0000625, + "input_cost_per_token_above_128k_tokens": 0.0000025, "input_cost_per_character_above_128k_tokens": 0.000000625, "output_cost_per_token": 0.000005, "output_cost_per_character": 0.00000125, @@ -4928,25 +4928,25 @@ "supports_vision": true, "supports_system_messages": true, "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, + "supports_tool_choice": true, + "supports_response_schema": true, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models", "deprecation_date": "2025-05-24" }, - "gemini-1.5-pro-preview-0514": { + "gemini-1.5-pro-preview-0514": { "max_tokens": 8192, "max_input_tokens": 1000000, "max_output_tokens": 8192, "input_cost_per_image": 0.00032875, "input_cost_per_audio_per_second": 0.00003125, "input_cost_per_video_per_second": 0.00032875, - "input_cost_per_token": 0.000000078125, - "input_cost_per_character": 0.0000003125, - "input_cost_per_image_above_128k_tokens": 0.0006575, - "input_cost_per_video_per_second_above_128k_tokens": 0.0006575, - "input_cost_per_audio_per_second_above_128k_tokens": 0.0000625, - "input_cost_per_token_above_128k_tokens": 0.00000015625, - "input_cost_per_character_above_128k_tokens": 0.000000625, + "input_cost_per_token": 0.000000078125, + "input_cost_per_character": 0.0000003125, + "input_cost_per_image_above_128k_tokens": 0.0006575, + "input_cost_per_video_per_second_above_128k_tokens": 0.0006575, + "input_cost_per_audio_per_second_above_128k_tokens": 0.0000625, + "input_cost_per_token_above_128k_tokens": 0.00000015625, + "input_cost_per_character_above_128k_tokens": 0.000000625, "output_cost_per_token": 0.0000003125, "output_cost_per_character": 0.00000125, "output_cost_per_token_above_128k_tokens": 0.000000625, @@ -4955,24 +4955,24 @@ "mode": "chat", "supports_system_messages": true, "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, + "supports_tool_choice": true, + "supports_response_schema": true, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" }, - "gemini-1.5-pro-preview-0215": { + "gemini-1.5-pro-preview-0215": { "max_tokens": 8192, "max_input_tokens": 1000000, "max_output_tokens": 8192, "input_cost_per_image": 0.00032875, "input_cost_per_audio_per_second": 0.00003125, "input_cost_per_video_per_second": 0.00032875, - "input_cost_per_token": 0.000000078125, - "input_cost_per_character": 0.0000003125, - "input_cost_per_image_above_128k_tokens": 0.0006575, - "input_cost_per_video_per_second_above_128k_tokens": 0.0006575, - "input_cost_per_audio_per_second_above_128k_tokens": 0.0000625, - "input_cost_per_token_above_128k_tokens": 0.00000015625, - "input_cost_per_character_above_128k_tokens": 0.000000625, + "input_cost_per_token": 0.000000078125, + "input_cost_per_character": 0.0000003125, + "input_cost_per_image_above_128k_tokens": 0.0006575, + "input_cost_per_video_per_second_above_128k_tokens": 0.0006575, + "input_cost_per_audio_per_second_above_128k_tokens": 0.0000625, + "input_cost_per_token_above_128k_tokens": 0.00000015625, + "input_cost_per_character_above_128k_tokens": 0.000000625, "output_cost_per_token": 0.0000003125, "output_cost_per_character": 0.00000125, "output_cost_per_token_above_128k_tokens": 0.000000625, @@ -4981,8 +4981,8 @@ "mode": "chat", "supports_system_messages": true, "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, + "supports_tool_choice": true, + "supports_response_schema": true, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" }, "gemini-1.5-pro-preview-0409": { @@ -4992,13 +4992,13 @@ "input_cost_per_image": 0.00032875, "input_cost_per_audio_per_second": 0.00003125, "input_cost_per_video_per_second": 0.00032875, - "input_cost_per_token": 0.000000078125, - "input_cost_per_character": 0.0000003125, - "input_cost_per_image_above_128k_tokens": 0.0006575, - "input_cost_per_video_per_second_above_128k_tokens": 0.0006575, - "input_cost_per_audio_per_second_above_128k_tokens": 0.0000625, - "input_cost_per_token_above_128k_tokens": 0.00000015625, - "input_cost_per_character_above_128k_tokens": 0.000000625, + "input_cost_per_token": 0.000000078125, + "input_cost_per_character": 0.0000003125, + "input_cost_per_image_above_128k_tokens": 0.0006575, + "input_cost_per_video_per_second_above_128k_tokens": 0.0006575, + "input_cost_per_audio_per_second_above_128k_tokens": 0.0000625, + "input_cost_per_token_above_128k_tokens": 0.00000015625, + "input_cost_per_character_above_128k_tokens": 0.000000625, "output_cost_per_token": 0.0000003125, "output_cost_per_character": 0.00000125, "output_cost_per_token_above_128k_tokens": 0.000000625, @@ -5007,7 +5007,7 @@ "mode": "chat", "supports_function_calling": true, "supports_tool_choice": true, - "supports_response_schema": true, + "supports_response_schema": true, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" }, "gemini-1.5-flash": { @@ -5024,9 +5024,9 @@ "input_cost_per_video_per_second": 0.00002, "input_cost_per_audio_per_second": 0.000002, "input_cost_per_token": 0.000000075, - "input_cost_per_character": 0.00000001875, - "input_cost_per_token_above_128k_tokens": 0.000001, - "input_cost_per_character_above_128k_tokens": 0.00000025, + "input_cost_per_character": 0.00000001875, + "input_cost_per_token_above_128k_tokens": 0.000001, + "input_cost_per_character_above_128k_tokens": 0.00000025, "input_cost_per_image_above_128k_tokens": 0.00004, "input_cost_per_video_per_second_above_128k_tokens": 0.00004, "input_cost_per_audio_per_second_above_128k_tokens": 0.000004, @@ -5056,10 +5056,10 @@ "input_cost_per_image": 0.00002, "input_cost_per_video_per_second": 0.00002, "input_cost_per_audio_per_second": 0.000002, - "input_cost_per_token": 0.000000004688, - "input_cost_per_character": 0.00000001875, - "input_cost_per_token_above_128k_tokens": 0.000001, - "input_cost_per_character_above_128k_tokens": 0.00000025, + "input_cost_per_token": 0.000000004688, + "input_cost_per_character": 0.00000001875, + "input_cost_per_token_above_128k_tokens": 0.000001, + "input_cost_per_character_above_128k_tokens": 0.00000025, "input_cost_per_image_above_128k_tokens": 0.00004, "input_cost_per_video_per_second_above_128k_tokens": 0.00004, "input_cost_per_audio_per_second_above_128k_tokens": 0.000004, @@ -5090,9 +5090,9 @@ "input_cost_per_video_per_second": 0.00002, "input_cost_per_audio_per_second": 0.000002, "input_cost_per_token": 0.000000075, - "input_cost_per_character": 0.00000001875, - "input_cost_per_token_above_128k_tokens": 0.000001, - "input_cost_per_character_above_128k_tokens": 0.00000025, + "input_cost_per_character": 0.00000001875, + "input_cost_per_token_above_128k_tokens": 0.000001, + "input_cost_per_character_above_128k_tokens": 0.00000025, "input_cost_per_image_above_128k_tokens": 0.00004, "input_cost_per_video_per_second_above_128k_tokens": 0.00004, "input_cost_per_audio_per_second_above_128k_tokens": 0.000004, @@ -5124,9 +5124,9 @@ "input_cost_per_video_per_second": 0.00002, "input_cost_per_audio_per_second": 0.000002, "input_cost_per_token": 0.000000075, - "input_cost_per_character": 0.00000001875, - "input_cost_per_token_above_128k_tokens": 0.000001, - "input_cost_per_character_above_128k_tokens": 0.00000025, + "input_cost_per_character": 0.00000001875, + "input_cost_per_token_above_128k_tokens": 0.000001, + "input_cost_per_character_above_128k_tokens": 0.00000025, "input_cost_per_image_above_128k_tokens": 0.00004, "input_cost_per_video_per_second_above_128k_tokens": 0.00004, "input_cost_per_audio_per_second_above_128k_tokens": 0.000004, @@ -5158,9 +5158,9 @@ "input_cost_per_video_per_second": 0.00002, "input_cost_per_audio_per_second": 0.000002, "input_cost_per_token": 0.000000075, - "input_cost_per_character": 0.00000001875, - "input_cost_per_token_above_128k_tokens": 0.000001, - "input_cost_per_character_above_128k_tokens": 0.00000025, + "input_cost_per_character": 0.00000001875, + "input_cost_per_token_above_128k_tokens": 0.000001, + "input_cost_per_character_above_128k_tokens": 0.00000025, "input_cost_per_image_above_128k_tokens": 0.00004, "input_cost_per_video_per_second_above_128k_tokens": 0.00004, "input_cost_per_audio_per_second_above_128k_tokens": 0.000004, @@ -5187,7 +5187,7 @@ "litellm_provider": "vertex_ai-language-models", "mode": "chat", "supports_function_calling": false, - "supports_tool_choice": true, + "supports_tool_choice": true, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/gemini-experimental" }, "gemini-flash-experimental": { @@ -5201,7 +5201,7 @@ "litellm_provider": "vertex_ai-language-models", "mode": "chat", "supports_function_calling": false, - "supports_tool_choice": true, + "supports_tool_choice": true, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/gemini-experimental" }, "gemini-pro-vision": { @@ -5211,7 +5211,7 @@ "max_images_per_prompt": 16, "max_videos_per_prompt": 1, "max_video_length": 2, - "input_cost_per_token": 0.0000005, + "input_cost_per_token": 0.0000005, "output_cost_per_token": 0.0000015, "input_cost_per_image": 0.0025, "litellm_provider": "vertex_ai-vision-models", @@ -5228,7 +5228,7 @@ "max_images_per_prompt": 16, "max_videos_per_prompt": 1, "max_video_length": 2, - "input_cost_per_token": 0.0000005, + "input_cost_per_token": 0.0000005, "output_cost_per_token": 0.0000015, "input_cost_per_image": 0.0025, "litellm_provider": "vertex_ai-vision-models", @@ -5245,7 +5245,7 @@ "max_images_per_prompt": 16, "max_videos_per_prompt": 1, "max_video_length": 2, - "input_cost_per_token": 0.0000005, + "input_cost_per_token": 0.0000005, "output_cost_per_token": 0.0000015, "input_cost_per_image": 0.0025, "litellm_provider": "vertex_ai-vision-models", @@ -5350,9 +5350,9 @@ "input_cost_per_video_per_second": 0, "input_cost_per_audio_per_second": 0, "input_cost_per_token": 0, - "input_cost_per_character": 0, - "input_cost_per_token_above_128k_tokens": 0, - "input_cost_per_character_above_128k_tokens": 0, + "input_cost_per_character": 0, + "input_cost_per_token_above_128k_tokens": 0, + "input_cost_per_character_above_128k_tokens": 0, "input_cost_per_image_above_128k_tokens": 0, "input_cost_per_video_per_second_above_128k_tokens": 0, "input_cost_per_audio_per_second_above_128k_tokens": 0, @@ -5411,9 +5411,9 @@ "input_cost_per_video_per_second": 0, "input_cost_per_audio_per_second": 0, "input_cost_per_token": 0, - "input_cost_per_character": 0, - "input_cost_per_token_above_128k_tokens": 0, - "input_cost_per_character_above_128k_tokens": 0, + "input_cost_per_character": 0, + "input_cost_per_token_above_128k_tokens": 0, + "input_cost_per_character_above_128k_tokens": 0, "input_cost_per_image_above_128k_tokens": 0, "input_cost_per_video_per_second_above_128k_tokens": 0, "input_cost_per_audio_per_second_above_128k_tokens": 0, @@ -5447,9 +5447,9 @@ "input_cost_per_video_per_second": 0, "input_cost_per_audio_per_second": 0, "input_cost_per_token": 0, - "input_cost_per_character": 0, - "input_cost_per_token_above_128k_tokens": 0, - "input_cost_per_character_above_128k_tokens": 0, + "input_cost_per_character": 0, + "input_cost_per_token_above_128k_tokens": 0, + "input_cost_per_character_above_128k_tokens": 0, "input_cost_per_image_above_128k_tokens": 0, "input_cost_per_video_per_second_above_128k_tokens": 0, "input_cost_per_audio_per_second_above_128k_tokens": 0, @@ -5615,7 +5615,7 @@ "input_cost_per_token": 0.00000125, "input_cost_per_token_above_200k_tokens": 0.0000025, "output_cost_per_token": 0.00001, - "output_cost_per_token_above_200k_tokens": 0.000015, + "output_cost_per_token_above_200k_tokens": 0.000015, "litellm_provider": "vertex_ai-language-models", "mode": "chat", "supports_reasoning": true, @@ -5644,9 +5644,9 @@ "input_cost_per_video_per_second": 0, "input_cost_per_audio_per_second": 0, "input_cost_per_token": 0, - "input_cost_per_character": 0, - "input_cost_per_token_above_128k_tokens": 0, - "input_cost_per_character_above_128k_tokens": 0, + "input_cost_per_character": 0, + "input_cost_per_token_above_128k_tokens": 0, + "input_cost_per_character_above_128k_tokens": 0, "input_cost_per_image_above_128k_tokens": 0, "input_cost_per_video_per_second_above_128k_tokens": 0, "input_cost_per_audio_per_second_above_128k_tokens": 0, @@ -5761,9 +5761,9 @@ "max_pdf_size_mb": 30, "input_cost_per_audio_token": 0.0000007, "input_cost_per_token": 0.00000125, - "input_cost_per_token_above_200k_tokens": 0.0000025, + "input_cost_per_token_above_200k_tokens": 0.0000025, "output_cost_per_token": 0.00001, - "output_cost_per_token_above_200k_tokens": 0.000015, + "output_cost_per_token_above_200k_tokens": 0.000015, "litellm_provider": "gemini", "mode": "chat", "rpm": 10000, @@ -5792,9 +5792,9 @@ "input_cost_per_video_per_second": 0, "input_cost_per_audio_per_second": 0, "input_cost_per_token": 0, - "input_cost_per_character": 0, - "input_cost_per_token_above_128k_tokens": 0, - "input_cost_per_character_above_128k_tokens": 0, + "input_cost_per_character": 0, + "input_cost_per_token_above_128k_tokens": 0, + "input_cost_per_character_above_128k_tokens": 0, "input_cost_per_image_above_128k_tokens": 0, "input_cost_per_video_per_second_above_128k_tokens": 0, "input_cost_per_audio_per_second_above_128k_tokens": 0, @@ -5857,9 +5857,9 @@ "input_cost_per_video_per_second": 0, "input_cost_per_audio_per_second": 0, "input_cost_per_token": 0, - "input_cost_per_character": 0, - "input_cost_per_token_above_128k_tokens": 0, - "input_cost_per_character_above_128k_tokens": 0, + "input_cost_per_character": 0, + "input_cost_per_token_above_128k_tokens": 0, + "input_cost_per_character_above_128k_tokens": 0, "input_cost_per_image_above_128k_tokens": 0, "input_cost_per_video_per_second_above_128k_tokens": 0, "input_cost_per_audio_per_second_above_128k_tokens": 0, @@ -5895,9 +5895,9 @@ "input_cost_per_video_per_second": 0, "input_cost_per_audio_per_second": 0, "input_cost_per_token": 0, - "input_cost_per_character": 0, - "input_cost_per_token_above_128k_tokens": 0, - "input_cost_per_character_above_128k_tokens": 0, + "input_cost_per_character": 0, + "input_cost_per_token_above_128k_tokens": 0, + "input_cost_per_character_above_128k_tokens": 0, "input_cost_per_image_above_128k_tokens": 0, "input_cost_per_video_per_second_above_128k_tokens": 0, "input_cost_per_audio_per_second_above_128k_tokens": 0, @@ -5927,9 +5927,9 @@ "input_cost_per_video_per_second": 0, "input_cost_per_audio_per_second": 0, "input_cost_per_token": 0, - "input_cost_per_character": 0, - "input_cost_per_token_above_128k_tokens": 0, - "input_cost_per_character_above_128k_tokens": 0, + "input_cost_per_character": 0, + "input_cost_per_token_above_128k_tokens": 0, + "input_cost_per_character_above_128k_tokens": 0, "input_cost_per_image_above_128k_tokens": 0, "input_cost_per_video_per_second_above_128k_tokens": 0, "input_cost_per_audio_per_second_above_128k_tokens": 0, @@ -5955,9 +5955,9 @@ "input_cost_per_video_per_second": 0, "input_cost_per_audio_per_second": 0, "input_cost_per_token": 0, - "input_cost_per_character": 0, - "input_cost_per_token_above_128k_tokens": 0, - "input_cost_per_character_above_128k_tokens": 0, + "input_cost_per_character": 0, + "input_cost_per_token_above_128k_tokens": 0, + "input_cost_per_character_above_128k_tokens": 0, "input_cost_per_image_above_128k_tokens": 0, "input_cost_per_video_per_second_above_128k_tokens": 0, "input_cost_per_audio_per_second_above_128k_tokens": 0, @@ -6079,7 +6079,7 @@ "supports_tool_choice": true }, "vertex_ai/claude-3-haiku": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 200000, "max_output_tokens": 4096, "input_cost_per_token": 0.00000025, @@ -6092,7 +6092,7 @@ "supports_tool_choice": true }, "vertex_ai/claude-3-haiku@20240307": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 200000, "max_output_tokens": 4096, "input_cost_per_token": 0.00000025, @@ -6621,7 +6621,7 @@ "max_video_length": 1, "max_audio_length_hours": 8.4, "max_audio_per_prompt": 1, - "max_pdf_size_mb": 30, + "max_pdf_size_mb": 30, "cache_read_input_token_cost": 0.00000001875, "cache_creation_input_token_cost": 0.000001, "input_cost_per_token": 0.000000075, @@ -6650,7 +6650,7 @@ "max_video_length": 1, "max_audio_length_hours": 8.4, "max_audio_per_prompt": 1, - "max_pdf_size_mb": 30, + "max_pdf_size_mb": 30, "cache_read_input_token_cost": 0.00000001875, "cache_creation_input_token_cost": 0.000001, "input_cost_per_token": 0.000000075, @@ -6679,7 +6679,7 @@ "max_video_length": 1, "max_audio_length_hours": 8.4, "max_audio_per_prompt": 1, - "max_pdf_size_mb": 30, + "max_pdf_size_mb": 30, "input_cost_per_token": 0.000000075, "input_cost_per_token_above_128k_tokens": 0.00000015, "output_cost_per_token": 0.0000003, @@ -6689,7 +6689,7 @@ "supports_system_messages": true, "supports_function_calling": true, "supports_vision": true, - "supports_response_schema": true, + "supports_response_schema": true, "tpm": 4000000, "rpm": 2000, "source": "https://ai.google.dev/pricing", @@ -6704,7 +6704,7 @@ "max_video_length": 1, "max_audio_length_hours": 8.4, "max_audio_per_prompt": 1, - "max_pdf_size_mb": 30, + "max_pdf_size_mb": 30, "input_cost_per_token": 0.000000075, "input_cost_per_token_above_128k_tokens": 0.00000015, "output_cost_per_token": 0.0000003, @@ -6730,7 +6730,7 @@ "max_video_length": 1, "max_audio_length_hours": 8.4, "max_audio_per_prompt": 1, - "max_pdf_size_mb": 30, + "max_pdf_size_mb": 30, "input_cost_per_token": 0, "input_cost_per_token_above_128k_tokens": 0, "output_cost_per_token": 0, @@ -6756,7 +6756,7 @@ "max_video_length": 1, "max_audio_length_hours": 8.4, "max_audio_per_prompt": 1, - "max_pdf_size_mb": 30, + "max_pdf_size_mb": 30, "input_cost_per_token": 0, "input_cost_per_token_above_128k_tokens": 0, "output_cost_per_token": 0, @@ -6782,7 +6782,7 @@ "max_video_length": 1, "max_audio_length_hours": 8.4, "max_audio_per_prompt": 1, - "max_pdf_size_mb": 30, + "max_pdf_size_mb": 30, "input_cost_per_token": 0, "input_cost_per_token_above_128k_tokens": 0, "output_cost_per_token": 0, @@ -6811,7 +6811,7 @@ "max_video_length": 1, "max_audio_length_hours": 8.4, "max_audio_per_prompt": 1, - "max_pdf_size_mb": 30, + "max_pdf_size_mb": 30, "input_cost_per_token": 0, "input_cost_per_token_above_128k_tokens": 0, "output_cost_per_token": 0, @@ -6840,7 +6840,7 @@ "max_video_length": 1, "max_audio_length_hours": 8.4, "max_audio_per_prompt": 1, - "max_pdf_size_mb": 30, + "max_pdf_size_mb": 30, "input_cost_per_token": 0, "input_cost_per_token_above_128k_tokens": 0, "output_cost_per_token": 0, @@ -6865,7 +6865,7 @@ "max_video_length": 1, "max_audio_length_hours": 8.4, "max_audio_per_prompt": 1, - "max_pdf_size_mb": 30, + "max_pdf_size_mb": 30, "input_cost_per_token": 0, "input_cost_per_token_above_128k_tokens": 0, "output_cost_per_token": 0, @@ -6885,10 +6885,10 @@ "max_tokens": 8192, "max_input_tokens": 32760, "max_output_tokens": 8192, - "input_cost_per_token": 0.00000035, - "input_cost_per_token_above_128k_tokens": 0.0000007, - "output_cost_per_token": 0.00000105, - "output_cost_per_token_above_128k_tokens": 0.0000021, + "input_cost_per_token": 0.00000035, + "input_cost_per_token_above_128k_tokens": 0.0000007, + "output_cost_per_token": 0.00000105, + "output_cost_per_token_above_128k_tokens": 0.0000021, "litellm_provider": "gemini", "mode": "chat", "supports_function_calling": true, @@ -6902,17 +6902,17 @@ "max_tokens": 8192, "max_input_tokens": 2097152, "max_output_tokens": 8192, - "input_cost_per_token": 0.0000035, - "input_cost_per_token_above_128k_tokens": 0.000007, - "output_cost_per_token": 0.0000105, - "output_cost_per_token_above_128k_tokens": 0.000021, + "input_cost_per_token": 0.0000035, + "input_cost_per_token_above_128k_tokens": 0.000007, + "output_cost_per_token": 0.0000105, + "output_cost_per_token_above_128k_tokens": 0.000021, "litellm_provider": "gemini", "mode": "chat", "supports_system_messages": true, "supports_function_calling": true, "supports_vision": true, - "supports_tool_choice": true, - "supports_response_schema": true, + "supports_tool_choice": true, + "supports_response_schema": true, "tpm": 4000000, "rpm": 1000, "source": "https://ai.google.dev/pricing" @@ -6921,17 +6921,17 @@ "max_tokens": 8192, "max_input_tokens": 2097152, "max_output_tokens": 8192, - "input_cost_per_token": 0.0000035, - "input_cost_per_token_above_128k_tokens": 0.000007, - "output_cost_per_token": 0.0000105, - "output_cost_per_token_above_128k_tokens": 0.000021, + "input_cost_per_token": 0.0000035, + "input_cost_per_token_above_128k_tokens": 0.000007, + "output_cost_per_token": 0.0000105, + "output_cost_per_token_above_128k_tokens": 0.000021, "litellm_provider": "gemini", "mode": "chat", "supports_system_messages": true, "supports_function_calling": true, "supports_vision": true, - "supports_tool_choice": true, - "supports_response_schema": true, + "supports_tool_choice": true, + "supports_response_schema": true, "supports_prompt_caching": true, "tpm": 4000000, "rpm": 1000, @@ -6942,17 +6942,17 @@ "max_tokens": 8192, "max_input_tokens": 2097152, "max_output_tokens": 8192, - "input_cost_per_token": 0.0000035, - "input_cost_per_token_above_128k_tokens": 0.000007, - "output_cost_per_token": 0.0000105, - "output_cost_per_token_above_128k_tokens": 0.000021, + "input_cost_per_token": 0.0000035, + "input_cost_per_token_above_128k_tokens": 0.000007, + "output_cost_per_token": 0.0000105, + "output_cost_per_token_above_128k_tokens": 0.000021, "litellm_provider": "gemini", "mode": "chat", "supports_system_messages": true, "supports_function_calling": true, "supports_vision": true, - "supports_tool_choice": true, - "supports_response_schema": true, + "supports_tool_choice": true, + "supports_response_schema": true, "supports_prompt_caching": true, "tpm": 4000000, "rpm": 1000, @@ -7001,17 +7001,17 @@ "max_tokens": 8192, "max_input_tokens": 1048576, "max_output_tokens": 8192, - "input_cost_per_token": 0.0000035, - "input_cost_per_token_above_128k_tokens": 0.000007, - "output_cost_per_token": 0.00000105, - "output_cost_per_token_above_128k_tokens": 0.000021, + "input_cost_per_token": 0.0000035, + "input_cost_per_token_above_128k_tokens": 0.000007, + "output_cost_per_token": 0.00000105, + "output_cost_per_token_above_128k_tokens": 0.000021, "litellm_provider": "gemini", "mode": "chat", "supports_system_messages": true, "supports_function_calling": true, "supports_vision": true, - "supports_tool_choice": true, - "supports_response_schema": true, + "supports_tool_choice": true, + "supports_response_schema": true, "tpm": 4000000, "rpm": 1000, "source": "https://ai.google.dev/pricing" @@ -7020,10 +7020,10 @@ "max_tokens": 2048, "max_input_tokens": 30720, "max_output_tokens": 2048, - "input_cost_per_token": 0.00000035, - "input_cost_per_token_above_128k_tokens": 0.0000007, - "output_cost_per_token": 0.00000105, - "output_cost_per_token_above_128k_tokens": 0.0000021, + "input_cost_per_token": 0.00000035, + "input_cost_per_token_above_128k_tokens": 0.0000007, + "output_cost_per_token": 0.00000105, + "output_cost_per_token_above_128k_tokens": 0.0000021, "litellm_provider": "gemini", "mode": "chat", "supports_function_calling": true, @@ -7037,8 +7037,8 @@ "gemini/gemini-gemma-2-27b-it": { "max_tokens": 8192, "max_output_tokens": 8192, - "input_cost_per_token": 0.00000035, - "output_cost_per_token": 0.00000105, + "input_cost_per_token": 0.00000035, + "output_cost_per_token": 0.00000105, "litellm_provider": "gemini", "mode": "chat", "supports_function_calling": true, @@ -7049,8 +7049,8 @@ "gemini/gemini-gemma-2-9b-it": { "max_tokens": 8192, "max_output_tokens": 8192, - "input_cost_per_token": 0.00000035, - "output_cost_per_token": 0.00000105, + "input_cost_per_token": 0.00000035, + "output_cost_per_token": 0.00000105, "litellm_provider": "gemini", "mode": "chat", "supports_function_calling": true, @@ -7125,7 +7125,7 @@ "supports_tool_choice": true }, "command-nightly": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 0.000001, @@ -7134,7 +7134,7 @@ "mode": "completion" }, "command": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 0.000001, @@ -7198,7 +7198,7 @@ "mode": "rerank" }, "embed-english-light-v3.0": { - "max_tokens": 1024, + "max_tokens": 1024, "max_input_tokens": 1024, "input_cost_per_token": 0.00000010, "output_cost_per_token": 0.00000, @@ -7206,7 +7206,7 @@ "mode": "embedding" }, "embed-multilingual-v3.0": { - "max_tokens": 1024, + "max_tokens": 1024, "max_input_tokens": 1024, "input_cost_per_token": 0.00000010, "output_cost_per_token": 0.00000, @@ -7215,7 +7215,7 @@ "mode": "embedding" }, "embed-english-v2.0": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 4096, "input_cost_per_token": 0.00000010, "output_cost_per_token": 0.00000, @@ -7223,7 +7223,7 @@ "mode": "embedding" }, "embed-english-light-v2.0": { - "max_tokens": 1024, + "max_tokens": 1024, "max_input_tokens": 1024, "input_cost_per_token": 0.00000010, "output_cost_per_token": 0.00000, @@ -7231,7 +7231,7 @@ "mode": "embedding" }, "embed-multilingual-v2.0": { - "max_tokens": 768, + "max_tokens": 768, "max_input_tokens": 768, "input_cost_per_token": 0.00000010, "output_cost_per_token": 0.00000, @@ -7239,7 +7239,7 @@ "mode": "embedding" }, "embed-english-v3.0": { - "max_tokens": 1024, + "max_tokens": 1024, "max_input_tokens": 1024, "input_cost_per_token": 0.00000010, "input_cost_per_image": 0.0001, @@ -7391,7 +7391,7 @@ "output_cost_per_token": 0.00000219, "litellm_provider": "openrouter", "mode": "chat", - "supports_function_calling": true, + "supports_function_calling": true, "supports_assistant_prefill": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -7433,7 +7433,7 @@ "max_output_tokens": 8192, "input_cost_per_token": 0.0000025, "output_cost_per_token": 0.0000075, - "input_cost_per_image": 0.00265, + "input_cost_per_image": 0.00265, "litellm_provider": "openrouter", "mode": "chat", "supports_function_calling": true, @@ -7490,7 +7490,7 @@ "max_tokens": 200000, "input_cost_per_token": 0.00000025, "output_cost_per_token": 0.00000125, - "input_cost_per_image": 0.0004, + "input_cost_per_image": 0.0004, "litellm_provider": "openrouter", "mode": "chat", "supports_function_calling": true, @@ -7593,7 +7593,7 @@ "max_tokens": 200000, "input_cost_per_token": 0.000003, "output_cost_per_token": 0.000015, - "input_cost_per_image": 0.0048, + "input_cost_per_image": 0.0048, "litellm_provider": "openrouter", "mode": "chat", "supports_function_calling": true, @@ -7628,7 +7628,7 @@ "max_tokens": 45875, "input_cost_per_token": 0.000000125, "output_cost_per_token": 0.000000375, - "input_cost_per_image": 0.0025, + "input_cost_per_image": 0.0025, "litellm_provider": "openrouter", "mode": "chat", "supports_function_calling": true, @@ -7802,7 +7802,7 @@ "max_tokens": 130000, "input_cost_per_token": 0.00001, "output_cost_per_token": 0.00003, - "input_cost_per_image": 0.01445, + "input_cost_per_image": 0.01445, "litellm_provider": "openrouter", "mode": "chat", "supports_function_calling": true, @@ -8094,60 +8094,60 @@ "mode": "chat" }, "luminous-base": { - "max_tokens": 2048, + "max_tokens": 2048, "input_cost_per_token": 0.00003, "output_cost_per_token": 0.000033, "litellm_provider": "aleph_alpha", "mode": "completion" }, "luminous-base-control": { - "max_tokens": 2048, + "max_tokens": 2048, "input_cost_per_token": 0.0000375, "output_cost_per_token": 0.00004125, "litellm_provider": "aleph_alpha", "mode": "chat" }, "luminous-extended": { - "max_tokens": 2048, + "max_tokens": 2048, "input_cost_per_token": 0.000045, "output_cost_per_token": 0.0000495, "litellm_provider": "aleph_alpha", "mode": "completion" }, "luminous-extended-control": { - "max_tokens": 2048, + "max_tokens": 2048, "input_cost_per_token": 0.00005625, "output_cost_per_token": 0.000061875, "litellm_provider": "aleph_alpha", "mode": "chat" }, "luminous-supreme": { - "max_tokens": 2048, + "max_tokens": 2048, "input_cost_per_token": 0.000175, "output_cost_per_token": 0.0001925, "litellm_provider": "aleph_alpha", "mode": "completion" }, "luminous-supreme-control": { - "max_tokens": 2048, + "max_tokens": 2048, "input_cost_per_token": 0.00021875, "output_cost_per_token": 0.000240625, "litellm_provider": "aleph_alpha", "mode": "chat" }, "ai21.j2-mid-v1": { - "max_tokens": 8191, - "max_input_tokens": 8191, - "max_output_tokens": 8191, + "max_tokens": 8191, + "max_input_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_token": 0.0000125, "output_cost_per_token": 0.0000125, "litellm_provider": "bedrock", "mode": "chat" }, "ai21.j2-ultra-v1": { - "max_tokens": 8191, - "max_input_tokens": 8191, - "max_output_tokens": 8191, + "max_tokens": 8191, + "max_input_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_token": 0.0000188, "output_cost_per_token": 0.0000188, "litellm_provider": "bedrock", @@ -8195,58 +8195,58 @@ "mode": "rerank" }, "amazon.titan-text-lite-v1": { - "max_tokens": 4000, + "max_tokens": 4000, "max_input_tokens": 42000, - "max_output_tokens": 4000, + "max_output_tokens": 4000, "input_cost_per_token": 0.0000003, "output_cost_per_token": 0.0000004, "litellm_provider": "bedrock", "mode": "chat" }, "amazon.titan-text-express-v1": { - "max_tokens": 8000, + "max_tokens": 8000, "max_input_tokens": 42000, - "max_output_tokens": 8000, + "max_output_tokens": 8000, "input_cost_per_token": 0.0000013, "output_cost_per_token": 0.0000017, "litellm_provider": "bedrock", "mode": "chat" }, "amazon.titan-text-premier-v1:0": { - "max_tokens": 32000, + "max_tokens": 32000, "max_input_tokens": 42000, - "max_output_tokens": 32000, + "max_output_tokens": 32000, "input_cost_per_token": 0.0000005, "output_cost_per_token": 0.0000015, "litellm_provider": "bedrock", "mode": "chat" }, "amazon.titan-embed-text-v1": { - "max_tokens": 8192, - "max_input_tokens": 8192, + "max_tokens": 8192, + "max_input_tokens": 8192, "output_vector_size": 1536, "input_cost_per_token": 0.0000001, "output_cost_per_token": 0.0, - "litellm_provider": "bedrock", + "litellm_provider": "bedrock", "mode": "embedding" }, "amazon.titan-embed-text-v2:0": { - "max_tokens": 8192, - "max_input_tokens": 8192, + "max_tokens": 8192, + "max_input_tokens": 8192, "output_vector_size": 1024, "input_cost_per_token": 0.0000002, "output_cost_per_token": 0.0, - "litellm_provider": "bedrock", + "litellm_provider": "bedrock", "mode": "embedding" }, "amazon.titan-embed-image-v1": { - "max_tokens": 128, - "max_input_tokens": 128, + "max_tokens": 128, + "max_input_tokens": 128, "output_vector_size": 1024, "input_cost_per_token": 0.0000008, "input_cost_per_image": 0.00006, "output_cost_per_token": 0.0, - "litellm_provider": "bedrock", + "litellm_provider": "bedrock", "supports_image_input": true, "supports_embedding_image_input": true, "mode": "embedding", @@ -8402,7 +8402,7 @@ "supports_tool_choice": true }, "amazon.nova-micro-v1:0": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 300000, "max_output_tokens": 4096, "input_cost_per_token": 0.000000035, @@ -8426,7 +8426,7 @@ "supports_response_schema": true }, "eu.amazon.nova-micro-v1:0": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 300000, "max_output_tokens": 4096, "input_cost_per_token": 0.000000046, @@ -8438,7 +8438,7 @@ "supports_response_schema": true }, "amazon.nova-lite-v1:0": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 128000, "max_output_tokens": 4096, "input_cost_per_token": 0.00000006, @@ -8466,7 +8466,7 @@ "supports_response_schema": true }, "eu.amazon.nova-lite-v1:0": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 128000, "max_output_tokens": 4096, "input_cost_per_token": 0.000000078, @@ -8480,7 +8480,7 @@ "supports_response_schema": true }, "amazon.nova-pro-v1:0": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 300000, "max_output_tokens": 4096, "input_cost_per_token": 0.0000008, @@ -8514,7 +8514,7 @@ "mode": "image_generation" }, "eu.amazon.nova-pro-v1:0": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 300000, "max_output_tokens": 4096, "input_cost_per_token": 0.00000105, @@ -8529,7 +8529,7 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "anthropic.claude-3-sonnet-20240229-v1:0": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 200000, "max_output_tokens": 4096, "input_cost_per_token": 0.000003, @@ -8543,7 +8543,7 @@ "supports_tool_choice": true }, "bedrock/invoke/anthropic.claude-3-5-sonnet-20240620-v1:0": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 200000, "max_output_tokens": 4096, "input_cost_per_token": 0.000003, @@ -8559,7 +8559,7 @@ } }, "anthropic.claude-3-5-sonnet-20240620-v1:0": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 200000, "max_output_tokens": 4096, "input_cost_per_token": 0.000003, @@ -8583,7 +8583,7 @@ "supports_function_calling": true, "supports_vision": true, "supports_assistant_prefill": true, - "supports_prompt_caching": true, + "supports_prompt_caching": true, "supports_response_schema": true, "supports_reasoning": true, "supports_tool_choice": true @@ -8600,12 +8600,12 @@ "supports_vision": true, "supports_pdf_input": true, "supports_assistant_prefill": true, - "supports_prompt_caching": true, + "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true }, "anthropic.claude-3-haiku-20240307-v1:0": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 200000, "max_output_tokens": 4096, "input_cost_per_token": 0.00000025, @@ -8701,7 +8701,7 @@ "supports_function_calling": true, "supports_vision": true, "supports_assistant_prefill": true, - "supports_prompt_caching": true, + "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_reasoning": true @@ -8835,7 +8835,7 @@ "supports_tool_choice": true }, "anthropic.claude-v1": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_token": 0.000008, @@ -8844,7 +8844,7 @@ "mode": "chat" }, "bedrock/us-east-1/anthropic.claude-v1": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_token": 0.000008, @@ -8854,7 +8854,7 @@ "supports_tool_choice": true }, "bedrock/us-west-2/anthropic.claude-v1": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_token": 0.000008, @@ -8864,7 +8864,7 @@ "supports_tool_choice": true }, "bedrock/ap-northeast-1/anthropic.claude-v1": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_token": 0.000008, @@ -8874,7 +8874,7 @@ "supports_tool_choice": true }, "bedrock/ap-northeast-1/1-month-commitment/anthropic.claude-v1": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.0455, @@ -8883,7 +8883,7 @@ "mode": "chat" }, "bedrock/ap-northeast-1/6-month-commitment/anthropic.claude-v1": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.02527, @@ -8892,72 +8892,72 @@ "mode": "chat" }, "bedrock/eu-central-1/anthropic.claude-v1": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_token": 0.000008, "output_cost_per_token": 0.000024, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/eu-central-1/1-month-commitment/anthropic.claude-v1": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_second": 0.0415, "output_cost_per_second": 0.0415, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/eu-central-1/6-month-commitment/anthropic.claude-v1": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_second": 0.02305, "output_cost_per_second": 0.02305, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/us-east-1/1-month-commitment/anthropic.claude-v1": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_second": 0.0175, "output_cost_per_second": 0.0175, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/us-east-1/6-month-commitment/anthropic.claude-v1": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_second": 0.00972, "output_cost_per_second": 0.00972, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/us-west-2/1-month-commitment/anthropic.claude-v1": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_second": 0.0175, "output_cost_per_second": 0.0175, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/us-west-2/6-month-commitment/anthropic.claude-v1": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_second": 0.00972, "output_cost_per_second": 0.00972, "litellm_provider": "bedrock", "mode": "chat" }, "anthropic.claude-v2": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_token": 0.000008, "output_cost_per_token": 0.000024, "litellm_provider": "bedrock", @@ -8965,9 +8965,9 @@ "supports_tool_choice": true }, "bedrock/us-east-1/anthropic.claude-v2": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_token": 0.000008, "output_cost_per_token": 0.000024, "litellm_provider": "bedrock", @@ -8975,9 +8975,9 @@ "supports_tool_choice": true }, "bedrock/us-west-2/anthropic.claude-v2": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_token": 0.000008, "output_cost_per_token": 0.000024, "litellm_provider": "bedrock", @@ -8985,9 +8985,9 @@ "supports_tool_choice": true }, "bedrock/ap-northeast-1/anthropic.claude-v2": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_token": 0.000008, "output_cost_per_token": 0.000024, "litellm_provider": "bedrock", @@ -8995,9 +8995,9 @@ "supports_tool_choice": true }, "bedrock/ap-northeast-1/1-month-commitment/anthropic.claude-v2": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_second": 0.0455, "output_cost_per_second": 0.0455, "litellm_provider": "bedrock", @@ -9005,9 +9005,9 @@ "supports_tool_choice": true }, "bedrock/ap-northeast-1/6-month-commitment/anthropic.claude-v2": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_second": 0.02527, "output_cost_per_second": 0.02527, "litellm_provider": "bedrock", @@ -9015,9 +9015,9 @@ "supports_tool_choice": true }, "bedrock/eu-central-1/anthropic.claude-v2": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_token": 0.000008, "output_cost_per_token": 0.000024, "litellm_provider": "bedrock", @@ -9025,9 +9025,9 @@ "supports_tool_choice": true }, "bedrock/eu-central-1/1-month-commitment/anthropic.claude-v2": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_second": 0.0415, "output_cost_per_second": 0.0415, "litellm_provider": "bedrock", @@ -9035,9 +9035,9 @@ "supports_tool_choice": true }, "bedrock/eu-central-1/6-month-commitment/anthropic.claude-v2": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_second": 0.02305, "output_cost_per_second": 0.02305, "litellm_provider": "bedrock", @@ -9045,9 +9045,9 @@ "supports_tool_choice": true }, "bedrock/us-east-1/1-month-commitment/anthropic.claude-v2": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_second": 0.0175, "output_cost_per_second": 0.0175, "litellm_provider": "bedrock", @@ -9055,9 +9055,9 @@ "supports_tool_choice": true }, "bedrock/us-east-1/6-month-commitment/anthropic.claude-v2": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_second": 0.00972, "output_cost_per_second": 0.00972, "litellm_provider": "bedrock", @@ -9065,9 +9065,9 @@ "supports_tool_choice": true }, "bedrock/us-west-2/1-month-commitment/anthropic.claude-v2": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_second": 0.0175, "output_cost_per_second": 0.0175, "litellm_provider": "bedrock", @@ -9075,9 +9075,9 @@ "supports_tool_choice": true }, "bedrock/us-west-2/6-month-commitment/anthropic.claude-v2": { - "max_tokens": 8191, + "max_tokens": 8191, "max_input_tokens": 100000, - "max_output_tokens": 8191, + "max_output_tokens": 8191, "input_cost_per_second": 0.00972, "output_cost_per_second": 0.00972, "litellm_provider": "bedrock", @@ -9085,8 +9085,8 @@ "supports_tool_choice": true }, "anthropic.claude-v2:1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_token": 0.000008, "output_cost_per_token": 0.000024, @@ -9095,8 +9095,8 @@ "supports_tool_choice": true }, "bedrock/us-east-1/anthropic.claude-v2:1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_token": 0.000008, "output_cost_per_token": 0.000024, @@ -9105,8 +9105,8 @@ "supports_tool_choice": true }, "bedrock/us-west-2/anthropic.claude-v2:1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_token": 0.000008, "output_cost_per_token": 0.000024, @@ -9115,8 +9115,8 @@ "supports_tool_choice": true }, "bedrock/ap-northeast-1/anthropic.claude-v2:1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_token": 0.000008, "output_cost_per_token": 0.000024, @@ -9125,8 +9125,8 @@ "supports_tool_choice": true }, "bedrock/ap-northeast-1/1-month-commitment/anthropic.claude-v2:1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.0455, "output_cost_per_second": 0.0455, @@ -9135,8 +9135,8 @@ "supports_tool_choice": true }, "bedrock/ap-northeast-1/6-month-commitment/anthropic.claude-v2:1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.02527, "output_cost_per_second": 0.02527, @@ -9145,8 +9145,8 @@ "supports_tool_choice": true }, "bedrock/eu-central-1/anthropic.claude-v2:1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_token": 0.000008, "output_cost_per_token": 0.000024, @@ -9155,8 +9155,8 @@ "supports_tool_choice": true }, "bedrock/eu-central-1/1-month-commitment/anthropic.claude-v2:1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.0415, "output_cost_per_second": 0.0415, @@ -9165,8 +9165,8 @@ "supports_tool_choice": true }, "bedrock/eu-central-1/6-month-commitment/anthropic.claude-v2:1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.02305, "output_cost_per_second": 0.02305, @@ -9175,8 +9175,8 @@ "supports_tool_choice": true }, "bedrock/us-east-1/1-month-commitment/anthropic.claude-v2:1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.0175, "output_cost_per_second": 0.0175, @@ -9185,8 +9185,8 @@ "supports_tool_choice": true }, "bedrock/us-east-1/6-month-commitment/anthropic.claude-v2:1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.00972, "output_cost_per_second": 0.00972, @@ -9195,8 +9195,8 @@ "supports_tool_choice": true }, "bedrock/us-west-2/1-month-commitment/anthropic.claude-v2:1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.0175, "output_cost_per_second": 0.0175, @@ -9205,8 +9205,8 @@ "supports_tool_choice": true }, "bedrock/us-west-2/6-month-commitment/anthropic.claude-v2:1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.00972, "output_cost_per_second": 0.00972, @@ -9215,8 +9215,8 @@ "supports_tool_choice": true }, "anthropic.claude-instant-v1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_token": 0.0000008, "output_cost_per_token": 0.0000024, @@ -9225,8 +9225,8 @@ "supports_tool_choice": true }, "bedrock/us-east-1/anthropic.claude-instant-v1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_token": 0.0000008, "output_cost_per_token": 0.0000024, @@ -9235,8 +9235,8 @@ "supports_tool_choice": true }, "bedrock/us-east-1/1-month-commitment/anthropic.claude-instant-v1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.011, "output_cost_per_second": 0.011, @@ -9245,8 +9245,8 @@ "supports_tool_choice": true }, "bedrock/us-east-1/6-month-commitment/anthropic.claude-instant-v1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.00611, "output_cost_per_second": 0.00611, @@ -9255,8 +9255,8 @@ "supports_tool_choice": true }, "bedrock/us-west-2/1-month-commitment/anthropic.claude-instant-v1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.011, "output_cost_per_second": 0.011, @@ -9265,8 +9265,8 @@ "supports_tool_choice": true }, "bedrock/us-west-2/6-month-commitment/anthropic.claude-instant-v1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.00611, "output_cost_per_second": 0.00611, @@ -9275,8 +9275,8 @@ "supports_tool_choice": true }, "bedrock/us-west-2/anthropic.claude-instant-v1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_token": 0.0000008, "output_cost_per_token": 0.0000024, @@ -9285,8 +9285,8 @@ "supports_tool_choice": true }, "bedrock/ap-northeast-1/anthropic.claude-instant-v1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_token": 0.00000223, "output_cost_per_token": 0.00000755, @@ -9295,8 +9295,8 @@ "supports_tool_choice": true }, "bedrock/ap-northeast-1/1-month-commitment/anthropic.claude-instant-v1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.01475, "output_cost_per_second": 0.01475, @@ -9305,8 +9305,8 @@ "supports_tool_choice": true }, "bedrock/ap-northeast-1/6-month-commitment/anthropic.claude-instant-v1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.008194, "output_cost_per_second": 0.008194, @@ -9315,8 +9315,8 @@ "supports_tool_choice": true }, "bedrock/eu-central-1/anthropic.claude-instant-v1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_token": 0.00000248, "output_cost_per_token": 0.00000838, @@ -9325,8 +9325,8 @@ "supports_tool_choice": true }, "bedrock/eu-central-1/1-month-commitment/anthropic.claude-instant-v1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.01635, "output_cost_per_second": 0.01635, @@ -9335,8 +9335,8 @@ "supports_tool_choice": true }, "bedrock/eu-central-1/6-month-commitment/anthropic.claude-instant-v1": { - "max_tokens": 8191, - "max_input_tokens": 100000, + "max_tokens": 8191, + "max_input_tokens": 100000, "max_output_tokens": 8191, "input_cost_per_second": 0.009083, "output_cost_per_second": 0.009083, @@ -9358,9 +9358,9 @@ "mode": "rerank" }, "cohere.command-text-v14": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 4096, - "max_output_tokens": 4096, + "max_output_tokens": 4096, "input_cost_per_token": 0.0000015, "output_cost_per_token": 0.0000020, "litellm_provider": "bedrock", @@ -9368,9 +9368,9 @@ "supports_tool_choice": true }, "bedrock/*/1-month-commitment/cohere.command-text-v14": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 4096, - "max_output_tokens": 4096, + "max_output_tokens": 4096, "input_cost_per_second": 0.011, "output_cost_per_second": 0.011, "litellm_provider": "bedrock", @@ -9378,9 +9378,9 @@ "supports_tool_choice": true }, "bedrock/*/6-month-commitment/cohere.command-text-v14": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 4096, - "max_output_tokens": 4096, + "max_output_tokens": 4096, "input_cost_per_second": 0.0066027, "output_cost_per_second": 0.0066027, "litellm_provider": "bedrock", @@ -9388,9 +9388,9 @@ "supports_tool_choice": true }, "cohere.command-light-text-v14": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 4096, - "max_output_tokens": 4096, + "max_output_tokens": 4096, "input_cost_per_token": 0.0000003, "output_cost_per_token": 0.0000006, "litellm_provider": "bedrock", @@ -9398,9 +9398,9 @@ "supports_tool_choice": true }, "bedrock/*/1-month-commitment/cohere.command-light-text-v14": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 4096, - "max_output_tokens": 4096, + "max_output_tokens": 4096, "input_cost_per_second": 0.001902, "output_cost_per_second": 0.001902, "litellm_provider": "bedrock", @@ -9408,9 +9408,9 @@ "supports_tool_choice": true }, "bedrock/*/6-month-commitment/cohere.command-light-text-v14": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 4096, - "max_output_tokens": 4096, + "max_output_tokens": 4096, "input_cost_per_second": 0.0011416, "output_cost_per_second": 0.0011416, "litellm_provider": "bedrock", @@ -9418,7 +9418,7 @@ "supports_tool_choice": true }, "cohere.command-r-plus-v1:0": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 128000, "max_output_tokens": 4096, "input_cost_per_token": 0.0000030, @@ -9428,7 +9428,7 @@ "supports_tool_choice": true }, "cohere.command-r-v1:0": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 128000, "max_output_tokens": 4096, "input_cost_per_token": 0.0000005, @@ -9438,17 +9438,17 @@ "supports_tool_choice": true }, "cohere.embed-english-v3": { - "max_tokens": 512, - "max_input_tokens": 512, + "max_tokens": 512, + "max_input_tokens": 512, "input_cost_per_token": 0.0000001, "output_cost_per_token": 0.000000, - "litellm_provider": "bedrock", + "litellm_provider": "bedrock", "mode": "embedding", "supports_embedding_image_input": true }, "cohere.embed-multilingual-v3": { - "max_tokens": 512, - "max_input_tokens": 512, + "max_tokens": 512, + "max_input_tokens": 512, "input_cost_per_token": 0.0000001, "output_cost_per_token": 0.000000, "litellm_provider": "bedrock", @@ -9456,7 +9456,7 @@ "supports_embedding_image_input": true }, "us.deepseek.r1-v1:0": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 128000, "max_output_tokens": 4096, "input_cost_per_token": 0.00000135, @@ -9464,178 +9464,178 @@ "litellm_provider": "bedrock_converse", "mode": "chat", "supports_reasoning": true, - "supports_function_calling": false, + "supports_function_calling": false, "supports_tool_choice": false }, "meta.llama3-3-70b-instruct-v1:0": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 128000, "max_output_tokens": 4096, "input_cost_per_token": 0.00000072, "output_cost_per_token": 0.00000072, "litellm_provider": "bedrock_converse", "mode": "chat", - "supports_function_calling": true, + "supports_function_calling": true, "supports_tool_choice": false }, "meta.llama2-13b-chat-v1": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, "input_cost_per_token": 0.00000075, "output_cost_per_token": 0.000001, "litellm_provider": "bedrock", "mode": "chat" }, "meta.llama2-70b-chat-v1": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, "input_cost_per_token": 0.00000195, "output_cost_per_token": 0.00000256, "litellm_provider": "bedrock", "mode": "chat" }, "meta.llama3-8b-instruct-v1:0": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, "input_cost_per_token": 0.0000003, "output_cost_per_token": 0.0000006, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/us-east-1/meta.llama3-8b-instruct-v1:0": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, "input_cost_per_token": 0.0000003, "output_cost_per_token": 0.0000006, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/us-west-1/meta.llama3-8b-instruct-v1:0": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, "input_cost_per_token": 0.0000003, "output_cost_per_token": 0.0000006, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/ap-south-1/meta.llama3-8b-instruct-v1:0": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, "input_cost_per_token": 0.00000036, "output_cost_per_token": 0.00000072, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/ca-central-1/meta.llama3-8b-instruct-v1:0": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, "input_cost_per_token": 0.00000035, "output_cost_per_token": 0.00000069, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/eu-west-1/meta.llama3-8b-instruct-v1:0": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, "input_cost_per_token": 0.00000032, "output_cost_per_token": 0.00000065, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/eu-west-2/meta.llama3-8b-instruct-v1:0": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, "input_cost_per_token": 0.00000039, "output_cost_per_token": 0.00000078, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/sa-east-1/meta.llama3-8b-instruct-v1:0": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, "input_cost_per_token": 0.0000005, "output_cost_per_token": 0.00000101, "litellm_provider": "bedrock", "mode": "chat" }, "meta.llama3-70b-instruct-v1:0": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, "input_cost_per_token": 0.00000265, "output_cost_per_token": 0.0000035, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/us-east-1/meta.llama3-70b-instruct-v1:0": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, "input_cost_per_token": 0.00000265, "output_cost_per_token": 0.0000035, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/us-west-1/meta.llama3-70b-instruct-v1:0": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, "input_cost_per_token": 0.00000265, "output_cost_per_token": 0.0000035, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/ap-south-1/meta.llama3-70b-instruct-v1:0": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, "input_cost_per_token": 0.00000318, "output_cost_per_token": 0.0000042, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/ca-central-1/meta.llama3-70b-instruct-v1:0": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, "input_cost_per_token": 0.00000305, "output_cost_per_token": 0.00000403, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/eu-west-1/meta.llama3-70b-instruct-v1:0": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, "input_cost_per_token": 0.00000286, "output_cost_per_token": 0.00000378, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/eu-west-2/meta.llama3-70b-instruct-v1:0": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, "input_cost_per_token": 0.00000345, "output_cost_per_token": 0.00000455, "litellm_provider": "bedrock", "mode": "chat" }, "bedrock/sa-east-1/meta.llama3-70b-instruct-v1:0": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, "input_cost_per_token": 0.00000445, "output_cost_per_token": 0.00000588, "litellm_provider": "bedrock", @@ -9649,7 +9649,7 @@ "output_cost_per_token": 0.00000022, "litellm_provider": "bedrock", "mode": "chat", - "supports_function_calling": true, + "supports_function_calling": true, "supports_tool_choice": false }, "us.meta.llama3-1-8b-instruct-v1:0": { @@ -9660,7 +9660,7 @@ "output_cost_per_token": 0.00000022, "litellm_provider": "bedrock", "mode": "chat", - "supports_function_calling": true, + "supports_function_calling": true, "supports_tool_choice": false }, "meta.llama3-1-70b-instruct-v1:0": { @@ -9671,7 +9671,7 @@ "output_cost_per_token": 0.00000099, "litellm_provider": "bedrock", "mode": "chat", - "supports_function_calling": true, + "supports_function_calling": true, "supports_tool_choice": false }, "us.meta.llama3-1-70b-instruct-v1:0": { @@ -9682,7 +9682,7 @@ "output_cost_per_token": 0.00000099, "litellm_provider": "bedrock", "mode": "chat", - "supports_function_calling": true, + "supports_function_calling": true, "supports_tool_choice": false }, "meta.llama3-1-405b-instruct-v1:0": { @@ -9693,7 +9693,7 @@ "output_cost_per_token": 0.000016, "litellm_provider": "bedrock", "mode": "chat", - "supports_function_calling": true, + "supports_function_calling": true, "supports_tool_choice": false }, "us.meta.llama3-1-405b-instruct-v1:0": { @@ -9704,7 +9704,7 @@ "output_cost_per_token": 0.000016, "litellm_provider": "bedrock", "mode": "chat", - "supports_function_calling": true, + "supports_function_calling": true, "supports_tool_choice": false }, "meta.llama3-2-1b-instruct-v1:0": { @@ -9715,7 +9715,7 @@ "output_cost_per_token": 0.0000001, "litellm_provider": "bedrock", "mode": "chat", - "supports_function_calling": true, + "supports_function_calling": true, "supports_tool_choice": false }, "us.meta.llama3-2-1b-instruct-v1:0": { @@ -9726,7 +9726,7 @@ "output_cost_per_token": 0.0000001, "litellm_provider": "bedrock", "mode": "chat", - "supports_function_calling": true, + "supports_function_calling": true, "supports_tool_choice": false }, "eu.meta.llama3-2-1b-instruct-v1:0": { @@ -9737,7 +9737,7 @@ "output_cost_per_token": 0.00000013, "litellm_provider": "bedrock", "mode": "chat", - "supports_function_calling": true, + "supports_function_calling": true, "supports_tool_choice": false }, "meta.llama3-2-3b-instruct-v1:0": { @@ -9748,7 +9748,7 @@ "output_cost_per_token": 0.00000015, "litellm_provider": "bedrock", "mode": "chat", - "supports_function_calling": true, + "supports_function_calling": true, "supports_tool_choice": false }, "us.meta.llama3-2-3b-instruct-v1:0": { @@ -9759,7 +9759,7 @@ "output_cost_per_token": 0.00000015, "litellm_provider": "bedrock", "mode": "chat", - "supports_function_calling": true, + "supports_function_calling": true, "supports_tool_choice": false }, "eu.meta.llama3-2-3b-instruct-v1:0": { @@ -9770,7 +9770,7 @@ "output_cost_per_token": 0.00000019, "litellm_provider": "bedrock", "mode": "chat", - "supports_function_calling": true, + "supports_function_calling": true, "supports_tool_choice": false }, "meta.llama3-2-11b-instruct-v1:0": { @@ -9781,7 +9781,7 @@ "output_cost_per_token": 0.00000035, "litellm_provider": "bedrock", "mode": "chat", - "supports_function_calling": true, + "supports_function_calling": true, "supports_tool_choice": false, "supports_vision": true }, @@ -9793,7 +9793,7 @@ "output_cost_per_token": 0.00000035, "litellm_provider": "bedrock", "mode": "chat", - "supports_function_calling": true, + "supports_function_calling": true, "supports_tool_choice": false, "supports_vision": true }, @@ -9805,7 +9805,7 @@ "output_cost_per_token": 0.000002, "litellm_provider": "bedrock", "mode": "chat", - "supports_function_calling": true, + "supports_function_calling": true, "supports_tool_choice": false, "supports_vision": true }, @@ -9817,66 +9817,66 @@ "output_cost_per_token": 0.000002, "litellm_provider": "bedrock", "mode": "chat", - "supports_function_calling": true, + "supports_function_calling": true, "supports_tool_choice": false, "supports_vision": true }, "us.meta.llama3-3-70b-instruct-v1:0": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 128000, "max_output_tokens": 4096, "input_cost_per_token": 0.00000072, "output_cost_per_token": 0.00000072, "litellm_provider": "bedrock_converse", "mode": "chat", - "supports_function_calling": true, + "supports_function_calling": true, "supports_tool_choice": false }, "512-x-512/50-steps/stability.stable-diffusion-xl-v0": { - "max_tokens": 77, - "max_input_tokens": 77, + "max_tokens": 77, + "max_input_tokens": 77, "output_cost_per_image": 0.018, "litellm_provider": "bedrock", "mode": "image_generation" }, "512-x-512/max-steps/stability.stable-diffusion-xl-v0": { - "max_tokens": 77, - "max_input_tokens": 77, + "max_tokens": 77, + "max_input_tokens": 77, "output_cost_per_image": 0.036, "litellm_provider": "bedrock", "mode": "image_generation" }, "max-x-max/50-steps/stability.stable-diffusion-xl-v0": { - "max_tokens": 77, - "max_input_tokens": 77, + "max_tokens": 77, + "max_input_tokens": 77, "output_cost_per_image": 0.036, "litellm_provider": "bedrock", "mode": "image_generation" }, "max-x-max/max-steps/stability.stable-diffusion-xl-v0": { - "max_tokens": 77, - "max_input_tokens": 77, + "max_tokens": 77, + "max_input_tokens": 77, "output_cost_per_image": 0.072, "litellm_provider": "bedrock", "mode": "image_generation" }, "1024-x-1024/50-steps/stability.stable-diffusion-xl-v1": { - "max_tokens": 77, - "max_input_tokens": 77, + "max_tokens": 77, + "max_input_tokens": 77, "output_cost_per_image": 0.04, "litellm_provider": "bedrock", "mode": "image_generation" }, "1024-x-1024/max-steps/stability.stable-diffusion-xl-v1": { - "max_tokens": 77, - "max_input_tokens": 77, + "max_tokens": 77, + "max_input_tokens": 77, "output_cost_per_image": 0.08, "litellm_provider": "bedrock", "mode": "image_generation" }, "stability.sd3-large-v1:0": { - "max_tokens": 77, - "max_input_tokens": 77, + "max_tokens": 77, + "max_input_tokens": 77, "output_cost_per_image": 0.08, "litellm_provider": "bedrock", "mode": "image_generation" @@ -9903,8 +9903,8 @@ "mode": "image_generation" }, "stability.stable-image-ultra-v1:0": { - "max_tokens": 77, - "max_input_tokens": 77, + "max_tokens": 77, + "max_input_tokens": 77, "output_cost_per_image": 0.14, "litellm_provider": "bedrock", "mode": "image_generation" @@ -9917,54 +9917,54 @@ "mode": "image_generation" }, "sagemaker/meta-textgeneration-llama-2-7b": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, "input_cost_per_token": 0.000, "output_cost_per_token": 0.000, "litellm_provider": "sagemaker", "mode": "completion" }, "sagemaker/meta-textgeneration-llama-2-7b-f": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, "input_cost_per_token": 0.000, "output_cost_per_token": 0.000, "litellm_provider": "sagemaker", "mode": "chat" }, "sagemaker/meta-textgeneration-llama-2-13b": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, "input_cost_per_token": 0.000, "output_cost_per_token": 0.000, "litellm_provider": "sagemaker", "mode": "completion" }, "sagemaker/meta-textgeneration-llama-2-13b-f": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, "input_cost_per_token": 0.000, "output_cost_per_token": 0.000, "litellm_provider": "sagemaker", "mode": "chat" }, "sagemaker/meta-textgeneration-llama-2-70b": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, "input_cost_per_token": 0.000, "output_cost_per_token": 0.000, "litellm_provider": "sagemaker", "mode": "completion" }, "sagemaker/meta-textgeneration-llama-2-70b-b-f": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, "input_cost_per_token": 0.000, "output_cost_per_token": 0.000, "litellm_provider": "sagemaker", @@ -10094,9 +10094,9 @@ "supports_tool_choice": true }, "ollama/codegemma": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "litellm_provider": "ollama", @@ -10109,7 +10109,7 @@ "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "litellm_provider": "ollama", - "mode": "chat", + "mode": "chat", "supports_function_calling": false }, "ollama/deepseek-coder-v2-instruct": { @@ -10119,7 +10119,7 @@ "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "litellm_provider": "ollama", - "mode": "chat", + "mode": "chat", "supports_function_calling": true }, "ollama/deepseek-coder-v2-base": { @@ -10129,7 +10129,7 @@ "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "litellm_provider": "ollama", - "mode": "completion", + "mode": "completion", "supports_function_calling": true }, "ollama/deepseek-coder-v2-lite-instruct": { @@ -10139,7 +10139,7 @@ "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "litellm_provider": "ollama", - "mode": "chat", + "mode": "chat", "supports_function_calling": true }, "ollama/deepseek-coder-v2-lite-base": { @@ -10149,7 +10149,7 @@ "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "litellm_provider": "ollama", - "mode": "completion", + "mode": "completion", "supports_function_calling": true }, "ollama/internlm2_5-20b-chat": { @@ -10159,49 +10159,49 @@ "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "litellm_provider": "ollama", - "mode": "chat", + "mode": "chat", "supports_function_calling": true }, "ollama/llama2": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "litellm_provider": "ollama", "mode": "chat" }, "ollama/llama2:7b": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "litellm_provider": "ollama", "mode": "chat" }, "ollama/llama2:13b": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "litellm_provider": "ollama", "mode": "chat" }, "ollama/llama2:70b": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "litellm_provider": "ollama", "mode": "chat" }, "ollama/llama2-uncensored": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "litellm_provider": "ollama", @@ -10241,7 +10241,7 @@ "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "litellm_provider": "ollama", - "mode": "chat", + "mode": "chat", "supports_function_calling": true }, "ollama/mistral-large-instruct-2407": { @@ -10299,8 +10299,8 @@ "mode": "chat" }, "ollama/codellama": { - "max_tokens": 4096, - "max_input_tokens": 4096, + "max_tokens": 4096, + "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, @@ -10308,8 +10308,8 @@ "mode": "completion" }, "ollama/orca-mini": { - "max_tokens": 4096, - "max_input_tokens": 4096, + "max_tokens": 4096, + "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, @@ -10534,183 +10534,183 @@ "mode": "chat", "supports_tool_choice": true }, - "perplexity/codellama-34b-instruct": { + "perplexity/codellama-34b-instruct": { "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, - "input_cost_per_token": 0.00000035, - "output_cost_per_token": 0.00000140, - "litellm_provider": "perplexity", - "mode": "chat" + "input_cost_per_token": 0.00000035, + "output_cost_per_token": 0.00000140, + "litellm_provider": "perplexity", + "mode": "chat" }, - "perplexity/codellama-70b-instruct": { + "perplexity/codellama-70b-instruct": { "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, - "input_cost_per_token": 0.00000070, - "output_cost_per_token": 0.00000280, - "litellm_provider": "perplexity", - "mode": "chat" + "input_cost_per_token": 0.00000070, + "output_cost_per_token": 0.00000280, + "litellm_provider": "perplexity", + "mode": "chat" }, - "perplexity/llama-3.1-70b-instruct": { + "perplexity/llama-3.1-70b-instruct": { "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 0.000001, + "input_cost_per_token": 0.000001, "output_cost_per_token": 0.000001, - "litellm_provider": "perplexity", - "mode": "chat" + "litellm_provider": "perplexity", + "mode": "chat" }, - "perplexity/llama-3.1-8b-instruct": { + "perplexity/llama-3.1-8b-instruct": { "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 0.0000002, - "output_cost_per_token": 0.0000002, - "litellm_provider": "perplexity", - "mode": "chat" + "input_cost_per_token": 0.0000002, + "output_cost_per_token": 0.0000002, + "litellm_provider": "perplexity", + "mode": "chat" }, - "perplexity/llama-3.1-sonar-huge-128k-online": { + "perplexity/llama-3.1-sonar-huge-128k-online": { "max_tokens": 127072, "max_input_tokens": 127072, "max_output_tokens": 127072, - "input_cost_per_token": 0.000005, + "input_cost_per_token": 0.000005, "output_cost_per_token": 0.000005, - "litellm_provider": "perplexity", + "litellm_provider": "perplexity", "mode": "chat", "deprecation_date": "2025-02-22" }, - "perplexity/llama-3.1-sonar-large-128k-online": { + "perplexity/llama-3.1-sonar-large-128k-online": { "max_tokens": 127072, "max_input_tokens": 127072, "max_output_tokens": 127072, - "input_cost_per_token": 0.000001, + "input_cost_per_token": 0.000001, "output_cost_per_token": 0.000001, - "litellm_provider": "perplexity", + "litellm_provider": "perplexity", "mode": "chat", "deprecation_date": "2025-02-22" }, - "perplexity/llama-3.1-sonar-large-128k-chat": { + "perplexity/llama-3.1-sonar-large-128k-chat": { "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 0.000001, + "input_cost_per_token": 0.000001, "output_cost_per_token": 0.000001, - "litellm_provider": "perplexity", + "litellm_provider": "perplexity", "mode": "chat", "deprecation_date": "2025-02-22" }, - "perplexity/llama-3.1-sonar-small-128k-chat": { + "perplexity/llama-3.1-sonar-small-128k-chat": { "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 0.0000002, - "output_cost_per_token": 0.0000002, - "litellm_provider": "perplexity", + "input_cost_per_token": 0.0000002, + "output_cost_per_token": 0.0000002, + "litellm_provider": "perplexity", "mode": "chat", "deprecation_date": "2025-02-22" }, - "perplexity/llama-3.1-sonar-small-128k-online": { + "perplexity/llama-3.1-sonar-small-128k-online": { "max_tokens": 127072, "max_input_tokens": 127072, "max_output_tokens": 127072, - "input_cost_per_token": 0.0000002, - "output_cost_per_token": 0.0000002, - "litellm_provider": "perplexity", + "input_cost_per_token": 0.0000002, + "output_cost_per_token": 0.0000002, + "litellm_provider": "perplexity", "mode": "chat" , "deprecation_date": "2025-02-22" }, - "perplexity/sonar": { + "perplexity/sonar": { "max_tokens": 127072, "max_input_tokens": 127072, "max_output_tokens": 127072, - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000001, - "litellm_provider": "perplexity", - "mode": "chat" + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000001, + "litellm_provider": "perplexity", + "mode": "chat" }, - "perplexity/sonar-pro": { + "perplexity/sonar-pro": { "max_tokens": 200000, "max_input_tokens": 200000, "max_output_tokens": 8096, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "litellm_provider": "perplexity", - "mode": "chat" + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "litellm_provider": "perplexity", + "mode": "chat" }, - "perplexity/sonar": { + "perplexity/sonar": { "max_tokens": 127072, "max_input_tokens": 127072, "max_output_tokens": 127072, - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000001, - "litellm_provider": "perplexity", - "mode": "chat" + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000001, + "litellm_provider": "perplexity", + "mode": "chat" }, - "perplexity/sonar-pro": { + "perplexity/sonar-pro": { "max_tokens": 200000, "max_input_tokens": 200000, "max_output_tokens": 8096, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "litellm_provider": "perplexity", - "mode": "chat" + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "litellm_provider": "perplexity", + "mode": "chat" }, - "perplexity/pplx-7b-chat": { + "perplexity/pplx-7b-chat": { "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, - "input_cost_per_token": 0.00000007, - "output_cost_per_token": 0.00000028, - "litellm_provider": "perplexity", - "mode": "chat" - }, - "perplexity/pplx-70b-chat": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "input_cost_per_token": 0.00000070, - "output_cost_per_token": 0.00000280, - "litellm_provider": "perplexity", - "mode": "chat" - }, - "perplexity/pplx-7b-online": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "input_cost_per_token": 0.0000000, - "output_cost_per_token": 0.00000028, - "input_cost_per_request": 0.005, - "litellm_provider": "perplexity", - "mode": "chat" - }, - "perplexity/pplx-70b-online": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "input_cost_per_token": 0.0000000, - "output_cost_per_token": 0.00000280, - "input_cost_per_request": 0.005, - "litellm_provider": "perplexity", - "mode": "chat" - }, - "perplexity/llama-2-70b-chat": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "input_cost_per_token": 0.00000070, - "output_cost_per_token": 0.00000280, - "litellm_provider": "perplexity", - "mode": "chat" - }, - "perplexity/mistral-7b-instruct": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, "input_cost_per_token": 0.00000007, "output_cost_per_token": 0.00000028, - "litellm_provider": "perplexity", - "mode": "chat" + "litellm_provider": "perplexity", + "mode": "chat" + }, + "perplexity/pplx-70b-chat": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000070, + "output_cost_per_token": 0.00000280, + "litellm_provider": "perplexity", + "mode": "chat" + }, + "perplexity/pplx-7b-online": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.0000000, + "output_cost_per_token": 0.00000028, + "input_cost_per_request": 0.005, + "litellm_provider": "perplexity", + "mode": "chat" + }, + "perplexity/pplx-70b-online": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.0000000, + "output_cost_per_token": 0.00000280, + "input_cost_per_request": 0.005, + "litellm_provider": "perplexity", + "mode": "chat" + }, + "perplexity/llama-2-70b-chat": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000070, + "output_cost_per_token": 0.00000280, + "litellm_provider": "perplexity", + "mode": "chat" + }, + "perplexity/mistral-7b-instruct": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000007, + "output_cost_per_token": 0.00000028, + "litellm_provider": "perplexity", + "mode": "chat" }, "perplexity/mixtral-8x7b-instruct": { "max_tokens": 4096, @@ -10763,9 +10763,9 @@ "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, - "input_cost_per_token": 0.0000001, + "input_cost_per_token": 0.0000001, "output_cost_per_token": 0.0000001, - "litellm_provider": "fireworks_ai", + "litellm_provider": "fireworks_ai", "mode": "chat", "supports_function_calling": true, "supports_response_schema": true, @@ -10776,9 +10776,9 @@ "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, - "input_cost_per_token": 0.0000001, + "input_cost_per_token": 0.0000001, "output_cost_per_token": 0.0000001, - "litellm_provider": "fireworks_ai", + "litellm_provider": "fireworks_ai", "mode": "chat", "supports_function_calling": true, "supports_response_schema": true, @@ -10789,9 +10789,9 @@ "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, - "input_cost_per_token": 0.0000001, + "input_cost_per_token": 0.0000001, "output_cost_per_token": 0.0000001, - "litellm_provider": "fireworks_ai", + "litellm_provider": "fireworks_ai", "mode": "chat", "supports_function_calling": true, "supports_response_schema": true, @@ -10802,9 +10802,9 @@ "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, - "input_cost_per_token": 0.0000002, + "input_cost_per_token": 0.0000002, "output_cost_per_token": 0.0000002, - "litellm_provider": "fireworks_ai", + "litellm_provider": "fireworks_ai", "mode": "chat", "supports_function_calling": true, "supports_vision": true, @@ -10816,9 +10816,9 @@ "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, - "input_cost_per_token": 0.0000009, + "input_cost_per_token": 0.0000009, "output_cost_per_token": 0.0000009, - "litellm_provider": "fireworks_ai", + "litellm_provider": "fireworks_ai", "mode": "chat", "supports_function_calling": true, "supports_vision": true, @@ -10829,9 +10829,9 @@ "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, - "input_cost_per_token": 0.0000009, + "input_cost_per_token": 0.0000009, "output_cost_per_token": 0.0000009, - "litellm_provider": "fireworks_ai", + "litellm_provider": "fireworks_ai", "mode": "chat", "supports_function_calling": true, "supports_response_schema": true, @@ -10842,9 +10842,9 @@ "max_tokens": 65536, "max_input_tokens": 65536, "max_output_tokens": 65536, - "input_cost_per_token": 0.0000012, + "input_cost_per_token": 0.0000012, "output_cost_per_token": 0.0000012, - "litellm_provider": "fireworks_ai", + "litellm_provider": "fireworks_ai", "mode": "chat", "supports_function_calling": true, "supports_response_schema": true, @@ -10855,9 +10855,9 @@ "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, - "input_cost_per_token": 0.0000009, + "input_cost_per_token": 0.0000009, "output_cost_per_token": 0.0000009, - "litellm_provider": "fireworks_ai", + "litellm_provider": "fireworks_ai", "mode": "chat", "supports_function_calling": true, "supports_response_schema": true, @@ -10868,9 +10868,9 @@ "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.0000009, + "input_cost_per_token": 0.0000009, "output_cost_per_token": 0.0000009, - "litellm_provider": "fireworks_ai", + "litellm_provider": "fireworks_ai", "mode": "chat", "supports_function_calling": true, "supports_response_schema": true, @@ -10881,9 +10881,9 @@ "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, - "input_cost_per_token": 0.000003, + "input_cost_per_token": 0.000003, "output_cost_per_token": 0.000003, - "litellm_provider": "fireworks_ai", + "litellm_provider": "fireworks_ai", "mode": "chat", "supports_function_calling": true, "supports_response_schema": true, @@ -10894,9 +10894,9 @@ "max_tokens": 65536, "max_input_tokens": 65536, "max_output_tokens": 8192, - "input_cost_per_token": 0.0000012, + "input_cost_per_token": 0.0000012, "output_cost_per_token": 0.0000012, - "litellm_provider": "fireworks_ai", + "litellm_provider": "fireworks_ai", "mode": "chat", "supports_function_calling": true, "supports_response_schema": true, @@ -11000,9 +11000,9 @@ "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, - "input_cost_per_token": 0.00000015, + "input_cost_per_token": 0.00000015, "output_cost_per_token": 0.00000015, - "litellm_provider": "anyscale", + "litellm_provider": "anyscale", "mode": "chat", "supports_function_calling": true, "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/mistralai-Mistral-7B-Instruct-v0.1" @@ -11011,9 +11011,9 @@ "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, - "input_cost_per_token": 0.00000015, + "input_cost_per_token": 0.00000015, "output_cost_per_token": 0.00000015, - "litellm_provider": "anyscale", + "litellm_provider": "anyscale", "mode": "chat", "supports_function_calling": true, "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/mistralai-Mixtral-8x7B-Instruct-v0.1" @@ -11022,9 +11022,9 @@ "max_tokens": 65536, "max_input_tokens": 65536, "max_output_tokens": 65536, - "input_cost_per_token": 0.00000090, + "input_cost_per_token": 0.00000090, "output_cost_per_token": 0.00000090, - "litellm_provider": "anyscale", + "litellm_provider": "anyscale", "mode": "chat", "supports_function_calling": true, "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/mistralai-Mixtral-8x22B-Instruct-v0.1" @@ -11033,18 +11033,18 @@ "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, - "input_cost_per_token": 0.00000015, + "input_cost_per_token": 0.00000015, "output_cost_per_token": 0.00000015, - "litellm_provider": "anyscale", + "litellm_provider": "anyscale", "mode": "chat" }, "anyscale/google/gemma-7b-it": { "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, - "input_cost_per_token": 0.00000015, + "input_cost_per_token": 0.00000015, "output_cost_per_token": 0.00000015, - "litellm_provider": "anyscale", + "litellm_provider": "anyscale", "mode": "chat", "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/google-gemma-7b-it" }, @@ -11052,45 +11052,45 @@ "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000015, - "output_cost_per_token": 0.00000015, - "litellm_provider": "anyscale", + "input_cost_per_token": 0.00000015, + "output_cost_per_token": 0.00000015, + "litellm_provider": "anyscale", "mode": "chat" }, "anyscale/meta-llama/Llama-2-13b-chat-hf": { "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.00000025, - "output_cost_per_token": 0.00000025, - "litellm_provider": "anyscale", + "input_cost_per_token": 0.00000025, + "output_cost_per_token": 0.00000025, + "litellm_provider": "anyscale", "mode": "chat" }, "anyscale/meta-llama/Llama-2-70b-chat-hf": { "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000001, - "litellm_provider": "anyscale", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000001, + "litellm_provider": "anyscale", "mode": "chat" }, "anyscale/codellama/CodeLlama-34b-Instruct-hf": { "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000001, - "litellm_provider": "anyscale", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000001, + "litellm_provider": "anyscale", "mode": "chat" }, "anyscale/codellama/CodeLlama-70b-Instruct-hf": { "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000001, - "litellm_provider": "anyscale", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000001, + "litellm_provider": "anyscale", "mode": "chat", "source" : "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/codellama-CodeLlama-70b-Instruct-hf" }, @@ -11098,9 +11098,9 @@ "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, - "input_cost_per_token": 0.00000015, - "output_cost_per_token": 0.00000015, - "litellm_provider": "anyscale", + "input_cost_per_token": 0.00000015, + "output_cost_per_token": 0.00000015, + "litellm_provider": "anyscale", "mode": "chat", "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/meta-llama-Meta-Llama-3-8B-Instruct" }, @@ -11108,46 +11108,46 @@ "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, - "input_cost_per_token": 0.00000100, - "output_cost_per_token": 0.00000100, - "litellm_provider": "anyscale", + "input_cost_per_token": 0.00000100, + "output_cost_per_token": 0.00000100, + "litellm_provider": "anyscale", "mode": "chat", "source" : "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/meta-llama-Meta-Llama-3-70B-Instruct" }, "cloudflare/@cf/meta/llama-2-7b-chat-fp16": { - "max_tokens": 3072, - "max_input_tokens": 3072, - "max_output_tokens": 3072, - "input_cost_per_token": 0.000001923, - "output_cost_per_token": 0.000001923, - "litellm_provider": "cloudflare", + "max_tokens": 3072, + "max_input_tokens": 3072, + "max_output_tokens": 3072, + "input_cost_per_token": 0.000001923, + "output_cost_per_token": 0.000001923, + "litellm_provider": "cloudflare", "mode": "chat" }, "cloudflare/@cf/meta/llama-2-7b-chat-int8": { - "max_tokens": 2048, - "max_input_tokens": 2048, - "max_output_tokens": 2048, - "input_cost_per_token": 0.000001923, - "output_cost_per_token": 0.000001923, - "litellm_provider": "cloudflare", + "max_tokens": 2048, + "max_input_tokens": 2048, + "max_output_tokens": 2048, + "input_cost_per_token": 0.000001923, + "output_cost_per_token": 0.000001923, + "litellm_provider": "cloudflare", "mode": "chat" }, "cloudflare/@cf/mistral/mistral-7b-instruct-v0.1": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "input_cost_per_token": 0.000001923, - "output_cost_per_token": 0.000001923, - "litellm_provider": "cloudflare", + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.000001923, + "output_cost_per_token": 0.000001923, + "litellm_provider": "cloudflare", "mode": "chat" }, "cloudflare/@hf/thebloke/codellama-7b-instruct-awq": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "input_cost_per_token": 0.000001923, - "output_cost_per_token": 0.000001923, - "litellm_provider": "cloudflare", + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.000001923, + "output_cost_per_token": 0.000001923, + "litellm_provider": "cloudflare", "mode": "chat" }, "voyage/voyage-01": { @@ -11279,7 +11279,7 @@ "databricks/databricks-claude-3-7-sonnet": { "max_tokens": 200000, "max_input_tokens": 200000, - "max_output_tokens": 128000, + "max_output_tokens": 128000, "input_cost_per_token": 0.0000025, "input_dbu_cost_per_token": 0.00003571, "output_cost_per_token": 0.00017857, @@ -11296,7 +11296,7 @@ "databricks/databricks-meta-llama-3-1-405b-instruct": { "max_tokens": 128000, "max_input_tokens": 128000, - "max_output_tokens": 128000, + "max_output_tokens": 128000, "input_cost_per_token": 0.000005, "input_dbu_cost_per_token": 0.000071429, "output_cost_per_token": 0.00001500002, @@ -11310,7 +11310,7 @@ "databricks/databricks-meta-llama-3-1-70b-instruct": { "max_tokens": 128000, "max_input_tokens": 128000, - "max_output_tokens": 128000, + "max_output_tokens": 128000, "input_cost_per_token": 0.00000100002, "input_dbu_cost_per_token": 0.000014286, "output_cost_per_token": 0.00000299999, @@ -11324,7 +11324,7 @@ "databricks/databricks-meta-llama-3-3-70b-instruct": { "max_tokens": 128000, "max_input_tokens": 128000, - "max_output_tokens": 128000, + "max_output_tokens": 128000, "input_cost_per_token": 0.00000100002, "input_dbu_cost_per_token": 0.000014286, "output_cost_per_token": 0.00000299999, @@ -11338,7 +11338,7 @@ "databricks/databricks-dbrx-instruct": { "max_tokens": 32768, "max_input_tokens": 32768, - "max_output_tokens": 32768, + "max_output_tokens": 32768, "input_cost_per_token": 0.00000074998, "input_dbu_cost_per_token": 0.000010714, "output_cost_per_token": 0.00000224901, @@ -11352,7 +11352,7 @@ "databricks/databricks-meta-llama-3-70b-instruct": { "max_tokens": 128000, "max_input_tokens": 128000, - "max_output_tokens": 128000, + "max_output_tokens": 128000, "input_cost_per_token": 0.00000100002, "input_dbu_cost_per_token": 0.000014286, "output_cost_per_token": 0.00000299999, @@ -11366,7 +11366,7 @@ "databricks/databricks-llama-2-70b-chat": { "max_tokens": 4096, "max_input_tokens": 4096, - "max_output_tokens": 4096, + "max_output_tokens": 4096, "input_cost_per_token": 0.00000050001, "input_dbu_cost_per_token": 0.000007143, "output_cost_per_token": 0.0000015, @@ -11380,7 +11380,7 @@ "databricks/databricks-mixtral-8x7b-instruct": { "max_tokens": 4096, "max_input_tokens": 4096, - "max_output_tokens": 4096, + "max_output_tokens": 4096, "input_cost_per_token": 0.00000050001, "input_dbu_cost_per_token": 0.000007143, "output_cost_per_token": 0.00000099902, @@ -11394,7 +11394,7 @@ "databricks/databricks-mpt-30b-instruct": { "max_tokens": 8192, "max_input_tokens": 8192, - "max_output_tokens": 8192, + "max_output_tokens": 8192, "input_cost_per_token": 0.00000099902, "input_dbu_cost_per_token": 0.000014286, "output_cost_per_token": 0.00000099902, @@ -11408,7 +11408,7 @@ "databricks/databricks-mpt-7b-instruct": { "max_tokens": 8192, "max_input_tokens": 8192, - "max_output_tokens": 8192, + "max_output_tokens": 8192, "input_cost_per_token": 0.00000050001, "input_dbu_cost_per_token": 0.000007143, "output_cost_per_token": 0.0, @@ -11422,7 +11422,7 @@ "databricks/databricks-bge-large-en": { "max_tokens": 512, "max_input_tokens": 512, - "output_vector_size": 1024, + "output_vector_size": 1024, "input_cost_per_token": 0.00000010003, "input_dbu_cost_per_token": 0.000001429, "output_cost_per_token": 0.0, @@ -11435,7 +11435,7 @@ "databricks/databricks-gte-large-en": { "max_tokens": 8192, "max_input_tokens": 8192, - "output_vector_size": 1024, + "output_vector_size": 1024, "input_cost_per_token": 0.00000012999, "input_dbu_cost_per_token": 0.000001857, "output_cost_per_token": 0.0, @@ -11448,7 +11448,7 @@ "sambanova/Meta-Llama-3.1-8B-Instruct": { "max_tokens": 16000, "max_input_tokens": 16000, - "max_output_tokens": 16000, + "max_output_tokens": 16000, "input_cost_per_token": 0.0000001, "output_cost_per_token": 0.0000002, "litellm_provider": "sambanova", @@ -11459,7 +11459,7 @@ "sambanova/Meta-Llama-3.1-70B-Instruct": { "max_tokens": 128000, "max_input_tokens": 128000, - "max_output_tokens": 128000, + "max_output_tokens": 128000, "input_cost_per_token": 0.0000006, "output_cost_per_token": 0.0000012, "litellm_provider": "sambanova", @@ -11470,7 +11470,7 @@ "sambanova/Meta-Llama-3.1-405B-Instruct": { "max_tokens": 16000, "max_input_tokens": 16000, - "max_output_tokens": 16000, + "max_output_tokens": 16000, "input_cost_per_token": 0.000005, "output_cost_per_token": 0.000010, "litellm_provider": "sambanova", @@ -11481,7 +11481,7 @@ "sambanova/Meta-Llama-3.2-1B-Instruct": { "max_tokens": 16000, "max_input_tokens": 16000, - "max_output_tokens": 16000, + "max_output_tokens": 16000, "input_cost_per_token": 0.0000004, "output_cost_per_token": 0.0000008, "litellm_provider": "sambanova", @@ -11492,7 +11492,7 @@ "sambanova/Meta-Llama-3.2-3B-Instruct": { "max_tokens": 4000, "max_input_tokens": 4000, - "max_output_tokens": 4000, + "max_output_tokens": 4000, "input_cost_per_token": 0.0000008, "output_cost_per_token": 0.0000016, "litellm_provider": "sambanova", @@ -11503,7 +11503,7 @@ "sambanova/Qwen2.5-Coder-32B-Instruct": { "max_tokens": 8000, "max_input_tokens": 8000, - "max_output_tokens": 8000, + "max_output_tokens": 8000, "input_cost_per_token": 0.0000015, "output_cost_per_token": 0.000003, "litellm_provider": "sambanova", @@ -11514,7 +11514,7 @@ "sambanova/Qwen2.5-72B-Instruct": { "max_tokens": 8000, "max_input_tokens": 8000, - "max_output_tokens": 8000, + "max_output_tokens": 8000, "input_cost_per_token": 0.000002, "output_cost_per_token": 0.000004, "litellm_provider": "sambanova", @@ -11525,13 +11525,13 @@ "assemblyai/nano": { "mode": "audio_transcription", "input_cost_per_second": 0.00010278, - "output_cost_per_second": 0.00, + "output_cost_per_second": 0.00, "litellm_provider": "assemblyai" }, "assemblyai/best": { "mode": "audio_transcription", "input_cost_per_second": 0.00003333, - "output_cost_per_second": 0.00, + "output_cost_per_second": 0.00, "litellm_provider": "assemblyai" }, "jina-reranker-v2-base-multilingual": { @@ -11712,5 +11712,45 @@ "max_output_tokens": 8192, "litellm_provider": "snowflake", "mode": "chat" + }, + "digitalocean/deepseek-r1-distill-llama-70b": { + "input_cost_per_token": 0.00000099, + "output_cost_per_token": 0.00000099, + "litellm_provider": "digitalocean", + "mode": "chat", + "supported_endpoints": ["/v1/chat/completions"], + "supported_modalities": ["text"] + }, + "digitalocean/llama-3.1-8b": { + "input_cost_per_token": 0.0000002, + "output_cost_per_token": 0.0000002, + "litellm_provider": "digitalocean", + "mode": "chat", + "supported_endpoints": ["/v1/chat/completions"], + "supported_modalities": ["text"] + }, + "digitalocean/llama-3.1-70b": { + "input_cost_per_token": 0.0000007, + "output_cost_per_token": 0.0000007, + "litellm_provider": "digitalocean", + "mode": "chat", + "supported_endpoints": ["/v1/chat/completions"], + "supported_modalities": ["text"] + }, + "digitalocean/llama-3.3-70b": { + "input_cost_per_token": 0.00000065, + "output_cost_per_token": 0.00000065, + "litellm_provider": "digitalocean", + "mode": "chat", + "supported_endpoints": ["/v1/chat/completions"], + "supported_modalities": ["text"] + }, + "digitalocean/mistral-nemo": { + "input_cost_per_token": 0.0000003, + "output_cost_per_token": 0.0000003, + "litellm_provider": "digitalocean", + "mode": "chat", + "supported_endpoints": ["/v1/chat/completions"], + "supported_modalities": ["text"] } } diff --git a/proxy_server_config.yaml b/proxy_server_config.yaml index 23316c4a71e..3803bc8b61e 100644 --- a/proxy_server_config.yaml +++ b/proxy_server_config.yaml @@ -18,7 +18,7 @@ model_list: api_version: "2023-05-15" api_key: os.environ/AZURE_API_KEY # The `os.environ/` prefix tells litellm to read this from the env. See https://docs.litellm.ai/docs/simple_proxy#load-api-keys-from-vault - model_name: gpt-3.5-turbo-large - litellm_params: + litellm_params: model: "gpt-3.5-turbo-1106" api_key: os.environ/OPENAI_API_KEY rpm: 480 @@ -36,9 +36,9 @@ model_list: - model_name: sagemaker-completion-model litellm_params: model: sagemaker/berri-benchmarking-Llama-2-70b-chat-hf-4 - input_cost_per_second: 0.000420 + input_cost_per_second: 0.000420 - model_name: text-embedding-ada-002 - litellm_params: + litellm_params: model: azure/azure-embedding-model api_key: os.environ/AZURE_API_KEY api_base: https://openai-gpt-4-test-v-1.openai.azure.com/ @@ -106,7 +106,7 @@ model_list: litellm_params: model: openai/* api_key: os.environ/OPENAI_API_KEY - + # provider specific wildcard routing - model_name: "anthropic/*" @@ -137,18 +137,23 @@ model_list: model: openai/my-fake-model api_key: my-fake-key api_base: https://exampleopenaiendpoint-production.up.railway.appxxxx/ + - model_name: "digitalocean/*" + litellm_params: + model: "digitalocean/*" + api_key: os.environ/DIGITALOCEAN_API_KEY + api_base: os.environ/DIGITALOCEAN_AGENT_ENDPOINT litellm_settings: # set_verbose: True # Uncomment this if you want to see verbose logs; not recommended in production drop_params: True - # max_budget: 100 + # max_budget: 100 # budget_duration: 30d num_retries: 5 request_timeout: 600 telemetry: False context_window_fallbacks: [{"gpt-3.5-turbo": ["gpt-3.5-turbo-large"]}] - default_team_settings: + default_team_settings: - team_id: team-1 success_callback: ["langfuse"] failure_callback: ["langfuse"] @@ -180,14 +185,14 @@ files_settings: api_key: os.environ/OPENAI_API_KEY router_settings: - routing_strategy: usage-based-routing-v2 + routing_strategy: usage-based-routing-v2 redis_host: os.environ/REDIS_HOST redis_password: os.environ/REDIS_PASSWORD redis_port: os.environ/REDIS_PORT enable_pre_call_checks: true - model_group_alias: {"my-special-fake-model-alias-name": "fake-openai-endpoint-3"} + model_group_alias: {"my-special-fake-model-alias-name": "fake-openai-endpoint-3"} -general_settings: +general_settings: master_key: sk-1234 # [OPTIONAL] Use to enforce auth on proxy. See - https://docs.litellm.ai/docs/proxy/virtual_keys store_model_in_db: True proxy_budget_rescheduler_min_time: 60 @@ -200,7 +205,7 @@ general_settings: - path: "/v1/rerank" # route you want to add to LiteLLM Proxy Server target: "https://api.cohere.com/v1/rerank" # URL this route should forward requests to headers: # headers to forward to this URL - content-type: application/json # (Optional) Extra Headers to pass to this endpoint + content-type: application/json # (Optional) Extra Headers to pass to this endpoint accept: application/json forward_headers: True @@ -208,4 +213,4 @@ general_settings: # settings for using redis caching # REDIS_HOST: redis-16337.c322.us-east-1-2.ec2.cloud.redislabs.com # REDIS_PORT: "16337" - # REDIS_PASSWORD: + # REDIS_PASSWORD: diff --git a/tests/litellm/llms/digitalocean/chat/test_digitalocean_chat_transformation.py b/tests/litellm/llms/digitalocean/chat/test_digitalocean_chat_transformation.py new file mode 100644 index 00000000000..a6d03f8bd6e --- /dev/null +++ b/tests/litellm/llms/digitalocean/chat/test_digitalocean_chat_transformation.py @@ -0,0 +1,60 @@ +import os +import sys +import pytest + +sys.path.insert( + 0, os.path.abspath("../../../../..") +) # Adds the parent directory to the system path + +from litellm.llms.digitalocean.chat.transformation import DigitalOceanConfig + +DO_ENDPOINT_PATH = "/api/v1/chat/completions" +DO_BASE_URL = "https://api.digitalocean.com" + +@pytest.fixture +def config(): + return DigitalOceanConfig() + +def test_validate_environment_sets_headers(monkeypatch, config): + monkeypatch.setenv("DIGITALOCEAN_API_KEY", "test-key") + headers = {} + result = config.validate_environment( + headers=headers, + model="digitalocean/test-model", + messages=[], + optional_params={}, + litellm_params={}, + api_key=None, + api_base=None, + ) + assert result["Authorization"] == "Bearer test-key" + assert result["Content-Type"] == "application/json" + +def test_get_complete_url(config): + url = config.get_complete_url( + api_base=DO_BASE_URL, + api_key="test-key", + model="digitalocean/test-model", + optional_params={}, + litellm_params={}, + stream=False, + ) + assert url == f"{DO_BASE_URL}{DO_ENDPOINT_PATH}" + +def test_transform_messages_handles_dicts_only(config): + messages = [ + {"role": "assistant", "content": "Hello!"}, + {"role": "user", "content": "Hi!"}, + ] + out = config._transform_messages(messages, model="digitalocean/test-model") + assert out[0]["role"] == "assistant" + assert out[0]["content"] == "Hello!" + assert out[1]["role"] == "user" + assert out[1]["content"] == "Hi!" + +def test_get_openai_compatible_provider_info_env(monkeypatch, config): + monkeypatch.setenv("DIGITALOCEAN_AGENT_ENDPOINT", DO_BASE_URL) + monkeypatch.setenv("DIGITALOCEAN_API_KEY", "env-key") + api_base, api_key = config._get_openai_compatible_provider_info(None, None) + assert api_base == DO_BASE_URL + assert api_key == "env-key" \ No newline at end of file diff --git a/ui/litellm-dashboard/out/assets/logos/digitalocean.svg b/ui/litellm-dashboard/out/assets/logos/digitalocean.svg new file mode 100644 index 00000000000..7e99cdda8fd --- /dev/null +++ b/ui/litellm-dashboard/out/assets/logos/digitalocean.svg @@ -0,0 +1,229 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx index f44a374fd8a..8f1640c7d76 100644 --- a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx +++ b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx @@ -327,7 +327,27 @@ const PROVIDER_CREDENTIAL_FIELDS: Record = label: "API Key", type: "password", required: true - }] + }], + [Providers.DigitalOcean]: [ + { + key: "endpoint", + label: "DigitalOcean Endpoint", + placeholder: "https://...", + required: true + }, + + { + key: "base_model", + label: "Base Model", + placeholder: "digitalocean/mistral-nemo" + }, + { + key: "api_key", + label: "DigitalOcean API Key", + type: "password", + required: true + } + ] }; const ProviderSpecificFields: React.FC = ({ @@ -335,7 +355,7 @@ const ProviderSpecificFields: React.FC = ({ uploadProps }) => { const selectedProviderEnum = Providers[selectedProvider as keyof typeof Providers] as Providers; - + // Simply use the fields as defined in PROVIDER_CREDENTIAL_FIELDS const allFields = React.useMemo(() => { return PROVIDER_CREDENTIAL_FIELDS[selectedProviderEnum] || []; @@ -353,7 +373,7 @@ const ProviderSpecificFields: React.FC = ({ className={field.key === "vertex_credentials" ? "mb-0" : undefined} > {field.type === "select" ? ( -