From 05029fdcc77dc7e94b49436c37600a971b589fa9 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 23 Mar 2024 15:53:04 -0700 Subject: [PATCH] feat(vertex_ai_anthropic.py): Add support for claude 3 on vertex ai --- litellm/llms/custom_httpx/http_handler.py | 41 +++++++++ litellm/llms/vertex_ai_anthropic.py | 90 +++++++++++++++++++ ...odel_prices_and_context_window_backup.json | 16 ++++ model_prices_and_context_window.json | 16 ++++ 4 files changed, 163 insertions(+) create mode 100644 litellm/llms/custom_httpx/http_handler.py create mode 100644 litellm/llms/vertex_ai_anthropic.py diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py new file mode 100644 index 00000000000..98fecb5d8ab --- /dev/null +++ b/litellm/llms/custom_httpx/http_handler.py @@ -0,0 +1,41 @@ +import httpx, asyncio +from typing import Optional + + +class AsyncHTTPHandler: + def __init__(self, concurrent_limit=1000): + # Create a client with a connection pool + self.client = httpx.AsyncClient( + limits=httpx.Limits( + max_connections=concurrent_limit, + max_keepalive_connections=concurrent_limit, + ) + ) + + async def close(self): + # Close the client when you're done with it + await self.client.aclose() + + async def get( + self, url: str, params: Optional[dict] = None, headers: Optional[dict] = None + ): + response = await self.client.get(url, params=params, headers=headers) + return response + + async def post( + self, + url: str, + data: Optional[dict] = None, + params: Optional[dict] = None, + headers: Optional[dict] = None, + ): + response = await self.client.post( + url, data=data, params=params, headers=headers + ) + return response + + def __del__(self) -> None: + try: + asyncio.get_running_loop().create_task(self.close()) + except Exception: + pass diff --git a/litellm/llms/vertex_ai_anthropic.py b/litellm/llms/vertex_ai_anthropic.py new file mode 100644 index 00000000000..4a54e087b7c --- /dev/null +++ b/litellm/llms/vertex_ai_anthropic.py @@ -0,0 +1,90 @@ +# What is this? +## Handler file for calling claude-3 on vertex ai +from typing import Callable, Optional, Any, Union, List +import litellm + + +class VertexAIAnthropicConfig: + """ + Reference: https://docs.anthropic.com/claude/reference/messages_post + + Note that the API for Claude on Vertex differs from the Anthropic API documentation in the following ways: + + - `model` is not a valid parameter. The model is instead specified in the Google Cloud endpoint URL. + - `anthropic_version` is a required parameter and must be set to "vertex-2023-10-16". + + The class `VertexAIAnthropicConfig` provides configuration for the VertexAI's Anthropic API interface. Below are the parameters: + + - `max_tokens` Required (integer) max tokens, + - `anthropic_version` Required (string) version of anthropic for bedrock - e.g. "bedrock-2023-05-31" + - `system` Optional (string) the system prompt, conversion from openai format to this is handled in factory.py + - `temperature` Optional (float) The amount of randomness injected into the response + - `top_p` Optional (float) Use nucleus sampling. + - `top_k` Optional (int) Only sample from the top K options for each subsequent token + - `stop_sequences` Optional (List[str]) Custom text sequences that cause the model to stop generating + + Note: Please make sure to modify the default parameters as required for your use case. + """ + + max_tokens: Optional[int] = litellm.max_tokens + anthropic_version: Optional[str] = "bedrock-2023-05-31" + system: Optional[str] = None + temperature: Optional[float] = None + top_p: Optional[float] = None + top_k: Optional[int] = None + stop_sequences: Optional[List[str]] = None + + def __init__( + self, + max_tokens: Optional[int] = None, + anthropic_version: Optional[str] = None, + ) -> None: + locals_ = locals() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + + @classmethod + def get_config(cls): + return { + k: v + for k, v in cls.__dict__.items() + if not k.startswith("__") + and not isinstance( + v, + ( + types.FunctionType, + types.BuiltinFunctionType, + classmethod, + staticmethod, + ), + ) + and v is not None + } + + def get_supported_openai_params(self): + return [ + "max_tokens", + "tools", + "tool_choice", + "stream", + "stop", + "temperature", + "top_p", + ] + + def map_openai_params(self, non_default_params: dict, optional_params: dict): + for param, value in non_default_params.items(): + if param == "max_tokens": + optional_params["max_tokens"] = value + if param == "tools": + optional_params["tools"] = value + if param == "stream": + optional_params["stream"] = value + if param == "stop": + optional_params["stop_sequences"] = value + if param == "temperature": + optional_params["temperature"] = value + if param == "top_p": + optional_params["top_p"] = value + return optional_params diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 7cbece52838..a9fc993c90d 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -904,6 +904,22 @@ "litellm_provider": "vertex_ai-vision-models", "mode": "chat" }, + "vertex_ai/claude-3-sonnet@20240229": { + "max_tokens": 200000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "litellm_provider": "vertex_ai", + "mode": "chat" + }, + "vertex_ai/claude-3-haiku@20240307": { + "max_tokens": 200000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000025, + "output_cost_per_token": 0.00000125, + "litellm_provider": "vertex_ai", + "mode": "chat" + }, "textembedding-gecko": { "max_tokens": 3072, "max_input_tokens": 3072, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 7cbece52838..a9fc993c90d 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -904,6 +904,22 @@ "litellm_provider": "vertex_ai-vision-models", "mode": "chat" }, + "vertex_ai/claude-3-sonnet@20240229": { + "max_tokens": 200000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "litellm_provider": "vertex_ai", + "mode": "chat" + }, + "vertex_ai/claude-3-haiku@20240307": { + "max_tokens": 200000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000025, + "output_cost_per_token": 0.00000125, + "litellm_provider": "vertex_ai", + "mode": "chat" + }, "textembedding-gecko": { "max_tokens": 3072, "max_input_tokens": 3072,