From 68a36600c2f4f25b2b24ff6e6815edd92be87fa4 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 8 Aug 2024 08:03:08 -0700 Subject: [PATCH 01/18] fix use get_file_check_sum --- litellm/litellm_core_utils/core_helpers.py | 22 +++++++++++++++++++++- litellm/utils.py | 13 ++++++------- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 8148147ef95..a9e53531670 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -1,6 +1,9 @@ # What is this? ## Helper utilities -from typing import List, Literal, Optional, Tuple +import os +from typing import BinaryIO, List, Literal, Optional, Tuple + +from litellm._logging import verbose_logger def map_finish_reason( @@ -83,3 +86,20 @@ def _get_parent_otel_span_from_kwargs(kwargs: Optional[dict] = None): return kwargs["litellm_parent_otel_span"] except: return None + + +def get_file_check_sum(_file: BinaryIO): + """ + Helper to safely get file checksum - used as a cache key + """ + try: + file_descriptor = _file.fileno() + file_stat = os.fstat(file_descriptor) + file_size = str(file_stat.st_size) + file_checksum = _file.name + file_size + return file_checksum + except Exception as e: + verbose_logger.error(f"Error getting file_checksum: {(str(e))}") + file_checksum = _file.name + return file_checksum + return file_checksum diff --git a/litellm/utils.py b/litellm/utils.py index a20e9617272..35596c784cf 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -55,7 +55,10 @@ import litellm._service_logger # for storing API inputs, outputs, and metadata import litellm.litellm_core_utils import litellm.litellm_core_utils.json_validation_rule from litellm.caching import DualCache -from litellm.litellm_core_utils.core_helpers import map_finish_reason +from litellm.litellm_core_utils.core_helpers import ( + get_file_check_sum, + map_finish_reason, +) from litellm.litellm_core_utils.exception_mapping_utils import get_error_message from litellm.litellm_core_utils.llm_request_utils import _ensure_extra_body_is_safe from litellm.litellm_core_utils.redact_messages import ( @@ -557,12 +560,8 @@ def function_setup( or call_type == CallTypes.transcription.value ): _file_name: BinaryIO = args[1] if len(args) > 1 else kwargs["file"] - file_name = getattr(_file_name, "name", "audio_file") - file_descriptor = _file_name.fileno() - file_stat = os.fstat(file_descriptor) - file_size = str(file_stat.st_size) - - file_checksum = _file_name.name + file_size + file_checksum = get_file_check_sum(_file=_file_name) + file_name = _file_name.name if "metadata" in kwargs: kwargs["metadata"]["file_checksum"] = file_checksum else: From 7d4e834091e14731eaf42a6f40be4cb90f0611c9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 8 Aug 2024 08:23:29 -0700 Subject: [PATCH 02/18] fix handle case when service logger has no attribute prometheusServicesLogger --- litellm/_service_logger.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/litellm/_service_logger.py b/litellm/_service_logger.py index 5e9ab03cf42..5b9e3b085ab 100644 --- a/litellm/_service_logger.py +++ b/litellm/_service_logger.py @@ -90,7 +90,13 @@ class ServiceLogging(CustomLogger): ) async def init_prometheus_services_logger_if_none(self): - if self.prometheusServicesLogger is None: + """ + initializes prometheusServicesLogger if it is None or no attribute exists on ServiceLogging Object + + """ + if not hasattr(self, "prometheusServicesLogger"): + self.prometheusServicesLogger = PrometheusServicesLogger() + elif self.prometheusServicesLogger is None: self.prometheusServicesLogger = self.prometheusServicesLogger() return From d237e625cacff04e870acf0cc747c3b94d8efe8d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 8 Aug 2024 11:09:31 -0700 Subject: [PATCH 03/18] use v1beta1 when using cached_content --- litellm/llms/vertex_httpx.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/litellm/llms/vertex_httpx.py b/litellm/llms/vertex_httpx.py index 8ab60b197b2..9018a901b0b 100644 --- a/litellm/llms/vertex_httpx.py +++ b/litellm/llms/vertex_httpx.py @@ -881,6 +881,21 @@ class VertexLLM(BaseLLM): return self._credentials.token, self.project_id + def is_using_v1beta1_features(self, optional_params: dict) -> bool: + """ + VertexAI only supports ContextCaching on v1beta1 + + use this helper to decide if request should be sent to v1 or v1beta1 + + Returns v1beta1 if context caching is enabled + Returns v1 in all other cases + """ + if "cached_content" in optional_params: + return True + if "CachedContent" in optional_params: + return True + return False + def _get_token_and_url( self, model: str, @@ -891,6 +906,7 @@ class VertexLLM(BaseLLM): stream: Optional[bool], custom_llm_provider: Literal["vertex_ai", "vertex_ai_beta", "gemini"], api_base: Optional[str], + should_use_v1beta1_features: Optional[bool] = False, ) -> Tuple[Optional[str], str]: """ Internal function. Returns the token and url for the call. @@ -920,12 +936,13 @@ class VertexLLM(BaseLLM): vertex_location = self.get_vertex_region(vertex_region=vertex_location) ### SET RUNTIME ENDPOINT ### + version = "v1beta1" if should_use_v1beta1_features is True else "v1" endpoint = "generateContent" if stream is True: endpoint = "streamGenerateContent" - url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model}:{endpoint}?alt=sse" + url = f"https://{vertex_location}-aiplatform.googleapis.com/{version}/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model}:{endpoint}?alt=sse" else: - url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model}:{endpoint}" + url = f"https://{vertex_location}-aiplatform.googleapis.com/{version}/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model}:{endpoint}" if ( api_base is not None @@ -1055,6 +1072,9 @@ class VertexLLM(BaseLLM): ) -> Union[ModelResponse, CustomStreamWrapper]: stream: Optional[bool] = optional_params.pop("stream", None) # type: ignore + should_use_v1beta1_features = self.is_using_v1beta1_features( + optional_params=optional_params + ) auth_header, url = self._get_token_and_url( model=model, gemini_api_key=gemini_api_key, @@ -1064,6 +1084,7 @@ class VertexLLM(BaseLLM): stream=stream, custom_llm_provider=custom_llm_provider, api_base=api_base, + should_use_v1beta1_features=should_use_v1beta1_features, ) ## TRANSFORMATION ## From 7b6110bbb32ee39b88a78e8f1c978a98132e727a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 8 Aug 2024 11:28:19 -0700 Subject: [PATCH 04/18] add tests to make sure correct vertex ai route is used --- .../tests/test_amazing_vertex_completion.py | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/litellm/tests/test_amazing_vertex_completion.py b/litellm/tests/test_amazing_vertex_completion.py index bad2428fbe2..a7327bde4e7 100644 --- a/litellm/tests/test_amazing_vertex_completion.py +++ b/litellm/tests/test_amazing_vertex_completion.py @@ -1969,3 +1969,58 @@ def test_prompt_factory_nested(): assert isinstance( message["parts"][0]["text"], str ), "'text' value not a string." + + +def test_get_token_url(): + from litellm.llms.vertex_httpx import VertexLLM + + vertex_llm = VertexLLM() + vertex_ai_project = "adroit-crow-413218" + vertex_ai_location = "us-central1" + json_obj = get_vertex_ai_creds_json() + vertex_credentials = json.dumps(json_obj) + + should_use_v1beta1_features = vertex_llm.is_using_v1beta1_features( + optional_params={"cached_content": "hi"} + ) + + assert should_use_v1beta1_features is True + + _, url = vertex_llm._get_token_and_url( + vertex_project=vertex_ai_project, + vertex_location=vertex_ai_location, + vertex_credentials=vertex_credentials, + gemini_api_key="", + custom_llm_provider="vertex_ai_beta", + should_use_v1beta1_features=should_use_v1beta1_features, + api_base=None, + model="", + stream=False, + ) + + print("url=", url) + + assert "/v1beta1/" in url + + should_use_v1beta1_features = vertex_llm.is_using_v1beta1_features( + optional_params={"temperature": 0.1} + ) + + _, url = vertex_llm._get_token_and_url( + vertex_project=vertex_ai_project, + vertex_location=vertex_ai_location, + vertex_credentials=vertex_credentials, + gemini_api_key="", + custom_llm_provider="vertex_ai_beta", + should_use_v1beta1_features=should_use_v1beta1_features, + api_base=None, + model="", + stream=False, + ) + + print("url for normal request", url) + + assert "v1beta1" not in url + assert "/v1/" in url + + pass From 909977ace66d32b6c15ebc2c1684a3a6d68851c9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 8 Aug 2024 11:35:41 -0700 Subject: [PATCH 05/18] add example using litellm proxy with gemini context caching --- litellm/proxy/proxy_config.yaml | 43 ++++--------------- .../tests/test_gemini_context_caching.py | 38 ++++++++++++++++ 2 files changed, 47 insertions(+), 34 deletions(-) create mode 100644 litellm/proxy/tests/test_gemini_context_caching.py diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index d4bddd9a0ad..6279d0e2963 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -3,20 +3,14 @@ model_list: litellm_params: model: openai/fake api_key: fake-key - api_base: https://exampleopenaiendpoint-production.up.railwaz.app/ + api_base: https://exampleopenaiendpoint-production.up.railway.app/ - model_name: fireworks-llama-v3-70b-instruct litellm_params: model: fireworks_ai/accounts/fireworks/models/llama-v3-70b-instruct api_key: "os.environ/FIREWORKS" - # provider specific wildcard routing - - model_name: "anthropic/*" + - model_name: "*" litellm_params: - model: "anthropic/*" - api_key: os.environ/ANTHROPIC_API_KEY - - model_name: "groq/*" - litellm_params: - model: "groq/*" - api_key: os.environ/GROQ_API_KEY + model: "*" - model_name: "*" litellm_params: model: openai/* @@ -25,37 +19,18 @@ model_list: litellm_params: model: mistral/mistral-small-latest api_key: "os.environ/MISTRAL_API_KEY" - - model_name: tts + - model_name: gemini-1.5-pro-001 litellm_params: - model: openai/tts-1 - api_key: "os.environ/OPENAI_API_KEY" - model_info: - mode: audio_speech - - -# for /files endpoints -files_settings: - - custom_llm_provider: azure - api_base: https://exampleopenaiendpoint-production.up.railway.app - api_key: fake-key - api_version: "2023-03-15-preview" - - custom_llm_provider: openai - api_key: os.environ/OPENAI_API_KEY + model: vertex_ai_beta/gemini-1.5-pro-001 + vertex_project: "adroit-crow-413218" + vertex_location: "us-central1" + vertex_credentials: "adroit-crow-413218-a956eef1a2a8.json" + # Add path to service account.json general_settings: master_key: sk-1234 - pass_through_endpoints: - - path: "/v1/rerank" # route you want to add to LiteLLM Proxy Server - target: "https://api.cohere.com/v1/rerank" # URL this route should forward requests to - headers: # headers to forward to this URL - content-type: application/json # (Optional) Extra Headers to pass to this endpoint - accept: application/json - forward_headers: True - litellm_settings: callbacks: ["otel"] # 👈 KEY CHANGE - success_callback: ["prometheus"] - failure_callback: ["prometheus"] \ No newline at end of file diff --git a/litellm/proxy/tests/test_gemini_context_caching.py b/litellm/proxy/tests/test_gemini_context_caching.py new file mode 100644 index 00000000000..40caaf1eafc --- /dev/null +++ b/litellm/proxy/tests/test_gemini_context_caching.py @@ -0,0 +1,38 @@ +import datetime + +import openai +import vertexai +from vertexai.generative_models import Content, Part +from vertexai.preview import caching +from vertexai.preview.generative_models import GenerativeModel + +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") +vertexai.init(project="adroit-crow-413218", location="us-central1") +print("creating cached content") +contents_here: list[Content] = [ + Content(role="user", parts=[Part.from_text("huge string of text here" * 10000)]) +] +cached_content = caching.CachedContent.create( + model_name="gemini-1.5-pro-001", + contents=contents_here, + expire_time=datetime.datetime(2024, 8, 10), +) + +created_Caches = caching.CachedContent.list() + +print("created_Caches contents=", created_Caches) + +response = client.chat.completions.create( # type: ignore + model="gemini-1.5-pro-001", + max_tokens=8192, + messages=[ + { + "role": "user", + "content": "quote all everything above this message", + }, + ], + temperature="0.7", + extra_body={"cached_content": cached_content.resource_name}, +) + +print("response from proxy", response) From 8ad5a402832e5705830109e2f5c21798f71d7894 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 8 Aug 2024 11:45:46 -0700 Subject: [PATCH 06/18] doc on using litellm proxy with vertex ai content caching --- docs/my-website/docs/providers/vertex.md | 73 ++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/docs/my-website/docs/providers/vertex.md b/docs/my-website/docs/providers/vertex.md index e68bfd129d8..a0e243a656b 100644 --- a/docs/my-website/docs/providers/vertex.md +++ b/docs/my-website/docs/providers/vertex.md @@ -427,6 +427,79 @@ print(resp) ``` +### **Context Caching** + +Use Vertex AI Context Caching + +[**Relevant VertexAI Docs**](https://cloud.google.com/vertex-ai/generative-ai/docs/context-cache/context-cache-overview) + + + + + +1. Add model to config.yaml +```yaml +model_list: + - model_name: gemini-1.5-pro-001 + litellm_params: + model: vertex_ai_beta/gemini-1.5-pro-001 + vertex_project: "project-id" + vertex_location: "us-central1" + vertex_credentials: "/path/to/service_account.json" # [OPTIONAL] Do this OR `!gcloud auth application-default login` - run this to add vertex credentials to your env +``` + +2. Start Proxy + +``` +$ litellm --config /path/to/config.yaml +``` + +3. Make Request! + +```python +import datetime +import openai +import vertexai +from vertexai.generative_models import Content, Part +from vertexai.preview import caching +from vertexai.preview.generative_models import GenerativeModel + +# use Vertex AI SDK to create CachedContent +vertexai.init(project="adroit-crow-413218", location="us-central1") +print("creating cached content") +contents_here: list[Content] = [ + Content(role="user", parts=[Part.from_text("huge string of text here" * 10000)]) +] +cached_content = caching.CachedContent.create( + model_name="gemini-1.5-pro-001", + contents=contents_here, + expire_time=datetime.datetime(2024, 8, 10), +) + + +# use OpenAI SDK to send a request to LiteLLM Proxy +# base_url is litellm proxy server and api_key is api key to litellm proxy +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") +response = client.chat.completions.create( + model="gemini-1.5-pro-001", + messages=[ + { + "role": "user", + "content": "hello!", + }, + ], + temperature="0.7", + extra_body={"cached_content": cached_content.resource_name}, +) + +print("response from proxy", response) + +``` + + + + + ## Pre-requisites * `pip install google-cloud-aiplatform` (pre-installed on proxy docker image) * Authentication: From 5da4c27e8d569b760ffccad7392f1e0f88f7d31f Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 8 Aug 2024 13:05:03 -0700 Subject: [PATCH 07/18] fix(internal_user_endpoints.py): expose new 'internal_user_budget_duration' flag Relevant to - https://github.com/BerriAI/litellm/issues/5106 --- litellm/__init__.py | 1 + litellm/proxy/_experimental/out/404.html | 1 - litellm/proxy/_experimental/out/model_hub.html | 1 - litellm/proxy/_experimental/out/onboarding.html | 1 - litellm/proxy/_new_secret_config.yaml | 5 +++-- .../proxy/management_endpoints/internal_user_endpoints.py | 4 ++++ 6 files changed, 8 insertions(+), 5 deletions(-) delete mode 100644 litellm/proxy/_experimental/out/404.html delete mode 100644 litellm/proxy/_experimental/out/model_hub.html delete mode 100644 litellm/proxy/_experimental/out/onboarding.html diff --git a/litellm/__init__.py b/litellm/__init__.py index 0429c654748..8d0e1b31495 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -261,6 +261,7 @@ default_user_params: Optional[Dict] = None default_team_settings: Optional[List] = None max_user_budget: Optional[float] = None max_internal_user_budget: Optional[float] = None +internal_user_budget_duration: Optional[str] = None max_end_user_budget: Optional[float] = None #### REQUEST PRIORITIZATION #### priority_reservation: Optional[Dict[str, float]] = None diff --git a/litellm/proxy/_experimental/out/404.html b/litellm/proxy/_experimental/out/404.html deleted file mode 100644 index 4428091a354..00000000000 --- a/litellm/proxy/_experimental/out/404.html +++ /dev/null @@ -1 +0,0 @@ -404: This page could not be found.LiteLLM Dashboard

404

This page could not be found.

\ No newline at end of file diff --git a/litellm/proxy/_experimental/out/model_hub.html b/litellm/proxy/_experimental/out/model_hub.html deleted file mode 100644 index 700debc2f1a..00000000000 --- a/litellm/proxy/_experimental/out/model_hub.html +++ /dev/null @@ -1 +0,0 @@ -LiteLLM Dashboard \ No newline at end of file diff --git a/litellm/proxy/_experimental/out/onboarding.html b/litellm/proxy/_experimental/out/onboarding.html deleted file mode 100644 index 16f3f259334..00000000000 --- a/litellm/proxy/_experimental/out/onboarding.html +++ /dev/null @@ -1 +0,0 @@ -LiteLLM Dashboard \ No newline at end of file diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 35ef59c965d..0369ee7e3e3 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -11,5 +11,6 @@ model_list: model: "gpt-4o" litellm_settings: - enable_json_schema_validation: true - fallbacks: [{"gpt-3.5-turbo": ["gpt-4", "gpt-4o"]}] + max_internal_user_budget: 0.001 + internal_user_budget_duration: "5m" + diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index f54eca9a2fc..173b205fc30 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -91,6 +91,10 @@ async def new_user( if litellm.max_internal_user_budget is not None: data_json["max_budget"] = litellm.max_internal_user_budget + if "budget_duration" in data_json and data_json["budget_duration"] is None: + if litellm.internal_user_budget_duration is not None: + data_json["budget_duration"] = litellm.internal_user_budget_duration + response = await generate_key_helper_fn(request_type="user", **data_json) # Admin UI Logic From b65a27035e598bf2ba31f193bbf3a9815e4f9a0e Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 8 Aug 2024 13:42:15 -0700 Subject: [PATCH 08/18] test(test_langsmith.py): skip flaky test --- litellm/tests/test_langsmith.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/litellm/tests/test_langsmith.py b/litellm/tests/test_langsmith.py index 68182e73d52..9575c18da7b 100644 --- a/litellm/tests/test_langsmith.py +++ b/litellm/tests/test_langsmith.py @@ -22,6 +22,7 @@ litellm.set_verbose = True import time +@pytest.mark.skip(reason="Flaky test. covered by unit tests on custom logger.") @pytest.mark.asyncio() async def test_async_langsmith_logging(): try: @@ -89,6 +90,7 @@ async def test_async_langsmith_logging(): # test_langsmith_logging() +@pytest.mark.skip(reason="Flaky test. covered by unit tests on custom logger.") def test_async_langsmith_logging_with_metadata(): try: litellm.success_callback = ["langsmith"] @@ -111,6 +113,7 @@ def test_async_langsmith_logging_with_metadata(): print(e) +@pytest.mark.skip(reason="Flaky test. covered by unit tests on custom logger.") @pytest.mark.parametrize("sync_mode", [False, True]) @pytest.mark.asyncio async def test_async_langsmith_logging_with_streaming_and_metadata(sync_mode): From 0d662368acbd1ce6134ae49966207d0ce45e0f88 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 8 Aug 2024 15:32:07 -0700 Subject: [PATCH 09/18] add native cachedContents endpoint --- litellm/llms/fine_tuning_apis/vertex_ai.py | 2 ++ .../vertex_ai_endpoints/vertex_endpoints.py | 27 +++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/litellm/llms/fine_tuning_apis/vertex_ai.py b/litellm/llms/fine_tuning_apis/vertex_ai.py index 5f96f04831a..beebb0d565f 100644 --- a/litellm/llms/fine_tuning_apis/vertex_ai.py +++ b/litellm/llms/fine_tuning_apis/vertex_ai.py @@ -278,6 +278,8 @@ class VertexFineTuningAPI(VertexLLM): url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}{request_route}" elif "countTokens" in request_route: url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}{request_route}" + elif "cachedContents" in request_route: + url = f"https://{vertex_location}-aiplatform.googleapis.com/v1beta1/projects/{vertex_project}/locations/{vertex_location}{request_route}" else: raise ValueError(f"Unsupported Vertex AI request route: {request_route}") if self.async_handler is None: diff --git a/litellm/proxy/vertex_ai_endpoints/vertex_endpoints.py b/litellm/proxy/vertex_ai_endpoints/vertex_endpoints.py index b8c04583c3f..1bfb1c2a098 100644 --- a/litellm/proxy/vertex_ai_endpoints/vertex_endpoints.py +++ b/litellm/proxy/vertex_ai_endpoints/vertex_endpoints.py @@ -303,3 +303,30 @@ async def vertex_cancel_fine_tuning_job( return response except Exception as e: raise exception_handler(e) from e + + +@router.post( + "/vertex-ai/cachedContents", + dependencies=[Depends(user_api_key_auth)], + tags=["Vertex AI endpoints"], +) +async def vertex_create_add_cached_content( + request: Request, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + this is a pass through endpoint for the Vertex AI API. /cachedContents endpoint + + Vertex API Reference: https://cloud.google.com/vertex-ai/generative-ai/docs/context-cache/context-cache-create#create-context-cache-sample-drest + + it uses the vertex ai credentials on the proxy and forwards to vertex ai api + """ + try: + response = await execute_post_vertex_ai_request( + request=request, + route="/cachedContents", + ) + return response + except Exception as e: + raise exception_handler(e) from e From 1bf8af5cb1da7ce7ca17a53611f38611dae8653d Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 8 Aug 2024 15:55:01 -0700 Subject: [PATCH 10/18] fix(anthropic.py): fix translation from /v1/messages format to openai format --- litellm/llms/anthropic.py | 3 + litellm/proxy/_new_secret_config.yaml | 11 +-- litellm/proxy/proxy_server.py | 8 +- litellm/tests/test_anthropic_completion.py | 93 ++++++++++++++++++++++ 4 files changed, 105 insertions(+), 10 deletions(-) diff --git a/litellm/llms/anthropic.py b/litellm/llms/anthropic.py index 78888cf4adc..20c74fb3d50 100644 --- a/litellm/llms/anthropic.py +++ b/litellm/llms/anthropic.py @@ -287,6 +287,9 @@ class AnthropicConfig: if user_message is not None: new_messages.append(user_message) + if len(new_user_content_list) > 0: + new_messages.append({"role": "user", "content": new_user_content_list}) + if len(tool_message_list) > 0: new_messages.extend(tool_message_list) diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 0369ee7e3e3..6c50147340f 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -1,14 +1,7 @@ model_list: - - model_name: "gpt-3.5-turbo" + - model_name: "claude-3-5-sonnet-20240620" litellm_params: - model: "gpt-3.5-turbo" - - model_name: "gpt-4" - litellm_params: - model: "gpt-4" - api_key: "bad_key" - - model_name: "gpt-4o" - litellm_params: - model: "gpt-4o" + model: "claude-3-5-sonnet-20240620" litellm_settings: max_internal_user_budget: 0.001 diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 299b390b9a0..c9ea1f7a4be 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -5374,7 +5374,13 @@ async def anthropic_response( litellm.adapters = [{"id": "anthropic", "adapter": anthropic_adapter}] global user_temperature, user_request_timeout, user_max_tokens, user_api_base - data: dict = {**anthropic_data, "adapter_id": "anthropic"} + body = await request.body() + body_str = body.decode() + try: + request_data: dict = ast.literal_eval(body_str) + except Exception: + request_data = json.loads(body_str) + data: dict = {**request_data, "adapter_id": "anthropic"} try: data["model"] = ( general_settings.get("completion_model", None) # server default diff --git a/litellm/tests/test_anthropic_completion.py b/litellm/tests/test_anthropic_completion.py index 3611a44caef..20260556ec2 100644 --- a/litellm/tests/test_anthropic_completion.py +++ b/litellm/tests/test_anthropic_completion.py @@ -183,3 +183,96 @@ async def test_anthropic_router_completion_e2e(): assert isinstance(response, AnthropicResponse) assert response.model == "gpt-3.5-turbo" + + +def test_anthropic_tool_calling_translation(): + kwargs = { + "model": "claude-3-5-sonnet-20240620", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Would development of a software platform be under ASC 350-40 or ASC 985?", + } + ], + }, + { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "37d6f703-cbcc-497d-95a1-2aa24a114adc", + "name": "TaskPlanningTool", + "input": { + "completed_steps": [], + "next_steps": [ + { + "tool_name": "AccountingResearchTool", + "description": "Research ASC 350-40 to understand its scope and applicability to software development.", + }, + { + "tool_name": "AccountingResearchTool", + "description": "Research ASC 985 to understand its scope and applicability to software development.", + }, + { + "tool_name": "AccountingResearchTool", + "description": "Compare the scopes of ASC 350-40 and ASC 985 to determine which is more applicable to software platform development.", + }, + ], + "learnings": [], + "potential_issues": [ + "The distinction between the two standards might not be clear-cut for all types of software development.", + "There might be specific circumstances or details about the software platform that could affect which standard applies.", + ], + "missing_info": [ + "Specific details about the type of software platform being developed (e.g., for internal use or for sale).", + "Whether the entity developing the software is also the end-user or if it's being developed for external customers.", + ], + "done": False, + "required_formatting": None, + }, + } + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "eb7023b1-5ee8-43b8-b90f-ac5a23d37c31", + "content": { + "completed_steps": [], + "next_steps": [ + { + "tool_name": "AccountingResearchTool", + "description": "Research ASC 350-40 to understand its scope and applicability to software development.", + }, + { + "tool_name": "AccountingResearchTool", + "description": "Research ASC 985 to understand its scope and applicability to software development.", + }, + { + "tool_name": "AccountingResearchTool", + "description": "Compare the scopes of ASC 350-40 and ASC 985 to determine which is more applicable to software platform development.", + }, + ], + "formatting_step": None, + }, + } + ], + }, + ], + } + + from litellm.adapters.anthropic_adapter import anthropic_adapter + + translated_params = anthropic_adapter.translate_completion_input_params( + kwargs=kwargs + ) + + print(translated_params["messages"]) + + assert len(translated_params["messages"]) > 0 + assert translated_params["messages"][1]["role"] == "user" From cae941f4c071e16732b21cc7f67839cceb576189 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 8 Aug 2024 15:56:46 -0700 Subject: [PATCH 11/18] auto update the model in cachedContents request --- litellm/llms/fine_tuning_apis/vertex_ai.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/litellm/llms/fine_tuning_apis/vertex_ai.py b/litellm/llms/fine_tuning_apis/vertex_ai.py index beebb0d565f..ffdb82c5b47 100644 --- a/litellm/llms/fine_tuning_apis/vertex_ai.py +++ b/litellm/llms/fine_tuning_apis/vertex_ai.py @@ -279,6 +279,12 @@ class VertexFineTuningAPI(VertexLLM): elif "countTokens" in request_route: url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}{request_route}" elif "cachedContents" in request_route: + _model = request_data.get("model") + if _model is not None and "/publishers/google/models/" not in _model: + request_data["model"] = ( + f"projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{_model}" + ) + url = f"https://{vertex_location}-aiplatform.googleapis.com/v1beta1/projects/{vertex_project}/locations/{vertex_location}{request_route}" else: raise ValueError(f"Unsupported Vertex AI request route: {request_route}") From a3dd3a19fa78d0e62c8cb02d90d459f613f29856 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 8 Aug 2024 16:06:23 -0700 Subject: [PATCH 12/18] docs cachedContent endpoint --- docs/my-website/docs/providers/vertex.md | 66 +++++++++++++------ .../tests/test_gemini_context_caching.py | 54 +++++++++------ 2 files changed, 81 insertions(+), 39 deletions(-) diff --git a/docs/my-website/docs/providers/vertex.md b/docs/my-website/docs/providers/vertex.md index a0e243a656b..e65a104ef47 100644 --- a/docs/my-website/docs/providers/vertex.md +++ b/docs/my-website/docs/providers/vertex.md @@ -440,12 +440,20 @@ Use Vertex AI Context Caching 1. Add model to config.yaml ```yaml model_list: + # used for /chat/completions, /completions, /embeddings endpoints - model_name: gemini-1.5-pro-001 litellm_params: model: vertex_ai_beta/gemini-1.5-pro-001 vertex_project: "project-id" vertex_location: "us-central1" vertex_credentials: "/path/to/service_account.json" # [OPTIONAL] Do this OR `!gcloud auth application-default login` - run this to add vertex credentials to your env + +# used for the /cachedContent and vertexAI native endpoints +default_vertex_config: + vertex_project: "adroit-crow-413218" + vertex_location: "us-central1" + vertex_credentials: "adroit-crow-413218-a956eef1a2a8.json" # Add path to service account.json + ``` 2. Start Proxy @@ -456,40 +464,58 @@ $ litellm --config /path/to/config.yaml 3. Make Request! +- First create a cachedContents object by calling the Vertex `cachedContents` endpoint. [VertexAI API Ref for cachedContents endpoint](https://cloud.google.com/vertex-ai/generative-ai/docs/context-cache/context-cache-create#create-context-cache-sample-drest). (LiteLLM proxy forwards the `/cachedContents` request to the VertexAI API) +- Use the `cachedContents` object in your /chat/completions request to vertexAI + ```python import datetime import openai -import vertexai -from vertexai.generative_models import Content, Part -from vertexai.preview import caching -from vertexai.preview.generative_models import GenerativeModel +import httpx -# use Vertex AI SDK to create CachedContent -vertexai.init(project="adroit-crow-413218", location="us-central1") +# Set Litellm proxy variables here +LITELLM_BASE_URL = "http://0.0.0.0:4000" +LITELLM_PROXY_API_KEY = "sk-1234" + +client = openai.OpenAI(api_key=LITELLM_PROXY_API_KEY, base_url=LITELLM_BASE_URL) +httpx_client = httpx.Client(timeout=30) + +################################ +# First create a cachedContents object +# this request gets forwarded as is to: https://cloud.google.com/vertex-ai/generative-ai/docs/context-cache/context-cache-create#create-context-cache-sample-drest print("creating cached content") -contents_here: list[Content] = [ - Content(role="user", parts=[Part.from_text("huge string of text here" * 10000)]) -] -cached_content = caching.CachedContent.create( - model_name="gemini-1.5-pro-001", - contents=contents_here, - expire_time=datetime.datetime(2024, 8, 10), +create_cache = httpx_client.post( + url=f"{LITELLM_BASE_URL}/vertex-ai/cachedContents", + headers = {"Authorization": f"Bearer {LITELLM_PROXY_API_KEY}"}, + json = { + "model": "gemini-1.5-pro-001", + "contents": [ + { + "role": "user", + "parts": [{ + "text": "This is sample text to demonstrate explicit caching."*4000 + }] + } + ], + } ) +print("response from create_cache", create_cache) +create_cache_response = create_cache.json() +print("json from create_cache", create_cache_response) +cached_content_name = create_cache_response["name"] - -# use OpenAI SDK to send a request to LiteLLM Proxy -# base_url is litellm proxy server and api_key is api key to litellm proxy -client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") -response = client.chat.completions.create( +################################# +# Use the `cachedContents` object in your /chat/completions +response = client.chat.completions.create( # type: ignore model="gemini-1.5-pro-001", + max_tokens=8192, messages=[ { "role": "user", - "content": "hello!", + "content": "what is the sample text about?", }, ], temperature="0.7", - extra_body={"cached_content": cached_content.resource_name}, + extra_body={"cached_content": cached_content_name}, # 👈 key change ) print("response from proxy", response) diff --git a/litellm/proxy/tests/test_gemini_context_caching.py b/litellm/proxy/tests/test_gemini_context_caching.py index 40caaf1eafc..6ee143dba16 100644 --- a/litellm/proxy/tests/test_gemini_context_caching.py +++ b/litellm/proxy/tests/test_gemini_context_caching.py @@ -1,38 +1,54 @@ import datetime +import httpx import openai -import vertexai -from vertexai.generative_models import Content, Part -from vertexai.preview import caching -from vertexai.preview.generative_models import GenerativeModel -client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") -vertexai.init(project="adroit-crow-413218", location="us-central1") +# Set Litellm proxy variables here +LITELLM_BASE_URL = "http://0.0.0.0:4000" +LITELLM_PROXY_API_KEY = "sk-1234" + +client = openai.OpenAI(api_key=LITELLM_PROXY_API_KEY, base_url=LITELLM_BASE_URL) +httpx_client = httpx.Client(timeout=30) + +################################ +# First create a cachedContents object print("creating cached content") -contents_here: list[Content] = [ - Content(role="user", parts=[Part.from_text("huge string of text here" * 10000)]) -] -cached_content = caching.CachedContent.create( - model_name="gemini-1.5-pro-001", - contents=contents_here, - expire_time=datetime.datetime(2024, 8, 10), +create_cache = httpx_client.post( + url=f"{LITELLM_BASE_URL}/vertex-ai/cachedContents", + headers={"Authorization": f"Bearer {LITELLM_PROXY_API_KEY}"}, + json={ + "model": "gemini-1.5-pro-001", + "contents": [ + { + "role": "user", + "parts": [ + { + "text": "This is sample text to demonstrate explicit caching." + * 4000 + } + ], + } + ], + }, ) +print("response from create_cache", create_cache) +create_cache_response = create_cache.json() +print("json from create_cache", create_cache_response) +cached_content_name = create_cache_response["name"] -created_Caches = caching.CachedContent.list() - -print("created_Caches contents=", created_Caches) - +################################# +# Use the `cachedContents` object in your /chat/completions response = client.chat.completions.create( # type: ignore model="gemini-1.5-pro-001", max_tokens=8192, messages=[ { "role": "user", - "content": "quote all everything above this message", + "content": "what is the sample text about?", }, ], temperature="0.7", - extra_body={"cached_content": cached_content.resource_name}, + extra_body={"cached_content": cached_content_name}, # 👈 key change ) print("response from proxy", response) From d78c38f8e70357cca54958967d8468b400ebb2b1 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 8 Aug 2024 16:07:14 -0700 Subject: [PATCH 13/18] docs vertex --- docs/my-website/docs/providers/vertex.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/providers/vertex.md b/docs/my-website/docs/providers/vertex.md index e65a104ef47..4ae9880ac48 100644 --- a/docs/my-website/docs/providers/vertex.md +++ b/docs/my-website/docs/providers/vertex.md @@ -446,7 +446,7 @@ model_list: model: vertex_ai_beta/gemini-1.5-pro-001 vertex_project: "project-id" vertex_location: "us-central1" - vertex_credentials: "/path/to/service_account.json" # [OPTIONAL] Do this OR `!gcloud auth application-default login` - run this to add vertex credentials to your env + vertex_credentials: "adroit-crow-413218-a956eef1a2a8.json" # Add path to service account.json # used for the /cachedContent and vertexAI native endpoints default_vertex_config: From 1924f11b4cf4a852ad3e12babc841683ad147617 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 8 Aug 2024 16:07:26 -0700 Subject: [PATCH 14/18] add default_vertex_config --- litellm/proxy/proxy_config.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 6279d0e2963..b0a5a073e65 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -27,6 +27,10 @@ model_list: vertex_credentials: "adroit-crow-413218-a956eef1a2a8.json" # Add path to service account.json +default_vertex_config: + vertex_project: "adroit-crow-413218" + vertex_location: "us-central1" + vertex_credentials: "adroit-crow-413218-a956eef1a2a8.json" # Add path to service account.json general_settings: From f179759672c56dbc76200186cd937707561cba21 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 8 Aug 2024 16:12:36 -0700 Subject: [PATCH 15/18] docs vertex ai --- docs/my-website/docs/vertex_ai.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/my-website/docs/vertex_ai.md b/docs/my-website/docs/vertex_ai.md index d9c8616a0bf..b895fe8c7f6 100644 --- a/docs/my-website/docs/vertex_ai.md +++ b/docs/my-website/docs/vertex_ai.md @@ -1,5 +1,11 @@ # [BETA] Vertex AI Endpoints +:::tip + +Looking for the Unified API (OpenAI format) for VertexAI ? [Go here - using vertexAI with LiteLLM SDK or LiteLLM Proxy Server](../docs/providers/vertex.md) + +::: + ## Supported API Endpoints - Gemini API From 84c05a57d6c17177409850db060f24f8ebe2220a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 8 Aug 2024 17:00:52 -0700 Subject: [PATCH 16/18] docs use (LLM Gateway) in some places --- docs/my-website/docs/index.md | 2 +- docs/my-website/docs/proxy/quick_start.md | 2 +- docs/my-website/sidebars.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/my-website/docs/index.md b/docs/my-website/docs/index.md index 2417b0cff43..4c1cdd017dc 100644 --- a/docs/my-website/docs/index.md +++ b/docs/my-website/docs/index.md @@ -14,7 +14,7 @@ https://github.com/BerriAI/litellm ## How to use LiteLLM You can use litellm through either: -1. [LiteLLM Proxy Server](#openai-proxy) - Server to call 100+ LLMs, load balance, cost tracking across projects +1. [LiteLLM Proxy Server](#openai-proxy) - Server (LLM Gateway) to call 100+ LLMs, load balance, cost tracking across projects 2. [LiteLLM python SDK](#basic-usage) - Python Client to call 100+ LLMs, load balance, cost tracking ### When to use LiteLLM Proxy Server diff --git a/docs/my-website/docs/proxy/quick_start.md b/docs/my-website/docs/proxy/quick_start.md index 9da860b0df9..2049f6bb263 100644 --- a/docs/my-website/docs/proxy/quick_start.md +++ b/docs/my-website/docs/proxy/quick_start.md @@ -5,7 +5,7 @@ import TabItem from '@theme/TabItem'; # Quick Start Quick start CLI, Config, Docker -LiteLLM Server manages: +LiteLLM Server (LLM Gateway) manages: * **Unified Interface**: Calling 100+ LLMs [Huggingface/Bedrock/TogetherAI/etc.](#other-supported-models) in the OpenAI `ChatCompletions` & `Completions` format * **Cost tracking**: Authentication, Spend Tracking & Budgets [Virtual Keys](https://docs.litellm.ai/docs/proxy/virtual_keys) diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index f84a43c56fe..58eebdc2cf8 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -24,7 +24,7 @@ const sidebars = { link: { type: "generated-index", title: "💥 LiteLLM Proxy Server", - description: `OpenAI Proxy Server to call 100+ LLMs in a unified interface & track spend, set budgets per virtual key/user`, + description: `OpenAI Proxy Server (LLM Gateway) to call 100+ LLMs in a unified interface & track spend, set budgets per virtual key/user`, slug: "/simple_proxy", }, items: [ From 68fdf9de2da9049b8dce95e0c565eedebf093a09 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 8 Aug 2024 17:06:29 -0700 Subject: [PATCH 17/18] docs readme --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2153ae948ed..e13732000e9 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@

Call all LLM APIs using the OpenAI format [Bedrock, Huggingface, VertexAI, TogetherAI, Azure, OpenAI, Groq etc.]

-

LiteLLM Proxy Server | Hosted Proxy (Preview) | Enterprise Tier

+

LiteLLM Proxy Server (LLM Gateway) | Hosted Proxy (Preview) | Enterprise Tier

PyPI Version @@ -35,9 +35,9 @@ LiteLLM manages: - Translate inputs to provider's `completion`, `embedding`, and `image_generation` endpoints - [Consistent output](https://docs.litellm.ai/docs/completion/output), text responses will always be available at `['choices'][0]['message']['content']` - Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing) -- Set Budgets & Rate limits per project, api key, model [LiteLLM Proxy Server](https://docs.litellm.ai/docs/simple_proxy) +- Set Budgets & Rate limits per project, api key, model [LiteLLM Proxy Server (LLM Gateway)](https://docs.litellm.ai/docs/simple_proxy) -[**Jump to OpenAI Proxy Docs**](https://github.com/BerriAI/litellm?tab=readme-ov-file#openai-proxy---docs)
+[**Jump to LiteLLM Proxy (LLM Gateway) Docs**](https://github.com/BerriAI/litellm?tab=readme-ov-file#openai-proxy---docs)
[**Jump to Supported LLM Providers**](https://github.com/BerriAI/litellm?tab=readme-ov-file#supported-providers-docs) 🚨 **Stable Release:** Use docker images with the `-stable` tag. These have undergone 12 hour load tests, before being published. @@ -134,7 +134,7 @@ litellm.success_callback = ["lunary", "langfuse", "athina", "helicone"] # log in response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}]) ``` -# OpenAI Proxy - ([Docs](https://docs.litellm.ai/docs/simple_proxy)) +# LiteLLM Proxy Server (LLM Gateway) - ([Docs](https://docs.litellm.ai/docs/simple_proxy)) Track spend + Load Balance across multiple projects From da0413f0a4f4e9d5f1a5f7a0449cc5a78348903a Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 8 Aug 2024 17:06:36 -0700 Subject: [PATCH 18/18] fix(factory.py): handle openai function message having tool call id --- litellm/llms/prompt_templates/factory.py | 3 +- litellm/tests/test_completion.py | 3 -- litellm/tests/test_prompt_factory.py | 41 ++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index 2cadfed6eb3..7fce1929678 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -1135,8 +1135,9 @@ def convert_to_anthropic_tool_result(message: dict) -> AnthropicMessagesToolResu return anthropic_tool_result if message["role"] == "function": content = message.get("content") # type: ignore + tool_call_id = message.get("tool_call_id") or str(uuid.uuid4()) anthropic_tool_result = AnthropicMessagesToolResultParam( - type="tool_result", tool_use_id=str(uuid.uuid4()), content=content + type="tool_result", tool_use_id=tool_call_id, content=content ) return anthropic_tool_result diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 693110bc0da..d2ecd609272 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -4405,6 +4405,3 @@ def test_moderation(): output = response.results[0] print(output) return output - - -# test_moderation() diff --git a/litellm/tests/test_prompt_factory.py b/litellm/tests/test_prompt_factory.py index 0dc0dbca20b..f7a715a2204 100644 --- a/litellm/tests/test_prompt_factory.py +++ b/litellm/tests/test_prompt_factory.py @@ -219,3 +219,44 @@ def test_base64_image_input(url, expected_media_type): response = convert_to_anthropic_image_obj(openai_image_url=url) assert response["media_type"] == expected_media_type + + +def test_anthropic_messages_tool_call(): + messages = [ + { + "role": "user", + "content": "Would development of a software platform be under ASC 350-40 or ASC 985?", + }, + { + "role": "assistant", + "content": "", + "tool_call_id": "bc8cb4b6-88c4-4138-8993-3a9d9cd51656", + "tool_calls": [ + { + "id": "bc8cb4b6-88c4-4138-8993-3a9d9cd51656", + "function": { + "arguments": '{"completed_steps": [], "next_steps": [{"tool_name": "AccountingResearchTool", "description": "Research ASC 350-40 to understand its scope and applicability to software development."}, {"tool_name": "AccountingResearchTool", "description": "Research ASC 985 to understand its scope and applicability to software development."}, {"tool_name": "AccountingResearchTool", "description": "Compare the scopes of ASC 350-40 and ASC 985 to determine which is more applicable to software platform development."}], "learnings": [], "potential_issues": ["The distinction between the two standards might not be clear-cut for all types of software development.", "There might be specific circumstances or details about the software platform that could affect which standard applies."], "missing_info": ["Specific details about the type of software platform being developed (e.g., for internal use or for sale).", "Whether the entity developing the software is also the end-user or if it\'s being developed for external customers."], "done": false, "required_formatting": null}', + "name": "TaskPlanningTool", + }, + "type": "function", + } + ], + }, + { + "role": "function", + "content": '{"completed_steps":[],"next_steps":[{"tool_name":"AccountingResearchTool","description":"Research ASC 350-40 to understand its scope and applicability to software development."},{"tool_name":"AccountingResearchTool","description":"Research ASC 985 to understand its scope and applicability to software development."},{"tool_name":"AccountingResearchTool","description":"Compare the scopes of ASC 350-40 and ASC 985 to determine which is more applicable to software platform development."}],"formatting_step":null}', + "name": "TaskPlanningTool", + "tool_call_id": "bc8cb4b6-88c4-4138-8993-3a9d9cd51656", + }, + ] + + translated_messages = anthropic_messages_pt( + messages, model="claude-3-sonnet-20240229", llm_provider="anthropic" + ) + + print(translated_messages) + + assert ( + translated_messages[-1]["content"][0]["tool_use_id"] + == "bc8cb4b6-88c4-4138-8993-3a9d9cd51656" + )