From 19474c3f0fc9d1e0d9328736657a66cf7994f71f Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 3 Aug 2024 11:46:39 -0700 Subject: [PATCH 01/27] support v1/projects/tuningJobs --- .../proxy/fine_tuning_endpoints/endpoints.py | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/litellm/proxy/fine_tuning_endpoints/endpoints.py b/litellm/proxy/fine_tuning_endpoints/endpoints.py index cda226b5aa7..c2d89dd251f 100644 --- a/litellm/proxy/fine_tuning_endpoints/endpoints.py +++ b/litellm/proxy/fine_tuning_endpoints/endpoints.py @@ -429,3 +429,72 @@ async def retrieve_fine_tuning_job( param=getattr(e, "param", "None"), code=getattr(e, "status_code", 500), ) + + +@router.post( + "/v1/projects/tuningJobs", + dependencies=[Depends(user_api_key_auth)], + tags=["fine-tuning"], + summary="✨ (Enterprise) Create Fine-Tuning Jobs", +) +async def vertex_create_fine_tuning_job( + request: Request, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + this is a pass through endpoint for the Vertex AI API. /tuningJobs endpoint + + it uses the vertex ai credentials on the proxy and forwards to vertex ai api + """ + try: + from litellm.fine_tuning.main import vertex_fine_tuning_apis_instance + from litellm.proxy.proxy_server import ( + add_litellm_data_to_request, + general_settings, + get_custom_headers, + premium_user, + proxy_config, + proxy_logging_obj, + version, + ) + + # get configs for custom_llm_provider + llm_provider_config = get_fine_tuning_provider_config( + custom_llm_provider="vertex_ai" + ) + + vertex_project = llm_provider_config.get("vertex_project", None) + vertex_location = llm_provider_config.get("vertex_location", None) + vertex_credentials = llm_provider_config.get("vertex_credentials", None) + request_data_json = await request.json() + response = await vertex_fine_tuning_apis_instance.pass_through_vertex_ai_fine_tuning_job( + request_data=request_data_json, + vertex_project=vertex_project, + vertex_location=vertex_location, + vertex_credentials=vertex_credentials, + ) + + return response + except Exception as e: + verbose_proxy_logger.error( + "litellm.proxy.proxy_server.v1/projects/tuningJobs(): Exception occurred - {}".format( + str(e) + ) + ) + verbose_proxy_logger.debug(traceback.format_exc()) + if isinstance(e, HTTPException): + raise ProxyException( + message=getattr(e, "message", str(e.detail)), + type=getattr(e, "type", "None"), + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), + ) + else: + error_msg = f"{str(e)}" + raise ProxyException( + message=getattr(e, "message", error_msg), + type=getattr(e, "type", "None"), + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", 500), + ) From 2e596c6c5b0ec8446745339f3c6a7351f727ad2f Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 3 Aug 2024 11:47:43 -0700 Subject: [PATCH 02/27] add support for pass through vertex ai ft jobs --- litellm/llms/fine_tuning_apis/vertex_ai.py | 42 ++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/litellm/llms/fine_tuning_apis/vertex_ai.py b/litellm/llms/fine_tuning_apis/vertex_ai.py index f370652d269..618894245d8 100644 --- a/litellm/llms/fine_tuning_apis/vertex_ai.py +++ b/litellm/llms/fine_tuning_apis/vertex_ai.py @@ -240,3 +240,45 @@ class VertexFineTuningAPI(VertexLLM): vertex_response ) return open_ai_response + + async def pass_through_vertex_ai_fine_tuning_job( + self, + request_data: dict, + vertex_project: str, + vertex_location: str, + vertex_credentials: str, + ): + auth_header, _ = self._get_token_and_url( + model="", + gemini_api_key=None, + vertex_credentials=vertex_credentials, + vertex_project=vertex_project, + vertex_location=vertex_location, + stream=False, + custom_llm_provider="vertex_ai_beta", + api_base="", + ) + + headers = { + "Authorization": f"Bearer {auth_header}", + "Content-Type": "application/json", + } + + fine_tuning_url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/tuningJobs" + + if self.async_handler is None: + raise ValueError("VertexAI Fine Tuning - async_handler is not initialized") + + response = await self.async_handler.post( + headers=headers, + url=fine_tuning_url, + json=request_data, # type: ignore + ) + + if response.status_code != 200: + raise Exception( + f"Error creating fine tuning job. Status code: {response.status_code}. Response: {response.text}" + ) + + response_json = response.json() + return response_json From af91740fbb0e3c9a9eede61d0303861cd9c811e8 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 3 Aug 2024 11:56:43 -0700 Subject: [PATCH 03/27] docs native vertex ft endpoint --- docs/my-website/docs/fine_tuning.md | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/docs/my-website/docs/fine_tuning.md b/docs/my-website/docs/fine_tuning.md index c69f4c1e660..fd3cbc792dc 100644 --- a/docs/my-website/docs/fine_tuning.md +++ b/docs/my-website/docs/fine_tuning.md @@ -124,7 +124,7 @@ ft_job = await client.fine_tuning.jobs.create( ``` - + ```shell curl http://localhost:4000/v1/fine_tuning/jobs \ @@ -136,6 +136,28 @@ curl http://localhost:4000/v1/fine_tuning/jobs \ "training_file": "gs://cloud-samples-data/ai-platform/generative_ai/sft_train_data.jsonl" }' ``` + + + + +:::info + +Use this to create Fine tuning Jobs in [the Vertex AI API Format](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/tuning#create-tuning) + +::: + +```shell +curl http://localhost:4000/v1/projects/tuningJobs \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "baseModel": "gemini-1.0-pro-002", + "supervisedTuningSpec" : { + "training_dataset_uri": "gs://cloud-samples-data/ai-platform/generative_ai/sft_train_data.jsonl" + } +}' +``` + From 6377304d9616c772dc2ce0b165bd066d310ba0a2 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 3 Aug 2024 12:24:23 -0700 Subject: [PATCH 04/27] docs supported models / providers --- docs/my-website/sidebars.js | 88 ++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 69fd32cb339..3e39348b971 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -83,50 +83,7 @@ const sidebars = { }, { type: "category", - label: "Completion()", - link: { - type: "generated-index", - title: "Completion()", - description: "Details on the completion() function", - slug: "/completion", - }, - items: [ - "completion/input", - "completion/provider_specific_params", - "completion/json_mode", - "completion/drop_params", - "completion/prompt_formatting", - "completion/output", - "exception_mapping", - "completion/stream", - "completion/message_trimming", - "completion/function_call", - "completion/vision", - "completion/model_alias", - "completion/batching", - "completion/mock_requests", - "completion/reliable_completions", - ], - }, - { - type: "category", - label: "Embedding(), Image Generation(), Assistants(), Moderation(), Audio Transcriptions(), TTS(), Batches(), Fine-Tuning()", - items: [ - "embedding/supported_embedding", - "embedding/async_embedding", - "embedding/moderation", - "image_generation", - "audio_transcription", - "text_to_speech", - "assistants", - "batches", - "fine_tuning", - "anthropic_completion" - ], - }, - { - type: "category", - label: "Supported Models & Providers", + label: "💯 Supported Models & Providers", link: { type: "generated-index", title: "Providers", @@ -183,6 +140,49 @@ const sidebars = { ], }, + { + type: "category", + label: "litellm.completion()", + link: { + type: "generated-index", + title: "Completion()", + description: "Details on the completion() function", + slug: "/completion", + }, + items: [ + "completion/input", + "completion/provider_specific_params", + "completion/json_mode", + "completion/drop_params", + "completion/prompt_formatting", + "completion/output", + "exception_mapping", + "completion/stream", + "completion/message_trimming", + "completion/function_call", + "completion/vision", + "completion/model_alias", + "completion/batching", + "completion/mock_requests", + "completion/reliable_completions", + ], + }, + { + type: "category", + label: "Embedding(), Image Generation(), Assistants(), Moderation(), Audio Transcriptions(), TTS(), Batches(), Fine-Tuning()", + items: [ + "embedding/supported_embedding", + "embedding/async_embedding", + "embedding/moderation", + "image_generation", + "audio_transcription", + "text_to_speech", + "assistants", + "batches", + "fine_tuning", + "anthropic_completion" + ], + }, "proxy/custom_pricing", "routing", "scheduler", From 4052b0d4c48e1614a0005ed190157b856dde98ab Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 3 Aug 2024 12:34:22 -0700 Subject: [PATCH 05/27] docs clean up organization --- docs/my-website/sidebars.js | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 3e39348b971..afb778373f1 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -184,10 +184,6 @@ const sidebars = { ], }, "proxy/custom_pricing", - "routing", - "scheduler", - "set_keys", - "budget_manager", { type: "category", label: "Secret Manager", @@ -196,6 +192,22 @@ const sidebars = { "oidc" ] }, + { + type: "category", + label: "🚅 LiteLLM Python SDK", + items: [ + "routing", + "scheduler", + "set_keys", + "budget_manager", + "caching/all_caches", + { + type: "category", + label: "LangChain, LlamaIndex, Instructor Integration", + items: ["langchain/langchain", "tutorials/instructor"], + }, + ], + }, "completion/token_usage", "load_test", { @@ -227,14 +239,12 @@ const sidebars = { `observability/telemetry`, ], }, - "caching/all_caches", { type: "category", label: "Tutorials", items: [ 'tutorials/azure_openai', 'tutorials/instructor', - 'tutorials/oobabooga', "tutorials/gradio_integration", "tutorials/huggingface_codellama", "tutorials/huggingface_tutorial", @@ -246,11 +256,6 @@ const sidebars = { "tutorials/model_fallbacks", ], }, - { - type: "category", - label: "LangChain, LlamaIndex, Instructor Integration", - items: ["langchain/langchain", "tutorials/instructor"], - }, { type: "category", label: "Extras", From 6faeac4cf6866dcc342e217710b155079f7f755b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 3 Aug 2024 12:47:22 -0700 Subject: [PATCH 06/27] organize docs --- docs/my-website/docs/proxy/custom_pricing.md | 69 ++------------------ docs/my-website/docs/sdk_custom_pricing.md | 65 ++++++++++++++++++ docs/my-website/sidebars.js | 5 +- 3 files changed, 72 insertions(+), 67 deletions(-) create mode 100644 docs/my-website/docs/sdk_custom_pricing.md diff --git a/docs/my-website/docs/proxy/custom_pricing.md b/docs/my-website/docs/proxy/custom_pricing.md index 0b747f1193b..51634021b73 100644 --- a/docs/my-website/docs/proxy/custom_pricing.md +++ b/docs/my-website/docs/proxy/custom_pricing.md @@ -1,6 +1,6 @@ import Image from '@theme/IdealImage'; -# Custom Pricing - Sagemaker, etc. +# Custom LLM Pricing - Sagemaker, Azure, etc Use this to register custom pricing for models. @@ -16,39 +16,9 @@ LiteLLM already has pricing for any model in our [model cost map](https://github ::: -## Quick Start +## Cost Per Second (e.g. Sagemaker) -Register custom pricing for sagemaker completion model. - -For cost per second pricing, you **just** need to register `input_cost_per_second`. - -```python -# !pip install boto3 -from litellm import completion, completion_cost - -os.environ["AWS_ACCESS_KEY_ID"] = "" -os.environ["AWS_SECRET_ACCESS_KEY"] = "" -os.environ["AWS_REGION_NAME"] = "" - - -def test_completion_sagemaker(): - try: - print("testing sagemaker") - response = completion( - model="sagemaker/berri-benchmarking-Llama-2-70b-chat-hf-4", - messages=[{"role": "user", "content": "Hey, how's it going?"}], - input_cost_per_second=0.000420, - ) - # Add any assertions here to check the response - print(response) - cost = completion_cost(completion_response=response) - print(cost) - except Exception as e: - raise Exception(f"Error occurred: {e}") - -``` - -### Usage with OpenAI Proxy Server +### Usage with LiteLLM Proxy Server **Step 1: Add pricing to config.yaml** ```yaml @@ -75,38 +45,7 @@ litellm /path/to/config.yaml ## Cost Per Token (e.g. Azure) - -```python -# !pip install boto3 -from litellm import completion, completion_cost - -## set ENV variables -os.environ["AZURE_API_KEY"] = "" -os.environ["AZURE_API_BASE"] = "" -os.environ["AZURE_API_VERSION"] = "" - - -def test_completion_azure_model(): - try: - print("testing azure custom pricing") - # azure call - response = completion( - model = "azure/", - messages = [{ "content": "Hello, how are you?","role": "user"}] - input_cost_per_token=0.005, - output_cost_per_token=1, - ) - # Add any assertions here to check the response - print(response) - cost = completion_cost(completion_response=response) - print(cost) - except Exception as e: - raise Exception(f"Error occurred: {e}") - -test_completion_azure_model() -``` - -### Usage with OpenAI Proxy Server +### Usage with LiteLLM Proxy Server ```yaml model_list: diff --git a/docs/my-website/docs/sdk_custom_pricing.md b/docs/my-website/docs/sdk_custom_pricing.md new file mode 100644 index 00000000000..c8577115109 --- /dev/null +++ b/docs/my-website/docs/sdk_custom_pricing.md @@ -0,0 +1,65 @@ +# Custom Pricing - SageMaker, Azure, etc + +Register custom pricing for sagemaker completion model. + +For cost per second pricing, you **just** need to register `input_cost_per_second`. + +```python +# !pip install boto3 +from litellm import completion, completion_cost + +os.environ["AWS_ACCESS_KEY_ID"] = "" +os.environ["AWS_SECRET_ACCESS_KEY"] = "" +os.environ["AWS_REGION_NAME"] = "" + + +def test_completion_sagemaker(): + try: + print("testing sagemaker") + response = completion( + model="sagemaker/berri-benchmarking-Llama-2-70b-chat-hf-4", + messages=[{"role": "user", "content": "Hey, how's it going?"}], + input_cost_per_second=0.000420, + ) + # Add any assertions here to check the response + print(response) + cost = completion_cost(completion_response=response) + print(cost) + except Exception as e: + raise Exception(f"Error occurred: {e}") + +``` + + +## Cost Per Token (e.g. Azure) + + +```python +# !pip install boto3 +from litellm import completion, completion_cost + +## set ENV variables +os.environ["AZURE_API_KEY"] = "" +os.environ["AZURE_API_BASE"] = "" +os.environ["AZURE_API_VERSION"] = "" + + +def test_completion_azure_model(): + try: + print("testing azure custom pricing") + # azure call + response = completion( + model = "azure/", + messages = [{ "content": "Hello, how are you?","role": "user"}] + input_cost_per_token=0.005, + output_cost_per_token=1, + ) + # Add any assertions here to check the response + print(response) + cost = completion_cost(completion_response=response) + print(cost) + except Exception as e: + raise Exception(f"Error occurred: {e}") + +test_completion_azure_model() +``` \ No newline at end of file diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index afb778373f1..6674d91ac79 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -42,6 +42,7 @@ const sidebars = { "proxy/configs", "proxy/reliability", "proxy/cost_tracking", + "proxy/custom_pricing", "proxy/self_serve", "proxy/virtual_keys", { @@ -183,7 +184,6 @@ const sidebars = { "anthropic_completion" ], }, - "proxy/custom_pricing", { type: "category", label: "Secret Manager", @@ -199,6 +199,8 @@ const sidebars = { "routing", "scheduler", "set_keys", + "completion/token_usage", + "sdk_custom_pricing", "budget_manager", "caching/all_caches", { @@ -208,7 +210,6 @@ const sidebars = { }, ], }, - "completion/token_usage", "load_test", { type: "category", From 5bc8b59b110c0f9c3432d869057e8d5819f1c2db Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 3 Aug 2024 12:49:35 -0700 Subject: [PATCH 07/27] docs - use consistent name for LiteLLM proxy server --- README.md | 4 +- cookbook/litellm_router/error_log.txt | 152 +++++++++--------- cookbook/litellm_router/request_log.txt | 4 +- .../test_questions/question3.txt | 2 +- docs/my-website/docs/budget_manager.md | 4 +- docs/my-website/docs/index.md | 4 +- docs/my-website/docs/proxy/deploy.md | 10 +- docs/my-website/docs/proxy_server.md | 2 +- docs/my-website/docs/routing.md | 4 +- docs/my-website/docs/secret.md | 4 +- docs/my-website/docs/simple_proxy_old_doc.md | 2 +- docs/my-website/sidebars.js | 4 +- docs/my-website/src/pages/index.md | 2 +- 13 files changed, 99 insertions(+), 99 deletions(-) diff --git a/README.md b/README.md index 306f07ec26f..2153ae948ed 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@

Call all LLM APIs using the OpenAI format [Bedrock, Huggingface, VertexAI, TogetherAI, Azure, OpenAI, Groq etc.]

-

OpenAI Proxy Server | Hosted Proxy (Preview) | Enterprise Tier

+

LiteLLM Proxy Server | Hosted Proxy (Preview) | Enterprise Tier

PyPI Version @@ -35,7 +35,7 @@ LiteLLM manages: - Translate inputs to provider's `completion`, `embedding`, and `image_generation` endpoints - [Consistent output](https://docs.litellm.ai/docs/completion/output), text responses will always be available at `['choices'][0]['message']['content']` - Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing) -- Set Budgets & Rate limits per project, api key, model [OpenAI Proxy Server](https://docs.litellm.ai/docs/simple_proxy) +- Set Budgets & Rate limits per project, api key, model [LiteLLM Proxy Server](https://docs.litellm.ai/docs/simple_proxy) [**Jump to OpenAI Proxy Docs**](https://github.com/BerriAI/litellm?tab=readme-ov-file#openai-proxy---docs)
[**Jump to Supported LLM Providers**](https://github.com/BerriAI/litellm?tab=readme-ov-file#supported-providers-docs) diff --git a/cookbook/litellm_router/error_log.txt b/cookbook/litellm_router/error_log.txt index 6853ef4659a..983b47cbbba 100644 --- a/cookbook/litellm_router/error_log.txt +++ b/cookbook/litellm_router/error_log.txt @@ -1,10 +1,10 @@ -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -21,13 +21,13 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -49,7 +49,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -61,7 +61,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -70,7 +70,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -79,7 +79,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -109,7 +109,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -128,7 +128,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -148,7 +148,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -162,7 +162,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -174,7 +174,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -184,7 +184,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -193,19 +193,19 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -214,7 +214,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -234,7 +234,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -244,7 +244,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -253,7 +253,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -267,31 +267,31 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -305,7 +305,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -330,7 +330,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -339,7 +339,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -360,7 +360,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -369,7 +369,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -378,7 +378,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -388,7 +388,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -409,7 +409,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -422,13 +422,13 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -438,7 +438,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -462,7 +462,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -482,7 +482,7 @@ Exception: 'Response' object has no attribute 'get' Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -492,7 +492,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -516,7 +516,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -529,7 +529,7 @@ Exception: 'Response' object has no attribute 'get' Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -546,13 +546,13 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -580,13 +580,13 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -624,7 +624,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -638,13 +638,13 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -660,7 +660,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -681,7 +681,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -691,31 +691,31 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -771,7 +771,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -780,7 +780,7 @@ Exception: 'Response' object has no attribute 'get' Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -800,7 +800,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -820,7 +820,7 @@ Exception: 'Response' object has no attribute 'get' Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -830,7 +830,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -840,7 +840,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -850,7 +850,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -862,13 +862,13 @@ Exception: 'Response' object has no attribute 'get' Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -877,7 +877,7 @@ Exception: 'Response' object has no attribute 'get' Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -898,7 +898,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -919,7 +919,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -936,19 +936,19 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -961,25 +961,25 @@ Exception: 'Response' object has no attribute 'get' Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -993,7 +993,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 diff --git a/cookbook/litellm_router/request_log.txt b/cookbook/litellm_router/request_log.txt index 0aed7490496..821d87ab56a 100644 --- a/cookbook/litellm_router/request_log.txt +++ b/cookbook/litellm_router/request_log.txt @@ -20,7 +20,7 @@ Call all LLM APIs using the OpenAI format. Response ID: 52dbbd49-eedb-4c11-8382-3ca7deb1af35 Url: /queue/response/52dbbd49-eedb-4c11-8382-3ca7deb1af35 Time: 3.50 seconds -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -35,7 +35,7 @@ Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. C Response ID: ae1e2b71-d711-456d-8df0-13ce0709eb04 Url: /queue/response/ae1e2b71-d711-456d-8df0-13ce0709eb04 Time: 5.60 seconds -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 diff --git a/cookbook/litellm_router/test_questions/question3.txt b/cookbook/litellm_router/test_questions/question3.txt index a122787504e..d6006f9c73c 100644 --- a/cookbook/litellm_router/test_questions/question3.txt +++ b/cookbook/litellm_router/test_questions/question3.txt @@ -1,4 +1,4 @@ -What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 100+ LLMs Huggingface/Bedrock/TogetherAI/etc. in the OpenAI ChatCompletions & Completions format diff --git a/docs/my-website/docs/budget_manager.md b/docs/my-website/docs/budget_manager.md index 1a2c7e7eecd..6bea96ef9ce 100644 --- a/docs/my-website/docs/budget_manager.md +++ b/docs/my-website/docs/budget_manager.md @@ -7,14 +7,14 @@ Don't want to get crazy bills because either while you're calling LLM APIs **or* :::info -If you want a server to manage user keys, budgets, etc. use our [OpenAI Proxy Server](./proxy/virtual_keys.md) +If you want a server to manage user keys, budgets, etc. use our [LiteLLM Proxy Server](./proxy/virtual_keys.md) ::: LiteLLM exposes: * `litellm.max_budget`: a global variable you can use to set the max budget (in USD) across all your litellm calls. If this budget is exceeded, it will raise a BudgetExceededError * `BudgetManager`: A class to help set budgets per user. BudgetManager creates a dictionary to manage the user budgets, where the key is user and the object is their current cost + model-specific costs. -* `OpenAI Proxy Server`: A server to call 100+ LLMs with an openai-compatible endpoint. Manages user budgets, spend tracking, load balancing etc. +* `LiteLLM Proxy Server`: A server to call 100+ LLMs with an openai-compatible endpoint. Manages user budgets, spend tracking, load balancing etc. ## quick start diff --git a/docs/my-website/docs/index.md b/docs/my-website/docs/index.md index 6b472ee6c6d..a560ecf76da 100644 --- a/docs/my-website/docs/index.md +++ b/docs/my-website/docs/index.md @@ -10,11 +10,11 @@ https://github.com/BerriAI/litellm - Translate inputs to provider's `completion`, `embedding`, and `image_generation` endpoints - [Consistent output](https://docs.litellm.ai/docs/completion/output), text responses will always be available at `['choices'][0]['message']['content']` - Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing) -- Track spend & set budgets per project [OpenAI Proxy Server](https://docs.litellm.ai/docs/simple_proxy) +- Track spend & set budgets per project [LiteLLM Proxy Server](https://docs.litellm.ai/docs/simple_proxy) ## How to use LiteLLM You can use litellm through either: -1. [OpenAI proxy Server](#openai-proxy) - Server to call 100+ LLMs, load balance, cost tracking across projects +1. [LiteLLM Proxy Server](#openai-proxy) - Server to call 100+ LLMs, load balance, cost tracking across projects 2. [LiteLLM python SDK](#basic-usage) - Python Client to call 100+ LLMs, load balance, cost tracking ## LiteLLM Python SDK diff --git a/docs/my-website/docs/proxy/deploy.md b/docs/my-website/docs/proxy/deploy.md index 35fc0a50866..c7617196e58 100644 --- a/docs/my-website/docs/proxy/deploy.md +++ b/docs/my-website/docs/proxy/deploy.md @@ -246,7 +246,7 @@ helm install lite-helm ./litellm-helm kubectl --namespace default port-forward $POD_NAME 8080:$CONTAINER_PORT ``` -Your OpenAI proxy server is now running on `http://127.0.0.1:4000`. +Your LiteLLM Proxy Server is now running on `http://127.0.0.1:4000`. @@ -301,7 +301,7 @@ docker run \ --config /app/config.yaml --detailed_debug ``` -Your OpenAI proxy server is now running on `http://0.0.0.0:4000`. +Your LiteLLM Proxy Server is now running on `http://0.0.0.0:4000`. @@ -399,7 +399,7 @@ kubectl apply -f /path/to/service.yaml kubectl port-forward service/litellm-service 4000:4000 ``` -Your OpenAI proxy server is now running on `http://0.0.0.0:4000`. +Your LiteLLM Proxy Server is now running on `http://0.0.0.0:4000`. @@ -441,7 +441,7 @@ kubectl \ 4000:4000 ``` -Your OpenAI proxy server is now running on `http://127.0.0.1:4000`. +Your LiteLLM Proxy Server is now running on `http://127.0.0.1:4000`. If you need to set your litellm proxy config.yaml, you can find this in [values.yaml](https://github.com/BerriAI/litellm/blob/main/deploy/charts/litellm-helm/values.yaml) @@ -486,7 +486,7 @@ helm install lite-helm ./litellm-helm kubectl --namespace default port-forward $POD_NAME 8080:$CONTAINER_PORT ``` -Your OpenAI proxy server is now running on `http://127.0.0.1:4000`. +Your LiteLLM Proxy Server is now running on `http://127.0.0.1:4000`. diff --git a/docs/my-website/docs/proxy_server.md b/docs/my-website/docs/proxy_server.md index ef9352ab1f7..0d08db7444e 100644 --- a/docs/my-website/docs/proxy_server.md +++ b/docs/my-website/docs/proxy_server.md @@ -1,7 +1,7 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# [OLD PROXY 👉 [NEW proxy here](./simple_proxy)] Local OpenAI Proxy Server +# [OLD PROXY 👉 [NEW proxy here](./simple_proxy)] Local LiteLLM Proxy Server A fast, and lightweight OpenAI-compatible server to call 100+ LLM APIs. diff --git a/docs/my-website/docs/routing.md b/docs/my-website/docs/routing.md index 905954e9797..d83755e68dd 100644 --- a/docs/my-website/docs/routing.md +++ b/docs/my-website/docs/routing.md @@ -14,7 +14,7 @@ In production, litellm supports using Redis as a way to track cooldown server an :::info -If you want a server to load balance across different LLM APIs, use our [OpenAI Proxy Server](./proxy/load_balancing.md) +If you want a server to load balance across different LLM APIs, use our [LiteLLM Proxy Server](./proxy/load_balancing.md) ::: @@ -1637,7 +1637,7 @@ response = router.completion( ## Deploy Router -If you want a server to load balance across different LLM APIs, use our [OpenAI Proxy Server](./simple_proxy#load-balancing---multiple-instances-of-1-model) +If you want a server to load balance across different LLM APIs, use our [LiteLLM Proxy Server](./simple_proxy#load-balancing---multiple-instances-of-1-model) ## Init Params for the litellm.Router diff --git a/docs/my-website/docs/secret.md b/docs/my-website/docs/secret.md index 91ae383686e..c44f2cd10cb 100644 --- a/docs/my-website/docs/secret.md +++ b/docs/my-website/docs/secret.md @@ -90,7 +90,7 @@ litellm.secret_manager = client litellm.get_secret("your-test-key") ``` -### Usage with OpenAI Proxy Server +### Usage with LiteLLM Proxy Server 1. Install Proxy dependencies ```bash @@ -129,7 +129,7 @@ litellm --config /path/to/config.yaml Use encrypted keys from Google KMS on the proxy -### Usage with OpenAI Proxy Server +### Usage with LiteLLM Proxy Server ## Step 1. Add keys to env ``` diff --git a/docs/my-website/docs/simple_proxy_old_doc.md b/docs/my-website/docs/simple_proxy_old_doc.md index 195728d1be4..2d68db32964 100644 --- a/docs/my-website/docs/simple_proxy_old_doc.md +++ b/docs/my-website/docs/simple_proxy_old_doc.md @@ -2,7 +2,7 @@ import Image from '@theme/IdealImage'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# 💥 OpenAI Proxy Server +# 💥 LiteLLM Proxy Server LiteLLM Server manages: diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 6674d91ac79..e57f340c70c 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -20,10 +20,10 @@ const sidebars = { { type: "doc", id: "index" }, // NEW { type: "category", - label: "💥 OpenAI Proxy Server", + label: "💥 LiteLLM Proxy Server", link: { type: "generated-index", - title: "💥 OpenAI Proxy Server", + title: "💥 LiteLLM Proxy Server", description: `Proxy Server to call 100+ LLMs in a unified interface & track spend, set budgets per virtual key/user`, slug: "/simple_proxy", }, diff --git a/docs/my-website/src/pages/index.md b/docs/my-website/src/pages/index.md index 308ed083175..36d47aedf7c 100644 --- a/docs/my-website/src/pages/index.md +++ b/docs/my-website/src/pages/index.md @@ -10,7 +10,7 @@ https://github.com/BerriAI/litellm - Translate inputs to provider's `completion`, `embedding`, and `image_generation` endpoints - [Consistent output](https://docs.litellm.ai/docs/completion/output), text responses will always be available at `['choices'][0]['message']['content']` - Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing) -- Track spend & set budgets per project [OpenAI Proxy Server](https://docs.litellm.ai/docs/simple_proxy) +- Track spend & set budgets per project [LiteLLM Proxy Server](https://docs.litellm.ai/docs/simple_proxy) ## Basic usage From 641192e0494f0d5ca7f116da6fb7161798e727d1 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 3 Aug 2024 12:53:35 -0700 Subject: [PATCH 08/27] fix(vertex_httpx.py): fix linting error --- litellm/llms/vertex_httpx.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/litellm/llms/vertex_httpx.py b/litellm/llms/vertex_httpx.py index 9995373f32e..954a30b8010 100644 --- a/litellm/llms/vertex_httpx.py +++ b/litellm/llms/vertex_httpx.py @@ -13,6 +13,7 @@ from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Union import httpx # type: ignore import requests # type: ignore +from openai.types.image import Image import litellm import litellm.litellm_core_utils @@ -1341,10 +1342,10 @@ class VertexLLM(BaseLLM): _json_response = response.json() _predictions = _json_response["predictions"] - _response_data: List[litellm.ImageObject] = [] + _response_data: List[Image] = [] for _prediction in _predictions: _bytes_base64_encoded = _prediction["bytesBase64Encoded"] - image_object = litellm.ImageObject(b64_json=_bytes_base64_encoded) + image_object = Image(b64_json=_bytes_base64_encoded) _response_data.append(image_object) model_response.data = _response_data @@ -1453,10 +1454,10 @@ class VertexLLM(BaseLLM): _json_response = response.json() _predictions = _json_response["predictions"] - _response_data: List[litellm.ImageObject] = [] + _response_data: List[Image] = [] for _prediction in _predictions: _bytes_base64_encoded = _prediction["bytesBase64Encoded"] - image_object = litellm.ImageObject(b64_json=_bytes_base64_encoded) + image_object = Image(b64_json=_bytes_base64_encoded) _response_data.append(image_object) model_response.data = _response_data From 39a98a288297692bc041777c37c7b7f993c78deb Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 3 Aug 2024 12:53:47 -0700 Subject: [PATCH 09/27] docs secret manager --- docs/my-website/docs/secret.md | 31 ++++--------------------------- docs/my-website/sidebars.js | 16 ++++++++-------- 2 files changed, 12 insertions(+), 35 deletions(-) diff --git a/docs/my-website/docs/secret.md b/docs/my-website/docs/secret.md index c44f2cd10cb..c2b6774c0b1 100644 --- a/docs/my-website/docs/secret.md +++ b/docs/my-website/docs/secret.md @@ -61,7 +61,7 @@ litellm --config /path/to/config.yaml ``` ## Azure Key Vault - + ### Usage with LiteLLM Proxy Server @@ -160,29 +160,6 @@ $ litellm --test [Quick Test Proxy](./proxy/quick_start#using-litellm-proxy---curl-request-openai-package-langchain-langchain-js) - -## Infisical Secret Manager -Integrates with [Infisical's Secret Manager](https://infisical.com/) for secure storage and retrieval of API keys and sensitive data. - -### Usage -liteLLM manages reading in your LLM API secrets/env variables from Infisical for you - -```python -import litellm -from infisical import InfisicalClient - -litellm.secret_manager = InfisicalClient(token="your-token") - -messages = [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What's the weather like today?"}, -] - -response = litellm.completion(model="gpt-3.5-turbo", messages=messages) - -print(response) -``` - - + diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index e57f340c70c..27084f3b451 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -50,6 +50,14 @@ const sidebars = { label: "🪢 Logging", items: ["proxy/logging", "proxy/bucket", "proxy/streaming_logging"], }, + { + type: "category", + label: "Secret Manager - storing LLM API Keys", + items: [ + "secret", + "oidc" + ] + }, "proxy/team_logging", "proxy/guardrails", "proxy/tag_routing", @@ -184,14 +192,6 @@ const sidebars = { "anthropic_completion" ], }, - { - type: "category", - label: "Secret Manager", - items: [ - "secret", - "oidc" - ] - }, { type: "category", label: "🚅 LiteLLM Python SDK", From af04bbeccaf713a4fe6beca531b1596737456f3c Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 3 Aug 2024 16:52:03 -0700 Subject: [PATCH 10/27] set native vertex endpoints --- litellm/proxy/proxy_server.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 0f57a5fd13b..83126b95451 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -213,6 +213,8 @@ from litellm.proxy.utils import ( send_email, update_spend, ) +from litellm.proxy.vertex_ai_endpoints.vertex_endpoints import router as vertex_router +from litellm.proxy.vertex_ai_endpoints.vertex_endpoints import set_default_vertex_config from litellm.router import ( AssistantsTypedDict, Deployment, @@ -1818,6 +1820,10 @@ class ProxyConfig: files_config = config.get("files_settings", None) set_files_config(config=files_config) + ## default config for vertex ai routes + default_vertex_config = config.get("default_vertex_config", None) + set_default_vertex_config(config=default_vertex_config) + ## ROUTER SETTINGS (e.g. routing_strategy, ...) router_settings = config.get("router_settings", None) if router_settings and isinstance(router_settings, dict): @@ -9631,6 +9637,7 @@ def cleanup_router_config_variables(): app.include_router(router) app.include_router(fine_tuning_router) +app.include_router(vertex_router) app.include_router(health_router) app.include_router(key_management_router) app.include_router(internal_user_router) From 64c008045f2c0bd1812dcd85a7585b06bf0dfdd9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 3 Aug 2024 16:52:43 -0700 Subject: [PATCH 11/27] use native endpoints --- litellm/llms/fine_tuning_apis/vertex_ai.py | 9 +- .../proxy/fine_tuning_endpoints/endpoints.py | 69 ---------- litellm/proxy/proxy_config.yaml | 5 + .../vertex_ai_endpoints/vertex_endpoints.py | 120 ++++++++++++++++++ 4 files changed, 131 insertions(+), 72 deletions(-) create mode 100644 litellm/proxy/vertex_ai_endpoints/vertex_endpoints.py diff --git a/litellm/llms/fine_tuning_apis/vertex_ai.py b/litellm/llms/fine_tuning_apis/vertex_ai.py index 618894245d8..189ace11ada 100644 --- a/litellm/llms/fine_tuning_apis/vertex_ai.py +++ b/litellm/llms/fine_tuning_apis/vertex_ai.py @@ -241,12 +241,13 @@ class VertexFineTuningAPI(VertexLLM): ) return open_ai_response - async def pass_through_vertex_ai_fine_tuning_job( + async def pass_through_vertex_ai_POST_request( self, request_data: dict, vertex_project: str, vertex_location: str, vertex_credentials: str, + request_route: str, ): auth_header, _ = self._get_token_and_url( model="", @@ -264,14 +265,16 @@ class VertexFineTuningAPI(VertexLLM): "Content-Type": "application/json", } - fine_tuning_url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/tuningJobs" + url = None + if request_route == "tuningJobs": + url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/tuningJobs" if self.async_handler is None: raise ValueError("VertexAI Fine Tuning - async_handler is not initialized") response = await self.async_handler.post( headers=headers, - url=fine_tuning_url, + url=url, json=request_data, # type: ignore ) diff --git a/litellm/proxy/fine_tuning_endpoints/endpoints.py b/litellm/proxy/fine_tuning_endpoints/endpoints.py index c2d89dd251f..cda226b5aa7 100644 --- a/litellm/proxy/fine_tuning_endpoints/endpoints.py +++ b/litellm/proxy/fine_tuning_endpoints/endpoints.py @@ -429,72 +429,3 @@ async def retrieve_fine_tuning_job( param=getattr(e, "param", "None"), code=getattr(e, "status_code", 500), ) - - -@router.post( - "/v1/projects/tuningJobs", - dependencies=[Depends(user_api_key_auth)], - tags=["fine-tuning"], - summary="✨ (Enterprise) Create Fine-Tuning Jobs", -) -async def vertex_create_fine_tuning_job( - request: Request, - fastapi_response: Response, - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), -): - """ - this is a pass through endpoint for the Vertex AI API. /tuningJobs endpoint - - it uses the vertex ai credentials on the proxy and forwards to vertex ai api - """ - try: - from litellm.fine_tuning.main import vertex_fine_tuning_apis_instance - from litellm.proxy.proxy_server import ( - add_litellm_data_to_request, - general_settings, - get_custom_headers, - premium_user, - proxy_config, - proxy_logging_obj, - version, - ) - - # get configs for custom_llm_provider - llm_provider_config = get_fine_tuning_provider_config( - custom_llm_provider="vertex_ai" - ) - - vertex_project = llm_provider_config.get("vertex_project", None) - vertex_location = llm_provider_config.get("vertex_location", None) - vertex_credentials = llm_provider_config.get("vertex_credentials", None) - request_data_json = await request.json() - response = await vertex_fine_tuning_apis_instance.pass_through_vertex_ai_fine_tuning_job( - request_data=request_data_json, - vertex_project=vertex_project, - vertex_location=vertex_location, - vertex_credentials=vertex_credentials, - ) - - return response - except Exception as e: - verbose_proxy_logger.error( - "litellm.proxy.proxy_server.v1/projects/tuningJobs(): Exception occurred - {}".format( - str(e) - ) - ) - verbose_proxy_logger.debug(traceback.format_exc()) - if isinstance(e, HTTPException): - raise ProxyException( - message=getattr(e, "message", str(e.detail)), - type=getattr(e, "type", "None"), - param=getattr(e, "param", "None"), - code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), - ) - else: - error_msg = f"{str(e)}" - raise ProxyException( - message=getattr(e, "message", error_msg), - type=getattr(e, "type", "None"), - param=getattr(e, "param", "None"), - code=getattr(e, "status_code", 500), - ) diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index aa2bfc5252d..0750a393763 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -48,6 +48,11 @@ files_settings: - custom_llm_provider: openai api_key: os.environ/OPENAI_API_KEY +default_vertex_config: + vertex_project: "adroit-crow-413218" + vertex_location: "us-central1" + vertex_credentials: "/Users/ishaanjaffer/Downloads/adroit-crow-413218-a956eef1a2a8.json" + general_settings: diff --git a/litellm/proxy/vertex_ai_endpoints/vertex_endpoints.py b/litellm/proxy/vertex_ai_endpoints/vertex_endpoints.py new file mode 100644 index 00000000000..be09a4932d1 --- /dev/null +++ b/litellm/proxy/vertex_ai_endpoints/vertex_endpoints.py @@ -0,0 +1,120 @@ +import asyncio +import traceback +from datetime import datetime, timedelta, timezone +from typing import List, Optional + +import fastapi +import httpx +from fastapi import ( + APIRouter, + Depends, + File, + Form, + Header, + HTTPException, + Request, + Response, + UploadFile, + status, +) + +import litellm +from litellm._logging import verbose_proxy_logger +from litellm.batches.main import FileObject +from litellm.fine_tuning.main import vertex_fine_tuning_apis_instance +from litellm.proxy._types import * +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + +router = APIRouter() +default_vertex_config = None + + +def set_default_vertex_config(config): + global default_vertex_config + if config is None: + return + + if not isinstance(config, list): + raise ValueError("invalid files config, expected a list is not a list") + + for element in config: + if isinstance(element, dict): + for key, value in element.items(): + if isinstance(value, str) and value.startswith("os.environ/"): + element[key] = litellm.get_secret(value) + + default_vertex_config = config + + +def exception_handler(e: Exception): + verbose_proxy_logger.error( + "litellm.proxy.proxy_server.v1/projects/tuningJobs(): Exception occurred - {}".format( + str(e) + ) + ) + verbose_proxy_logger.debug(traceback.format_exc()) + if isinstance(e, HTTPException): + return ProxyException( + message=getattr(e, "message", str(e.detail)), + type=getattr(e, "type", "None"), + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), + ) + else: + error_msg = f"{str(e)}" + return ProxyException( + message=getattr(e, "message", error_msg), + type=getattr(e, "type", "None"), + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", 500), + ) + + +async def execute_post_vertex_ai_request( + request: Request, + route: str, +): + from litellm.fine_tuning.main import vertex_fine_tuning_apis_instance + + vertex_project = default_vertex_config.get("vertex_project", None) + vertex_location = default_vertex_config.get("vertex_location", None) + vertex_credentials = default_vertex_config.get("vertex_credentials", None) + request_data_json = await request.json() + + response = ( + await vertex_fine_tuning_apis_instance.pass_through_vertex_ai_POST_request( + request_data=request_data_json, + vertex_project=vertex_project, + vertex_location=vertex_location, + vertex_credentials=vertex_credentials, + request_route=route, + ) + ) + + return response + + +@router.post( + "/vertex-ai/tuningJobs", + dependencies=[Depends(user_api_key_auth)], + tags=["Vertex AI endpoints"], +) +async def vertex_create_fine_tuning_job( + request: Request, + fastapi_response: Response, + endpoint_name: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + this is a pass through endpoint for the Vertex AI API. /tuningJobs endpoint + + it uses the vertex ai credentials on the proxy and forwards to vertex ai api + """ + try: + response = await execute_post_vertex_ai_request( + request=request, + route="/tuningJobs", + ) + return response + except Exception as e: + raise exception_handler(e) from e From 8051781af015bbbfd2c52a80067bd7078cef8358 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 3 Aug 2024 16:56:33 -0700 Subject: [PATCH 12/27] working code for vertex ai routes --- litellm/llms/fine_tuning_apis/vertex_ai.py | 2 +- .../proxy/vertex_ai_endpoints/vertex_endpoints.py | 14 ++++++-------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/litellm/llms/fine_tuning_apis/vertex_ai.py b/litellm/llms/fine_tuning_apis/vertex_ai.py index 189ace11ada..cb238b04b74 100644 --- a/litellm/llms/fine_tuning_apis/vertex_ai.py +++ b/litellm/llms/fine_tuning_apis/vertex_ai.py @@ -266,7 +266,7 @@ class VertexFineTuningAPI(VertexLLM): } url = None - if request_route == "tuningJobs": + if request_route == "/tuningJobs": url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/tuningJobs" if self.async_handler is None: diff --git a/litellm/proxy/vertex_ai_endpoints/vertex_endpoints.py b/litellm/proxy/vertex_ai_endpoints/vertex_endpoints.py index be09a4932d1..659459cec39 100644 --- a/litellm/proxy/vertex_ai_endpoints/vertex_endpoints.py +++ b/litellm/proxy/vertex_ai_endpoints/vertex_endpoints.py @@ -34,14 +34,13 @@ def set_default_vertex_config(config): if config is None: return - if not isinstance(config, list): - raise ValueError("invalid files config, expected a list is not a list") + if not isinstance(config, dict): + raise ValueError("invalid config, vertex default config must be a dictionary") - for element in config: - if isinstance(element, dict): - for key, value in element.items(): - if isinstance(value, str) and value.startswith("os.environ/"): - element[key] = litellm.get_secret(value) + if isinstance(config, dict): + for key, value in config.items(): + if isinstance(value, str) and value.startswith("os.environ/"): + config[key] = litellm.get_secret(value) default_vertex_config = config @@ -102,7 +101,6 @@ async def execute_post_vertex_ai_request( async def vertex_create_fine_tuning_job( request: Request, fastapi_response: Response, - endpoint_name: str, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ From c98733863afed9154be4d91ccff6434ec8297e56 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 3 Aug 2024 17:17:54 -0700 Subject: [PATCH 13/27] add vertex generateContent --- litellm/llms/fine_tuning_apis/vertex_ai.py | 4 + .../vertex_ai_endpoints/vertex_endpoints.py | 83 ++++++++++++++++++- 2 files changed, 86 insertions(+), 1 deletion(-) diff --git a/litellm/llms/fine_tuning_apis/vertex_ai.py b/litellm/llms/fine_tuning_apis/vertex_ai.py index cb238b04b74..2d3e8d1c38c 100644 --- a/litellm/llms/fine_tuning_apis/vertex_ai.py +++ b/litellm/llms/fine_tuning_apis/vertex_ai.py @@ -268,6 +268,10 @@ class VertexFineTuningAPI(VertexLLM): url = None if request_route == "/tuningJobs": url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/tuningJobs" + elif "/tuningJobs/" in request_route and "cancel" in request_route: + url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/tuningJobs{request_route}" + elif "generateContent" in request_route: + url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}{request_route}" if self.async_handler is None: raise ValueError("VertexAI Fine Tuning - async_handler is not initialized") diff --git a/litellm/proxy/vertex_ai_endpoints/vertex_endpoints.py b/litellm/proxy/vertex_ai_endpoints/vertex_endpoints.py index 659459cec39..7ef552508bd 100644 --- a/litellm/proxy/vertex_ai_endpoints/vertex_endpoints.py +++ b/litellm/proxy/vertex_ai_endpoints/vertex_endpoints.py @@ -1,3 +1,4 @@ +import ast import asyncio import traceback from datetime import datetime, timedelta, timezone @@ -78,7 +79,21 @@ async def execute_post_vertex_ai_request( vertex_project = default_vertex_config.get("vertex_project", None) vertex_location = default_vertex_config.get("vertex_location", None) vertex_credentials = default_vertex_config.get("vertex_credentials", None) - request_data_json = await request.json() + + request_data_json = {} + body = await request.body() + body_str = body.decode() + if len(body_str) > 0: + try: + request_data_json = ast.literal_eval(body_str) + except: + request_data_json = json.loads(body_str) + + verbose_proxy_logger.debug( + "Request received by LiteLLM:\n{}".format( + json.dumps(request_data_json, indent=4) + ), + ) response = ( await vertex_fine_tuning_apis_instance.pass_through_vertex_ai_POST_request( @@ -93,6 +108,41 @@ async def execute_post_vertex_ai_request( return response +@router.post( + "/vertex-ai/publishers/google/models/{model_id:path}:generateContent", + dependencies=[Depends(user_api_key_auth)], + tags=["Vertex AI endpoints"], +) +async def vertex_generate_content( + request: Request, + fastapi_response: Response, + model_id: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + this is a pass through endpoint for the Vertex AI API. /generateContent endpoint + + Example Curl: + ``` + curl http://localhost:4000/vertex-ai/publishers/google/models/gemini-1.5-flash-001:generateContent \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{"contents":[{"role": "user", "parts":[{"text": "hi"}]}]}' + ``` + + Vertex API Reference: https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/inference#rest + it uses the vertex ai credentials on the proxy and forwards to vertex ai api + """ + try: + response = await execute_post_vertex_ai_request( + request=request, + route=f"/publishers/google/models/{model_id}:generateContent", + ) + return response + except Exception as e: + raise exception_handler(e) from e + + @router.post( "/vertex-ai/tuningJobs", dependencies=[Depends(user_api_key_auth)], @@ -106,6 +156,8 @@ async def vertex_create_fine_tuning_job( """ this is a pass through endpoint for the Vertex AI API. /tuningJobs endpoint + Vertex API Reference: https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/tuning + it uses the vertex ai credentials on the proxy and forwards to vertex ai api """ try: @@ -116,3 +168,32 @@ async def vertex_create_fine_tuning_job( return response except Exception as e: raise exception_handler(e) from e + + +@router.post( + "/vertex-ai/tuningJobs/{job_id:path}:cancel", + dependencies=[Depends(user_api_key_auth)], + tags=["Vertex AI endpoints"], +) +async def vertex_cancel_fine_tuning_job( + request: Request, + job_id: str, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + this is a pass through endpoint for the Vertex AI API. tuningJobs/{job_id:path}:cancel + + Vertex API Reference: https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/tuning#cancel_a_tuning_job + + it uses the vertex ai credentials on the proxy and forwards to vertex ai api + """ + try: + + response = await execute_post_vertex_ai_request( + request=request, + route=f"/tuningJobs/{job_id}:cancel", + ) + return response + except Exception as e: + raise exception_handler(e) from e From 2d5c57e5457c09ed902dc18914a65953da87983a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 3 Aug 2024 17:26:49 -0700 Subject: [PATCH 14/27] add vertex embeddings endpoints --- litellm/llms/fine_tuning_apis/vertex_ai.py | 3 +- .../vertex_ai_endpoints/vertex_endpoints.py | 39 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/litellm/llms/fine_tuning_apis/vertex_ai.py b/litellm/llms/fine_tuning_apis/vertex_ai.py index 2d3e8d1c38c..f964e7bba22 100644 --- a/litellm/llms/fine_tuning_apis/vertex_ai.py +++ b/litellm/llms/fine_tuning_apis/vertex_ai.py @@ -272,7 +272,8 @@ class VertexFineTuningAPI(VertexLLM): url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/tuningJobs{request_route}" elif "generateContent" in request_route: url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}{request_route}" - + elif "predict" in request_route: + url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}{request_route}" if self.async_handler is None: raise ValueError("VertexAI Fine Tuning - async_handler is not initialized") diff --git a/litellm/proxy/vertex_ai_endpoints/vertex_endpoints.py b/litellm/proxy/vertex_ai_endpoints/vertex_endpoints.py index 7ef552508bd..da63aa02505 100644 --- a/litellm/proxy/vertex_ai_endpoints/vertex_endpoints.py +++ b/litellm/proxy/vertex_ai_endpoints/vertex_endpoints.py @@ -143,6 +143,45 @@ async def vertex_generate_content( raise exception_handler(e) from e +@router.post( + "/vertex-ai/publishers/google/models/{model_id:path}:predict", + dependencies=[Depends(user_api_key_auth)], + tags=["Vertex AI endpoints"], +) +async def vertex_predict_endpoint( + request: Request, + fastapi_response: Response, + model_id: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + this is a pass through endpoint for the Vertex AI API. /predict endpoint + Use this for: + - Embeddings API - Text Embedding, Multi Modal Embedding + - Imagen API + - Code Completion API + + Example Curl: + ``` + curl http://localhost:4000/vertex-ai/publishers/google/models/textembedding-gecko@001:predict \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{"instances":[{"content": "gm"}]}' + ``` + + Vertex API Reference: https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/text-embeddings-api#generative-ai-get-text-embedding-drest + it uses the vertex ai credentials on the proxy and forwards to vertex ai api + """ + try: + response = await execute_post_vertex_ai_request( + request=request, + route=f"/publishers/google/models/{model_id}:predict", + ) + return response + except Exception as e: + raise exception_handler(e) from e + + @router.post( "/vertex-ai/tuningJobs", dependencies=[Depends(user_api_key_auth)], From c8438715af437a336573808ee913660d0dbad130 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 3 Aug 2024 17:34:10 -0700 Subject: [PATCH 15/27] add vertex ai countTokens endpoint --- litellm/llms/fine_tuning_apis/vertex_ai.py | 5 ++ .../vertex_ai_endpoints/vertex_endpoints.py | 63 +++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/litellm/llms/fine_tuning_apis/vertex_ai.py b/litellm/llms/fine_tuning_apis/vertex_ai.py index f964e7bba22..c24deca941b 100644 --- a/litellm/llms/fine_tuning_apis/vertex_ai.py +++ b/litellm/llms/fine_tuning_apis/vertex_ai.py @@ -274,6 +274,11 @@ class VertexFineTuningAPI(VertexLLM): url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}{request_route}" elif "predict" in request_route: url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}{request_route}" + elif "/batchPredictionJobs" in request_route: + url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}{request_route}" + elif "countTokens" in request_route: + url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}{request_route}" + if self.async_handler is None: raise ValueError("VertexAI Fine Tuning - async_handler is not initialized") diff --git a/litellm/proxy/vertex_ai_endpoints/vertex_endpoints.py b/litellm/proxy/vertex_ai_endpoints/vertex_endpoints.py index da63aa02505..c7ce354b25f 100644 --- a/litellm/proxy/vertex_ai_endpoints/vertex_endpoints.py +++ b/litellm/proxy/vertex_ai_endpoints/vertex_endpoints.py @@ -182,6 +182,69 @@ async def vertex_predict_endpoint( raise exception_handler(e) from e +@router.post( + "/vertex-ai/publishers/google/models/{model_id:path}:countTokens", + dependencies=[Depends(user_api_key_auth)], + tags=["Vertex AI endpoints"], +) +async def vertex_countTokens_endpoint( + request: Request, + fastapi_response: Response, + model_id: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + this is a pass through endpoint for the Vertex AI API. /countTokens endpoint + https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/count-tokens#curl + + + Example Curl: + ``` + curl http://localhost:4000/vertex-ai/publishers/google/models/gemini-1.5-flash-001:countTokens \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{"contents":[{"role": "user", "parts":[{"text": "hi"}]}]}' + ``` + + it uses the vertex ai credentials on the proxy and forwards to vertex ai api + """ + try: + response = await execute_post_vertex_ai_request( + request=request, + route=f"/publishers/google/models/{model_id}:countTokens", + ) + return response + except Exception as e: + raise exception_handler(e) from e + + +@router.post( + "/vertex-ai/batchPredictionJobs", + dependencies=[Depends(user_api_key_auth)], + tags=["Vertex AI endpoints"], +) +async def vertex_create_batch_prediction_job( + request: Request, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + this is a pass through endpoint for the Vertex AI API. /batchPredictionJobs endpoint + + Vertex API Reference: https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/batch-prediction-api#syntax + + it uses the vertex ai credentials on the proxy and forwards to vertex ai api + """ + try: + response = await execute_post_vertex_ai_request( + request=request, + route="/batchPredictionJobs", + ) + return response + except Exception as e: + raise exception_handler(e) from e + + @router.post( "/vertex-ai/tuningJobs", dependencies=[Depends(user_api_key_auth)], From 94e5d0f73437086e4b5f8864f9f80873bcd58a20 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 3 Aug 2024 17:41:45 -0700 Subject: [PATCH 16/27] docs add vertex ai endpoints --- docs/my-website/docs/vertex_ai.md | 40 +++++++++++++++++++++++++++++++ docs/my-website/sidebars.js | 5 ++-- 2 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 docs/my-website/docs/vertex_ai.md diff --git a/docs/my-website/docs/vertex_ai.md b/docs/my-website/docs/vertex_ai.md new file mode 100644 index 00000000000..c18044c2044 --- /dev/null +++ b/docs/my-website/docs/vertex_ai.md @@ -0,0 +1,40 @@ +# [BETA] Vertex AI Endpoints + +## Supported APIs + +- Gemini API +- Embeddings API +- Imagen API +- Code Completion API +- Batch prediction API +- Tuning API +- CountTokens API + +## Quick Start Usage + +#### 1. Set `default_vertex_config` on your `config.yaml` + + +Add the following credentials to your litellm config.yaml to use the Vertex AI endpoints. + +```yaml +default_vertex_config: + vertex_project: "adroit-crow-413218" + vertex_location: "us-central1" + vertex_credentials: "/Users/ishaanjaffer/Downloads/adroit-crow-413218-a956eef1a2a8.json" # Add path to service account.json +``` + +#### 2. Start litellm proxy + +```shell +litellm --config /path/to/config.yaml +``` + +#### 3. Test it + +```shell +curl http://localhost:4000/vertex-ai/publishers/google/models/textembedding-gecko@001:countTokens \ +-H "Content-Type: application/json" \ +-H "Authorization: Bearer sk-1234" \ +-d '{"instances":[{"content": "gm"}]}' +``` \ No newline at end of file diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 69fd32cb339..a57a9aecd7d 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -110,7 +110,7 @@ const sidebars = { }, { type: "category", - label: "Embedding(), Image Generation(), Assistants(), Moderation(), Audio Transcriptions(), TTS(), Batches(), Fine-Tuning()", + label: "Supported Endpoints - /images, /audio/speech, /assistants etc", items: [ "embedding/supported_embedding", "embedding/async_embedding", @@ -121,7 +121,8 @@ const sidebars = { "assistants", "batches", "fine_tuning", - "anthropic_completion" + "anthropic_completion", + "vertex_ai" ], }, { From 50c9fa38f902b54e6cd8f98823636fe7203a3cd7 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 3 Aug 2024 17:44:28 -0700 Subject: [PATCH 17/27] docs link to vertex ai endpoints --- docs/my-website/docs/proxy/user_keys.md | 3 +++ docs/my-website/docs/vertex_ai.md | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/my-website/docs/proxy/user_keys.md b/docs/my-website/docs/proxy/user_keys.md index 75e547d17e1..79d019a20cd 100644 --- a/docs/my-website/docs/proxy/user_keys.md +++ b/docs/my-website/docs/proxy/user_keys.md @@ -23,6 +23,9 @@ LiteLLM Proxy is **Azure OpenAI-compatible**: LiteLLM Proxy is **Anthropic-compatible**: * /messages +LiteLLM Proxy is **Vertex AI compatible**: +- [Supports ALL Vertex Endpoints](../vertex_ai) + This doc covers: * /chat/completion diff --git a/docs/my-website/docs/vertex_ai.md b/docs/my-website/docs/vertex_ai.md index c18044c2044..2c9a6279a1c 100644 --- a/docs/my-website/docs/vertex_ai.md +++ b/docs/my-website/docs/vertex_ai.md @@ -1,6 +1,6 @@ # [BETA] Vertex AI Endpoints -## Supported APIs +## Supported API Endpoints - Gemini API - Embeddings API From 7f95a865fefe925b5d98ab7c46d8e433cb02dec1 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 3 Aug 2024 17:56:38 -0700 Subject: [PATCH 18/27] docs add example curl command --- docs/my-website/docs/vertex_ai.md | 37 +++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/docs/my-website/docs/vertex_ai.md b/docs/my-website/docs/vertex_ai.md index 2c9a6279a1c..7ae06f3af73 100644 --- a/docs/my-website/docs/vertex_ai.md +++ b/docs/my-website/docs/vertex_ai.md @@ -37,4 +37,41 @@ curl http://localhost:4000/vertex-ai/publishers/google/models/textembedding-geck -H "Content-Type: application/json" \ -H "Authorization: Bearer sk-1234" \ -d '{"instances":[{"content": "gm"}]}' +``` + + +### Gemini API (Generate Content) + +```shell +curl http://localhost:4000/vertex-ai/publishers/google/models/gemini-1.5-flash-001:generateContent \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{"contents":[{"role": "user", "parts":[{"text": "hi"}]}]}' +``` + +### Embeddings API + +```shell +curl http://localhost:4000/vertex-ai/publishers/google/models/textembedding-gecko@001:predict \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{"instances":[{"content": "gm"}]}' +``` + +### Imagen API + +```shell +curl http://localhost:4000/vertex-ai/publishers/google/models/imagen-3.0-generate-001:predict \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{"instances":[{"prompt": "make an otter"}], "parameters": {"sampleCount": 1}}' +``` + +### Count Tokens API + +```shell +curl http://localhost:4000/vertex-ai/publishers/google/models/gemini-1.5-flash-001:countTokens \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{"contents":[{"role": "user", "parts":[{"text": "hi"}]}]}' ``` \ No newline at end of file From 9d2eab555ba679b0bd41594556d3f8184cc80bc3 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 3 Aug 2024 17:58:20 -0700 Subject: [PATCH 19/27] docs tuning api --- docs/my-website/docs/vertex_ai.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/my-website/docs/vertex_ai.md b/docs/my-website/docs/vertex_ai.md index 7ae06f3af73..d9c8616a0bf 100644 --- a/docs/my-website/docs/vertex_ai.md +++ b/docs/my-website/docs/vertex_ai.md @@ -38,7 +38,7 @@ curl http://localhost:4000/vertex-ai/publishers/google/models/textembedding-geck -H "Authorization: Bearer sk-1234" \ -d '{"instances":[{"content": "gm"}]}' ``` - +## Usage Examples ### Gemini API (Generate Content) @@ -74,4 +74,20 @@ curl http://localhost:4000/vertex-ai/publishers/google/models/gemini-1.5-flash-0 -H "Content-Type: application/json" \ -H "Authorization: Bearer sk-1234" \ -d '{"contents":[{"role": "user", "parts":[{"text": "hi"}]}]}' +``` + +### Tuning API + +Create Fine Tuning Job + +```shell +curl http://localhost:4000/vertex-ai/tuningJobs \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "baseModel": "gemini-1.0-pro-002", + "supervisedTuningSpec" : { + "training_dataset_uri": "gs://cloud-samples-data/ai-platform/generative_ai/sft_train_data.jsonl" + } +}' ``` \ No newline at end of file From 3a94aac34389762b7fb81b17ec2a1b9c9041a726 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 3 Aug 2024 17:59:46 -0700 Subject: [PATCH 20/27] docs - fix merge conflicts --- docs/my-website/sidebars.js | 49 +++---------------------------------- 1 file changed, 3 insertions(+), 46 deletions(-) diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 65483e3925a..6f6bcfeeab2 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -90,50 +90,6 @@ const sidebars = { "proxy/cli", ] }, - { - type: "category", - label: "Completion()", - link: { - type: "generated-index", - title: "Completion()", - description: "Details on the completion() function", - slug: "/completion", - }, - items: [ - "completion/input", - "completion/provider_specific_params", - "completion/json_mode", - "completion/drop_params", - "completion/prompt_formatting", - "completion/output", - "exception_mapping", - "completion/stream", - "completion/message_trimming", - "completion/function_call", - "completion/vision", - "completion/model_alias", - "completion/batching", - "completion/mock_requests", - "completion/reliable_completions", - ], - }, - { - type: "category", - label: "Supported Endpoints - /images, /audio/speech, /assistants etc", - items: [ - "embedding/supported_embedding", - "embedding/async_embedding", - "embedding/moderation", - "image_generation", - "audio_transcription", - "text_to_speech", - "assistants", - "batches", - "fine_tuning", - "anthropic_completion", - "vertex_ai" - ], - }, { type: "category", label: "💯 Supported Models & Providers", @@ -222,7 +178,7 @@ const sidebars = { }, { type: "category", - label: "Embedding(), Image Generation(), Assistants(), Moderation(), Audio Transcriptions(), TTS(), Batches(), Fine-Tuning()", + label: "Supported Endpoints - /images, /audio/speech, /assistants etc", items: [ "embedding/supported_embedding", "embedding/async_embedding", @@ -233,7 +189,8 @@ const sidebars = { "assistants", "batches", "fine_tuning", - "anthropic_completion" + "anthropic_completion", + "vertex_ai" ], }, { From e73eb19678979e2b9e11ae62cbb9247cffb78c91 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 3 Aug 2024 18:09:48 -0700 Subject: [PATCH 21/27] docs default vertex --- litellm/llms/fine_tuning_apis/vertex_ai.py | 3 ++- litellm/proxy/vertex_ai_endpoints/vertex_endpoints.py | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/litellm/llms/fine_tuning_apis/vertex_ai.py b/litellm/llms/fine_tuning_apis/vertex_ai.py index c24deca941b..5f96f04831a 100644 --- a/litellm/llms/fine_tuning_apis/vertex_ai.py +++ b/litellm/llms/fine_tuning_apis/vertex_ai.py @@ -278,7 +278,8 @@ class VertexFineTuningAPI(VertexLLM): url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}{request_route}" elif "countTokens" in request_route: url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}{request_route}" - + else: + raise ValueError(f"Unsupported Vertex AI request route: {request_route}") if self.async_handler is None: raise ValueError("VertexAI Fine Tuning - async_handler is not initialized") diff --git a/litellm/proxy/vertex_ai_endpoints/vertex_endpoints.py b/litellm/proxy/vertex_ai_endpoints/vertex_endpoints.py index c7ce354b25f..b8c04583c3f 100644 --- a/litellm/proxy/vertex_ai_endpoints/vertex_endpoints.py +++ b/litellm/proxy/vertex_ai_endpoints/vertex_endpoints.py @@ -76,6 +76,10 @@ async def execute_post_vertex_ai_request( ): from litellm.fine_tuning.main import vertex_fine_tuning_apis_instance + if default_vertex_config is None: + raise ValueError( + "Vertex credentials not added on litellm proxy, please add `default_vertex_config` on your config.yaml" + ) vertex_project = default_vertex_config.get("vertex_project", None) vertex_location = default_vertex_config.get("vertex_location", None) vertex_credentials = default_vertex_config.get("vertex_credentials", None) From b6cf433ed9ceaf5bd5c9e74161906eae73e2646e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 3 Aug 2024 18:12:38 -0700 Subject: [PATCH 22/27] =?UTF-8?q?bump:=20version=201.42.11=20=E2=86=92=201?= =?UTF-8?q?.42.12?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6293b77fb01..a803c8e0fb8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.42.11" +version = "1.42.12" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -91,7 +91,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.42.11" +version = "1.42.12" version_files = [ "pyproject.toml:^version" ] From 1c1631222c8428fee37cd992aa7fbee8a161a059 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 3 Aug 2024 18:32:52 -0700 Subject: [PATCH 23/27] fix test test_aimage_generation_vertex_ai --- litellm/tests/test_image_generation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/tests/test_image_generation.py b/litellm/tests/test_image_generation.py index e59cf286519..6648b202b9d 100644 --- a/litellm/tests/test_image_generation.py +++ b/litellm/tests/test_image_generation.py @@ -7,6 +7,7 @@ import sys import traceback from dotenv import load_dotenv +from openai.types.image import Image logging.basicConfig(level=logging.DEBUG) load_dotenv() @@ -218,7 +219,7 @@ async def test_aimage_generation_vertex_ai(sync_mode): assert len(response.data) > 0 for d in response.data: - assert isinstance(d, litellm.ImageObject) + assert isinstance(d, Image) print("data in response.data", d) assert d.b64_json is not None except litellm.ServiceUnavailableError as e: From 009697425d15ad27b513ea3b365a11145170fbe9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 3 Aug 2024 18:34:26 -0700 Subject: [PATCH 24/27] fix fine tuning tests --- litellm/tests/test_fine_tuning_api.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/litellm/tests/test_fine_tuning_api.py b/litellm/tests/test_fine_tuning_api.py index 20a58c4d003..f02af1b9276 100644 --- a/litellm/tests/test_fine_tuning_api.py +++ b/litellm/tests/test_fine_tuning_api.py @@ -80,6 +80,8 @@ def test_create_fine_tune_job(): except openai.RateLimitError: pass except Exception as e: + if "Job has already completed" in str(e): + pass pytest.fail(f"Error occurred: {e}") From 7fae2aa394c403182b2537eb6e25b599a24b1aee Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 3 Aug 2024 18:48:10 -0700 Subject: [PATCH 25/27] ci/cd run again --- litellm/tests/test_completion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index c26035ad0ad..eec163f26a4 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -23,7 +23,7 @@ from litellm import RateLimitError, Timeout, completion, completion_cost, embedd from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.prompt_templates.factory import anthropic_messages_pt -# litellm.num_retries = 3 +# litellm.num_retries=3 litellm.cache = None litellm.success_callback = [] user_message = "Write a short poem about the sky" From 93b0d239979cc1e653058773c3970dd06ac5f2c6 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 3 Aug 2024 19:02:11 -0700 Subject: [PATCH 26/27] fix fine tune api tests --- litellm/tests/test_fine_tuning_api.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/litellm/tests/test_fine_tuning_api.py b/litellm/tests/test_fine_tuning_api.py index f02af1b9276..412ffb497c1 100644 --- a/litellm/tests/test_fine_tuning_api.py +++ b/litellm/tests/test_fine_tuning_api.py @@ -81,8 +81,9 @@ def test_create_fine_tune_job(): pass except Exception as e: if "Job has already completed" in str(e): - pass - pytest.fail(f"Error occurred: {e}") + return + else: + pytest.fail(f"Error occurred: {e}") @pytest.mark.asyncio @@ -137,7 +138,7 @@ async def test_create_fine_tune_jobs_async(): pass except Exception as e: if "Job has already completed" in str(e): - pass + return else: pytest.fail(f"Error occurred: {e}") pass From 22bc70633e5c42f7789d8a42418659b2b39a77eb Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 3 Aug 2024 19:10:16 -0700 Subject: [PATCH 27/27] docs add when to use litellm --- docs/my-website/docs/index.md | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/docs/my-website/docs/index.md b/docs/my-website/docs/index.md index a560ecf76da..dffc9501a8e 100644 --- a/docs/my-website/docs/index.md +++ b/docs/my-website/docs/index.md @@ -10,14 +10,40 @@ https://github.com/BerriAI/litellm - Translate inputs to provider's `completion`, `embedding`, and `image_generation` endpoints - [Consistent output](https://docs.litellm.ai/docs/completion/output), text responses will always be available at `['choices'][0]['message']['content']` - Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing) -- Track spend & set budgets per project [LiteLLM Proxy Server](https://docs.litellm.ai/docs/simple_proxy) +- Track spend & set budgets per project [OpenAI Proxy Server](https://docs.litellm.ai/docs/simple_proxy) ## How to use LiteLLM You can use litellm through either: -1. [LiteLLM Proxy Server](#openai-proxy) - Server to call 100+ LLMs, load balance, cost tracking across projects +1. [OpenAI proxy Server](#openai-proxy) - Server to call 100+ LLMs, load balance, cost tracking across projects 2. [LiteLLM python SDK](#basic-usage) - Python Client to call 100+ LLMs, load balance, cost tracking -## LiteLLM Python SDK +### When to use LiteLLM Proxy Server + +:::tip + +Use LiteLLM Proxy Server if you want a **central service to access multiple LLMs** + +Typically used by Gen AI Enablement / ML PLatform Teams + +::: + + - LiteLLM Proxy gives you a unified interface to access multiple LLMs (100+ LLMs) + - Track LLM Usage and setup guardrails + - Customize Logging, Guardrails, Caching per project + +### When to use LiteLLM Python SDK + +:::tip + + Use LiteLLM Python SDK if you want to use LiteLLM in your **python code** + +Typically used by developers building llm projects + +::: + + - LiteLLM SDK gives you a unified interface to access multiple LLMs (100+ LLMs) + - Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing) + ### Basic usage